From d42efdf420ceaaee2f4842d770bdc14a163155f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20H=C3=BCbner?= <martin.hubner@web.de>
Date: Thu, 5 Aug 2021 21:21:27 +0200
Subject: [PATCH] build_all_targets: limit concurrent jobs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This commit adds some code that limits the amount of docker
containers that run in parallel. This will hopefully fix the
nondeterministically missing files that we have seen in the
packagefeed.

Signed-off-by: Martin Hübner <martin.hubner@web.de>
---
 build_all_targets | 10 ++++++++++
 build_feed        |  4 ++++
 2 files changed, 14 insertions(+)

diff --git a/build_all_targets b/build_all_targets
index d9f2674..f9d5142 100755
--- a/build_all_targets
+++ b/build_all_targets
@@ -5,6 +5,10 @@ FEED="$2"
 OUTPUT_DIR="$3"
 BUILD_PARALLEL="$4"
 
+MAX_JOBS=6
+CUR_JOBS=0
+export COUNT_FILE="/tmp/count.tmp"
+
 # check for dependencies
 DEPS="docker-hub docker"
 
@@ -54,8 +58,14 @@ for key in ${!archs[@]}; do
 	SDK="${archs[${key}]}"
 	[ "$OPENWRT_VERSION" != "snapshot" ] && SDK+="-$OPENWRT_VERSION"
 	if [ -n "$BUILD_PARALLEL" ]; then
+		CUR_JOBS=$(( $CUR_JOBS + 1 ))
+		echo $CUR_JOBS > $COUNT_FILE
 		(./build_feed "$SDK" "$FEED" "$OUTPUT_DIR" PARALLEL) &
 	else
 		./build_feed "$SDK" "$FEED" "$OUTPUT_DIR"
 	fi
+	while [ $CUR_JOBS -ge $MAX_JOBS ]; do
+		sleep 20
+		CUR_JOBS=$(cat $COUNT_FILE)
+	done
 done
diff --git a/build_feed b/build_feed
index 8e6e8d5..9009956 100755
--- a/build_feed
+++ b/build_feed
@@ -43,5 +43,9 @@ else
       "$IMAGE" \
       ./build_feed incontainer
     fi
+    # decrement job counter
+    CUR_JOBS=$(cat $COUNT_FILE)
+    CUR_JOBS=$(( $CUR_JOBS -1 ))
+    echo $CUR_JOBS > $COUNT_FILE
 fi