From 1a70d971de58c8f56dddc1ef8fd25b5330b67500 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20H=C3=BCbner?= <martin.hubner@web.de>
Date: Thu, 5 Aug 2021 21:21:27 +0200
Subject: [PATCH] build_all_targets: limit concurrent jobs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This commit adds some code that limits the amount of docker
containers that run in parallel. This will hopefully fix the
nondeterministically missing files that we have seen in the
packagefeed.

Signed-off-by: Martin Hübner <martin.hubner@web.de>
---
 build_all_targets | 12 +++++++++++-
 build_feed        |  4 ++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/build_all_targets b/build_all_targets
index d9f2674..b60d5d5 100755
--- a/build_all_targets
+++ b/build_all_targets
@@ -5,6 +5,10 @@ FEED="$2"
 OUTPUT_DIR="$3"
 BUILD_PARALLEL="$4"
 
+MAX_JOBS=6
+CUR_JOBS=0
+export COUNT_FILE="/tmp/count.tmp"
+
 # check for dependencies
 DEPS="docker-hub docker"
 
@@ -53,9 +57,15 @@ archs+=( ["x86_64"]=x86-64 )
 for key in ${!archs[@]}; do
 	SDK="${archs[${key}]}"
 	[ "$OPENWRT_VERSION" != "snapshot" ] && SDK+="-$OPENWRT_VERSION"
-	if [ -n "$BUILD_PARALLEL" ]; then
+	if [ -n "$BUILD_PARALLEL" ] ; then
+		CUR_JOBS=$(( $CUR_JOBS + 1 ))
+		echo $CUR_JOBS > $COUNT_FILE
 		(./build_feed "$SDK" "$FEED" "$OUTPUT_DIR" PARALLEL) &
 	else
 		./build_feed "$SDK" "$FEED" "$OUTPUT_DIR"
 	fi
+	while [ $CUR_JOBS -ge $MAX_JOBS ]; do
+		sleep 20
+		CUR_JOBS=$(cat $COUNT_FILE)
+	done
 done
diff --git a/build_feed b/build_feed
index 8e6e8d5..9009956 100755
--- a/build_feed
+++ b/build_feed
@@ -43,5 +43,9 @@ else
       "$IMAGE" \
       ./build_feed incontainer
     fi
+    # decrement job counter
+    CUR_JOBS=$(cat $COUNT_FILE)
+    CUR_JOBS=$(( $CUR_JOBS -1 ))
+    echo $CUR_JOBS > $COUNT_FILE
 fi