From 6ef7fa5d570d701f30ad3891d8d6c6df44884ca0 Mon Sep 17 00:00:00 2001 From: Ani Balasubramaniam Date: Tue, 2 Jun 2020 00:41:45 -0700 Subject: [PATCH 1/9] Autmagically use 1386/debian:buster when running on 64-bit host to prevent error #271 --- Dockerfile | 3 ++- build-docker.sh | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2e53149b52..3d5874d17b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM debian:buster +ARG BASE_IMAGE=debian:buster +FROM ${BASE_IMAGE} ENV DEBIAN_FRONTEND noninteractive diff --git a/build-docker.sh b/build-docker.sh index b6a9ea3d81..6ab8f4c77b 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -73,7 +73,10 @@ fi # Modify original build-options to allow config file to be mounted in the docker container BUILD_OPTS="$(echo "${BUILD_OPTS:-}" | sed -E 's@\-c\s?([^ ]+)@-c /config@')" -${DOCKER} build -t pi-gen "${DIR}" +ARCH=$(uname -m) +[[ $ARCH == "x86_64" || $ARCH == "aarch64" ]] && BASE_IMAGE=i386/debian:buster || BASE_IMAGE=debian:buster +${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}" + if [ "${CONTAINER_EXISTS}" != "" ]; then trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}_cont' SIGINT SIGTERM time ${DOCKER} run --rm --privileged \ From c1d1015c9b08ca56c493f49f9399523f24f3023b Mon Sep 17 00:00:00 2001 From: Ani Balasubramaniam Date: Tue, 2 Jun 2020 00:47:37 -0700 Subject: [PATCH 2/9] Add comment --- build-docker.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build-docker.sh b/build-docker.sh index 6ab8f4c77b..7ba31949a2 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -73,6 +73,7 @@ fi # Modify original build-options to allow config file to be mounted in the docker container BUILD_OPTS="$(echo "${BUILD_OPTS:-}" | sed -E 's@\-c\s?([^ ]+)@-c /config@')" +# Check the arch of the machine we're running on. If it's 64-bit, use a 32-bit base image instead ARCH=$(uname -m) [[ $ARCH == "x86_64" || $ARCH == "aarch64" ]] && BASE_IMAGE=i386/debian:buster || BASE_IMAGE=debian:buster ${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}" From 4f5dfc4d1e4c1ccd25d015ebad4e43fc4c414385 Mon Sep 17 00:00:00 2001 From: Ani Balasubramaniam Date: Tue, 2 Jun 2020 08:58:25 -0700 Subject: [PATCH 3/9] Fix conditional to read better using a case statement --- build-docker.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/build-docker.sh b/build-docker.sh index 7ba31949a2..350f722bde 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -74,8 +74,14 @@ fi BUILD_OPTS="$(echo "${BUILD_OPTS:-}" | sed -E 's@\-c\s?([^ ]+)@-c /config@')" # Check the arch of the machine we're running on. If it's 64-bit, use a 32-bit base image instead -ARCH=$(uname -m) -[[ $ARCH == "x86_64" || $ARCH == "aarch64" ]] && BASE_IMAGE=i386/debian:buster || BASE_IMAGE=debian:buster +case "$(uname -m)" in + x86_64|aarch64) + BASE_IMAGE=i386/debian:buster + ;; + *) + BASE_IMAGE=debian:buster + ;; +esac ${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}" if [ "${CONTAINER_EXISTS}" != "" ]; then From b4d215eb168ee6a47a4211b41b0e9ee1dd6f17b1 Mon Sep 17 00:00:00 2001 From: Ani Balasubramaniam Date: Wed, 3 Jun 2020 12:33:44 -0700 Subject: [PATCH 4/9] Add the ability to configure additional mounts when running the image for custom stages, binaries, etc. --- build-docker.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/build-docker.sh b/build-docker.sh index 350f722bde..f332dd2252 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -84,12 +84,20 @@ case "$(uname -m)" in esac ${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}" +MOUNTS="" +for mount in ${ADDL_MOUNTS:=""} +do + echo $mount + MOUNTS="${MOUNTS} --volume ${mount}" +done +echo ${MOUNTS} + if [ "${CONTAINER_EXISTS}" != "" ]; then trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}_cont' SIGINT SIGTERM time ${DOCKER} run --rm --privileged \ --volume "${CONFIG_FILE}":/config:ro \ -e "GIT_HASH=${GIT_HASH}" \ - --volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" \ + --volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" ${MOUNTS} \ pi-gen \ bash -e -o pipefail -c "dpkg-reconfigure qemu-user-static && cd /pi-gen; ./build.sh ${BUILD_OPTS} && From 869edae286f0548a8328ebd4348c39d79461d00e Mon Sep 17 00:00:00 2001 From: Ani Balasubramaniam Date: Wed, 3 Jun 2020 14:29:30 -0700 Subject: [PATCH 5/9] Fix additional mount from configuration --- build-docker.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/build-docker.sh b/build-docker.sh index f332dd2252..f17f6fb152 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -87,7 +87,6 @@ ${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}" MOUNTS="" for mount in ${ADDL_MOUNTS:=""} do - echo $mount MOUNTS="${MOUNTS} --volume ${mount}" done echo ${MOUNTS} @@ -95,9 +94,9 @@ echo ${MOUNTS} if [ "${CONTAINER_EXISTS}" != "" ]; then trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}_cont' SIGINT SIGTERM time ${DOCKER} run --rm --privileged \ - --volume "${CONFIG_FILE}":/config:ro \ + --volume "${CONFIG_FILE}":/config:ro ${MOUNTS} \ -e "GIT_HASH=${GIT_HASH}" \ - --volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" ${MOUNTS} \ + --volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" \ pi-gen \ bash -e -o pipefail -c "dpkg-reconfigure qemu-user-static && cd /pi-gen; ./build.sh ${BUILD_OPTS} && @@ -106,7 +105,7 @@ if [ "${CONTAINER_EXISTS}" != "" ]; then else trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}' SIGINT SIGTERM time ${DOCKER} run --name "${CONTAINER_NAME}" --privileged \ - --volume "${CONFIG_FILE}":/config:ro \ + --volume "${CONFIG_FILE}":/config:ro ${MOUNTS} \ -e "GIT_HASH=${GIT_HASH}" \ pi-gen \ bash -e -o pipefail -c "dpkg-reconfigure qemu-user-static && From a40b72b353a947008812efca67be2608a617377b Mon Sep 17 00:00:00 2001 From: Ani Balasubramaniam Date: Thu, 15 Apr 2021 08:38:30 -0700 Subject: [PATCH 6/9] 1. Add ability to mount custom stages into the docker build process so we can build custom images without having to modify pi-gen 2. Add ability to pass in key=value pairs that can be accessed as custom environment variables in config, allowing injection of secrets into a custom image for wi-fi setup or other purposes. Together, this commit allows a repository using an un-modified version of this commit as a submodule to create a fully working, bootable, connect-able, custom image for a raspberry pi that needs NO manual setup. --- build-docker.sh | 51 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/build-docker.sh b/build-docker.sh index b698e93db5..9cfc3d9e96 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -31,6 +31,33 @@ do esac done +# Ability to pass in name=value pairs that will be available to config +# as well as sent to the docker run command as environment variables. +# This is so we can use command line arguments (or workflow secrets) +# to build a ready-to-use image without mucking around with saved configs +# and thus expose private information + +# Get rest of unparsed arguments +shift $(expr $OPTIND - 1 ) + +# Convert it into an array +ADDL_ENV=() +while test $# -gt 0; do + ADDL_ENV+=("$1") + shift +done + +# Parse key=value pairs and export them +while IFS='=' read -r name value; do + # Handle double-quoted ("...") values. + if [[ $value =~ ^\"(.*)\"$ ]]; then + # Using `read` without `-r` removes the \ from embedded \ sequences. + IFS= read value <<<"${BASH_REMATCH[1]}" + fi + # Export so config can access these values + export "${name}=${value}" +done <<< $( IFS=$'\n'; echo "${ADDL_ENV[*]}" ) + # Ensure that the configuration file is an absolute path if test -x /usr/bin/realpath; then CONFIG_FILE=$(realpath -s "$CONFIG_FILE" || realpath "$CONFIG_FILE") @@ -85,13 +112,27 @@ case "$(uname -m)" in esac ${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}" -MOUNTS="" +# Ability to add additional mounts so that custom stages can be mounted into the build process +# without modifying the pi-gen repository +DOCKER_ADDL_MOUNTS="" echo "Processing additional mounts..." for mount in ${ADDL_MOUNTS:=""} do - MOUNTS="${MOUNTS} --volume ${mount}" + DOCKER_ADDL_MOUNTS="${DOCKER_ADDL_MOUNTS} --volume ${mount}" done +# Pass in any additional env that we were given to the docker run commands +DOCKER_ADDL_ENV= +while IFS='=' read -r name value; do + # Handle double-quoted ("...") values. + if [[ $value =~ ^\"(.*)\"$ ]]; then + # Using `read` without `-r` removes the \ from embedded \ sequences. + IFS= read value <<<"${BASH_REMATCH[1]}" + fi + # Append to a string in docker env format + DOCKER_ADDL_ENV="${DOCKER_ADDL_ENV} -e ${name}=${value}" +done <<< $( IFS=$'\n'; echo "${ADDL_ENV[*]}" ) + if [ "${CONTAINER_EXISTS}" != "" ]; then trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}_cont' SIGINT SIGTERM time ${DOCKER} run --rm --privileged \ @@ -99,7 +140,8 @@ if [ "${CONTAINER_EXISTS}" != "" ]; then -v /dev:/dev \ -v /lib/modules:/lib/modules \ --volume "${CONFIG_FILE}":/config:ro \ - ${MOUNTS} \ + ${DOCKER_ADDL_MOUNTS} \ + ${DOCKER_ADDL_ENV} \ -e "GIT_HASH=${GIT_HASH}" \ --volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" \ pi-gen \ @@ -114,7 +156,8 @@ else -v /dev:/dev \ -v /lib/modules:/lib/modules \ --volume "${CONFIG_FILE}":/config:ro \ - ${MOUNTS} \ + ${DOCKER_ADDL_MOUNTS} \ + ${DOCKER_ADDL_ENV} \ -e "GIT_HASH=${GIT_HASH}" \ pi-gen \ bash -e -o pipefail -c "dpkg-reconfigure qemu-user-static && From 52fd636f6e5238db2e7ad50fe19fda1d75c6c78d Mon Sep 17 00:00:00 2001 From: Ani Balasubramaniam Date: Thu, 15 Apr 2021 22:23:00 -0700 Subject: [PATCH 7/9] Add wpasupplicant to packages list, we need it to generate wpa_supplicant.conf during the image build process. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index cb2b4ba785..74b3be1652 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,7 @@ RUN apt-get -y update && \ quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \ bsdtar libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc\ binfmt-support ca-certificates qemu-utils kpartx \ + wpasupplicant \ && rm -rf /var/lib/apt/lists/* COPY . /pi-gen/ From 82f67bf47103aca6b58b3ede14cb8383e5522cf9 Mon Sep 17 00:00:00 2001 From: Ani Balasubramaniam Date: Mon, 19 Apr 2021 17:19:08 -0700 Subject: [PATCH 8/9] Fix quote handling in ADDL_ENV --- build-docker.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build-docker.sh b/build-docker.sh index 9cfc3d9e96..aad95f3df7 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -43,7 +43,7 @@ shift $(expr $OPTIND - 1 ) # Convert it into an array ADDL_ENV=() while test $# -gt 0; do - ADDL_ENV+=("$1") + ADDL_ENV+=($1) shift done @@ -122,7 +122,8 @@ do done # Pass in any additional env that we were given to the docker run commands -DOCKER_ADDL_ENV= +DOCKER_ADDL_ENV="" +echo "Processing additional env..." while IFS='=' read -r name value; do # Handle double-quoted ("...") values. if [[ $value =~ ^\"(.*)\"$ ]]; then From 80bf59602d1ec4620a427a84a86da807dbc53ea7 Mon Sep 17 00:00:00 2001 From: Ani Balasubramaniam Date: Thu, 22 Apr 2021 23:08:11 -0700 Subject: [PATCH 9/9] Refactor of mount and env handling to read better --- build-docker.sh | 132 ++++++++++++++++++++++++++++-------------------- 1 file changed, 76 insertions(+), 56 deletions(-) diff --git a/build-docker.sh b/build-docker.sh index aad95f3df7..0f6ca88e7b 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -1,6 +1,6 @@ #!/bin/bash -eu -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" BUILD_OPTS="$*" @@ -20,14 +20,13 @@ if [ -f "${DIR}/config" ]; then CONFIG_FILE="${DIR}/config" fi -while getopts "c:" flag -do +while getopts "c:" flag; do case "${flag}" in - c) - CONFIG_FILE="${OPTARG}" - ;; - *) - ;; + c) + CONFIG_FILE="${OPTARG}" + ;; + *) ;; + esac done @@ -37,26 +36,35 @@ done # to build a ready-to-use image without mucking around with saved configs # and thus expose private information +# Process from command-line +CMDLINE_ENV=() + # Get rest of unparsed arguments -shift $(expr $OPTIND - 1 ) +shift $(expr $OPTIND - 1) -# Convert it into an array -ADDL_ENV=() +# Append to our array while test $# -gt 0; do - ADDL_ENV+=($1) - shift + CMDLINE_ENV+=($1) + shift done -# Parse key=value pairs and export them +# Either way, we may have some key=value pairs now +# parse and export them so config can get them +echo "Exporting command-line environment variables..." while IFS='=' read -r name value; do - # Handle double-quoted ("...") values. - if [[ $value =~ ^\"(.*)\"$ ]]; then - # Using `read` without `-r` removes the \ from embedded \ sequences. - IFS= read value <<<"${BASH_REMATCH[1]}" - fi - # Export so config can access these values - export "${name}=${value}" -done <<< $( IFS=$'\n'; echo "${ADDL_ENV[*]}" ) + + # Handle double-quoted ("...") values. + if [[ $value =~ ^\"(.*)\"$ ]]; then + # Using `read` without `-r` removes the \ from embedded \ sequences. + IFS= read value <<<"${BASH_REMATCH[1]}" + fi + + # Export so config can access these values + export "${name}=${value}" +done <<<$( + IFS=$'\n' + echo "${CMDLINE_ENV[*]}" +) # Ensure that the configuration file is an absolute path if test -x /usr/bin/realpath; then @@ -65,13 +73,47 @@ fi # Ensure that the confguration file is present if test -z "${CONFIG_FILE}"; then - echo "Configuration file need to be present in '${DIR}/config' or path passed as parameter" + echo "Configuration file needs to be present in '${DIR}/config', or path passed as parameter with -c" exit 1 else # shellcheck disable=SC1090 source ${CONFIG_FILE} fi +# Concatenate any command line env to DOCKER_ADDL_ENV +export DOCKER_ADDL_ENV+=("${CMDLINE_ENV[@]}") + +# Convert additional env into docker format +# so we can pass it to docker run +ADDL_ENV="" +echo "Processing additional environment for docker run..." +while IFS='=' read -r name value; do + # Handle double-quoted ("...") values. + if [[ $value =~ ^\"(.*)\"$ ]]; then + # Using `read` without `-r` removes the \ from embedded \ sequences. + IFS= read value <<<"${BASH_REMATCH[1]}" + fi + # Append to a string in docker env format + ADDL_ENV="${ADDL_ENV} -e ${name}=${value}" +done <<<$( + IFS=$'\n' + echo "${DOCKER_ADDL_ENV[*]}" +) + +# Ability to add additional mounts so that custom stages can be mounted into the build process +# without modifying the pi-gen repository +# Note: Mounts cannot be passed in via the command line +ADDL_MOUNTS="" +if [[ ! -z ${DOCKER_ADDL_MOUNTS:=""} ]]; then + echo "Processing additional mounts for docker run..." + while IFS=' ' read -r mount; do + ADDL_MOUNTS="${ADDL_MOUNTS} -v ${mount}" + done <<<$( + IFS=$'\n' + echo "${DOCKER_ADDL_MOUNTS[*]}" + ) +fi + CONTAINER_NAME=${CONTAINER_NAME:-pigen_work} CONTINUE=${CONTINUE:-0} PRESERVE_CONTAINER=${PRESERVE_CONTAINER:-0} @@ -79,7 +121,7 @@ PRESERVE_CONTAINER=${PRESERVE_CONTAINER:-0} if [ -z "${IMG_NAME}" ]; then echo "IMG_NAME not set in 'config'" 1>&2 echo 1>&2 -exit 1 + exit 1 fi # Ensure the Git Hash is recorded before entering the docker container @@ -103,37 +145,15 @@ BUILD_OPTS="$(echo "${BUILD_OPTS:-}" | sed -E 's@\-c\s?([^ ]+)@-c /config@')" # Check the arch of the machine we're running on. If it's 64-bit, use a 32-bit base image instead case "$(uname -m)" in - x86_64|aarch64) - BASE_IMAGE=i386/debian:buster - ;; - *) - BASE_IMAGE=debian:buster - ;; +x86_64 | aarch64) + BASE_IMAGE=i386/debian:buster + ;; +*) + BASE_IMAGE=debian:buster + ;; esac ${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}" -# Ability to add additional mounts so that custom stages can be mounted into the build process -# without modifying the pi-gen repository -DOCKER_ADDL_MOUNTS="" -echo "Processing additional mounts..." -for mount in ${ADDL_MOUNTS:=""} -do - DOCKER_ADDL_MOUNTS="${DOCKER_ADDL_MOUNTS} --volume ${mount}" -done - -# Pass in any additional env that we were given to the docker run commands -DOCKER_ADDL_ENV="" -echo "Processing additional env..." -while IFS='=' read -r name value; do - # Handle double-quoted ("...") values. - if [[ $value =~ ^\"(.*)\"$ ]]; then - # Using `read` without `-r` removes the \ from embedded \ sequences. - IFS= read value <<<"${BASH_REMATCH[1]}" - fi - # Append to a string in docker env format - DOCKER_ADDL_ENV="${DOCKER_ADDL_ENV} -e ${name}=${value}" -done <<< $( IFS=$'\n'; echo "${ADDL_ENV[*]}" ) - if [ "${CONTAINER_EXISTS}" != "" ]; then trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}_cont' SIGINT SIGTERM time ${DOCKER} run --rm --privileged \ @@ -141,8 +161,8 @@ if [ "${CONTAINER_EXISTS}" != "" ]; then -v /dev:/dev \ -v /lib/modules:/lib/modules \ --volume "${CONFIG_FILE}":/config:ro \ - ${DOCKER_ADDL_MOUNTS} \ - ${DOCKER_ADDL_ENV} \ + ${ADDL_MOUNTS} \ + ${ADDL_ENV} \ -e "GIT_HASH=${GIT_HASH}" \ --volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" \ pi-gen \ @@ -157,8 +177,8 @@ else -v /dev:/dev \ -v /lib/modules:/lib/modules \ --volume "${CONFIG_FILE}":/config:ro \ - ${DOCKER_ADDL_MOUNTS} \ - ${DOCKER_ADDL_ENV} \ + ${ADDL_MOUNTS} \ + ${ADDL_ENV} \ -e "GIT_HASH=${GIT_HASH}" \ pi-gen \ bash -e -o pipefail -c "dpkg-reconfigure qemu-user-static &&