Further Optimizations (#2092)

* optimize check for installed packages (apt, pip)

* fix jukebox_dir

* remove package optimization from dockerfile

add comments in package step

* add package installation from file

test reads the same file. only one place to maintain

* install internal packages in Dockerfile

dont install resolvconf as it will fail in containers

* added textfile for arm sepcific packages

* removed as already part of install script

* add package textfile for autohotspot setup

* correct variable usage.

* check homedir as prerequisite

* some fixes

* read modules to test from file

* renamed package txt for raspberry pi specifics

* update sed call

* renamed local variable

* save images as artifact

* clarify function name. add function comment

* fixed mpd service enablement

* fix unnecessary rebuild of unrelated package files

.github/workflows/test_docker_debian_codename_sub.yml

@@ -43,7 +43,7 @@ jobs:
 
     outputs:
         cache_key: ${{ steps.vars.outputs.cache_key }}
-        image_file_path: ${{ steps.vars.outputs.image_file_path }}
+        image_file_name: ${{ steps.vars.outputs.image_file_name }}
         image_tag_name: ${{ steps.vars.outputs.image_tag_name }}
 
     # create local docker registry to use locally build images
@@ -73,18 +73,18 @@ jobs:
         CACHE_SCOPE: ${{ inputs.cache_scope }}
       run: |
         echo "image_tag_name=${{ env.DOCKER_IMAGE_NAME }}:${{ env.DEBIAN_CODENAME }}-test" >> $GITHUB_OUTPUT
+        echo "image_file_name=${{ env.DOCKER_IMAGE_NAME }}-${{ env.DEBIAN_CODENAME }}.tar" >> $GITHUB_OUTPUT
         echo "cache_scope=${{ env.CACHE_SCOPE }}-${{ env.DEBIAN_CODENAME }}" >> $GITHUB_OUTPUT
 
     - name: Set Output vars
       id: vars
       env:
-        DEBIAN_CODENAME: ${{ inputs.debian_codename }}
-        DOCKER_IMAGE_NAME: ${{ inputs.docker_image_name }}
         LOCAL_REGISTRY_PORT: ${{ inputs.local_registry_port }}
       run: |
         echo "image_tag_name=${{ steps.pre-vars.outputs.image_tag_name }}" >> $GITHUB_OUTPUT
         echo "image_tag_name_local_base=localhost:${{ env.LOCAL_REGISTRY_PORT }}/${{ steps.pre-vars.outputs.image_tag_name }}-base" >> $GITHUB_OUTPUT
-        echo "image_file_path=./${{ env.DOCKER_IMAGE_NAME }}-${{ env.DEBIAN_CODENAME }}.tar" >> $GITHUB_OUTPUT
+        echo "image_file_name=${{ steps.pre-vars.outputs.image_file_name }}" >> $GITHUB_OUTPUT
+        echo "image_file_path=./${{ steps.pre-vars.outputs.image_file_name }}" >> $GITHUB_OUTPUT
         echo "cache_scope=${{ steps.pre-vars.outputs.cache_scope }}" >> $GITHUB_OUTPUT
         echo "cache_key=${{ steps.pre-vars.outputs.cache_scope }}-${{ github.sha }}#${{ github.run_attempt }}" >> $GITHUB_OUTPUT
 
@@ -123,12 +123,12 @@ jobs:
         build-args: |
           BASE_TEST_IMAGE=${{ steps.vars.outputs.image_tag_name_local_base }}
 
-    # Cache image file for next jobs
-    - name: Cache Save Docker Image
-      uses: actions/cache/save@v3
+    - name: Artifact Upload Docker Image
+      uses: actions/upload-artifact@v3
       with:
-        key: ${{ steps.vars.outputs.cache_key }}
+        name: ${{ steps.vars.outputs.image_file_name }}
         path: ${{ steps.vars.outputs.image_file_path }}
+        retention-days: 2
 
 
   # Run tests with build image
@@ -149,17 +149,14 @@ jobs:
     - name: Set up Docker Buildx
       uses: docker/[email protected]
 
-    # Load cached image file
-    - name: Cache Restore Docker Image
-      uses: actions/cache/restore@v3
+    - name: Artifact Download Docker Image
+      uses: actions/download-artifact@v3
       with:
-        key: ${{  needs.build.outputs.cache_key }}
-        path: ${{ needs.build.outputs.image_file_path }}
-        fail-on-cache-miss: true
+        name: ${{ needs.build.outputs.image_file_name }}
 
     - name: Load Docker Image
       run: |
-        docker load --input ${{ needs.build.outputs.image_file_path }}
+        docker load --input ${{ needs.build.outputs.image_file_name }}
 
     # Run test
     - name: Run Test ${{ inputs.debian_codename }}-${{ matrix.username }}-${{ matrix.test_script }}
@@ -170,3 +167,16 @@ jobs:
         name: ${{ matrix.test_script }}
         args: |
           ./${{ matrix.test_script }}
+
+  # cleanup after test execution
+  cleanup:
+    # run only if tests didn't fail: keep the artifact to make job reruns possible
+    if: ${{ !failure() }}
+    needs: [build, test]
+    runs-on: ${{ inputs.runs_on }}
+
+    steps:
+        - name: Artifact Delete Docker Image
+          uses: geekyeggo/delete-artifact@v2
+          with:
+            name: ${{ needs.build.outputs.image_file_name }}

README.md

@@ -39,7 +39,7 @@ Another bunch of wonderful designs! 2023 is the fifth Phoniebox calendar. If you
 
 If you want to be featured on next years calendar, please make sure to add your Phoniebox pics to the [design thread here on github](https://github.com/MiczFlor/RPi-Jukebox-RFID/issues/639).
 
-## <a name="install">Install Phoniebox
+## Install Phoniebox
 
 If you are looking for the latest **stable version 2.x**, use the [install script for Raspberry Pi](https://github.com/MiczFlor/RPi-Jukebox-RFID/wiki/INSTALL).
 

ci/Dockerfile.debian

@@ -8,30 +8,29 @@ ENV DOCKER_RUNNING=true
 RUN touch /boot/cmdlinetxt
 
 RUN export DEBIAN_FRONTEND=noninteractive \
+  && echo "--- install packages (1) ---" \
   && apt-get update \
   && apt-get -y install \
     apt-utils \
     curl \
     gnupg \
+  && echo "--- add sources ---" \
   && curl -fsSL http://raspbian.raspberrypi.org/raspbian.public.key | gpg --dearmor > /usr/share/keyrings/raspberrypi-raspbian-keyring.gpg \
   && curl -fsSL http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | gpg --dearmor > /usr/share/keyrings/raspberrypi-archive-debian-keyring.gpg \
   && echo "deb [signed-by=/usr/share/keyrings/raspberrypi-raspbian-keyring.gpg] http://raspbian.raspberrypi.org/raspbian/ ${DEBIAN_CODENAME} main contrib non-free rpi" > /etc/apt/sources.list.d/raspi.list \
   && echo "deb [signed-by=/usr/share/keyrings/raspberrypi-archive-debian-keyring.gpg] http://archive.raspberrypi.org/debian/ ${DEBIAN_CODENAME} main" >> /etc/apt/sources.list.d/raspi.list \
+  && echo "--- install packages (2) ---" \
   && apt-get update \
   && apt-get -y upgrade \
   && apt-get -y install \
     alsa-utils \
     build-essential \
-    git \
     locales \
     sudo \
     systemd \
     wireless-tools \
     wget \
     wpasupplicant \
-    # install internally used packages here to speed up GitHub Action
-    raspberrypi-kernel \
-    raspberrypi-kernel-headers \
   && rm -rf /var/lib/apt/lists/*
 # ------
 
@@ -87,9 +86,20 @@ ARG GIT_URL
 
 ENV GIT_BRANCH=$GIT_BRANCH GIT_URL=$GIT_URL
 
-COPY --chown=root:$TEST_USER_GROUP --chmod=770 scripts/installscripts/buster-install-default.sh .
+COPY --chown=root:$TEST_USER_GROUP --chmod=770 packages.txt packages-raspberrypi.txt ./
+
+RUN export DEBIAN_FRONTEND=noninteractive \
+  && echo "--- install internal packages ---" \
+  && apt-get update \
+  # remove resolvconf as installation will fail in container
+  && sed 's/#.*//g' packages.txt | sed 's/resolvconf//' | xargs apt-get -y install \
+  && sed 's/#.*//g' packages-raspberrypi.txt | xargs apt-get -y install \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY --chown=root:$TEST_USER_GROUP --chmod=770 scripts/installscripts/buster-install-default.sh ./
 WORKDIR /tests
-COPY --chown=root:$TEST_USER_GROUP --chmod=770 scripts/installscripts/tests/*.sh .
+COPY --chown=root:$TEST_USER_GROUP --chmod=770 scripts/installscripts/tests/*.sh ./
+
 # ------
 
 

packages-autohotspot.txt

@@ -0,0 +1,6 @@
+# Define packages for apt-get.  These can be installed with
+# 'sed 's/#.*//g' packages.txt | xargs sudo apt-get install'
+
+dnsmasq
+hostapd
+iw

packages-raspberrypi.txt

@@ -0,0 +1,4 @@
+# Define packages for apt-get.  These can be installed with
+# 'sed 's/#.*//g' packages.txt | xargs sudo apt-get install'
+
+raspberrypi-kernel-headers

packages-spotify.txt

@@ -0,0 +1,13 @@
+# Define packages for apt-get.  These can be installed with
+# 'sed 's/#.*//g' packages.txt | xargs sudo apt-get install'
+
+libspotify-dev
+libspotify12
+python3-cffi
+python3-ply
+python3-pycparser
+python3-spotify
+mopidy
+mopidy-mpd
+mopidy-local
+mopidy-spotify

packages.txt

@@ -0,0 +1,30 @@
+# Define packages for apt-get.  These can be installed with
+# 'sed 's/#.*//g' packages.txt | xargs sudo apt-get install'
+
+samba
+samba-common-bin
+gcc
+lighttpd
+php-common
+php-cgi
+php
+at
+mpd
+mpc
+mpg123
+git
+ffmpeg
+resolvconf
+spi-tools
+netcat-traditional
+alsa-utils
+lsof
+procps
+python3
+python3-dev
+python3-pip
+python3-setuptools
+python3-wheel
+python3-mutagen
+python3-gpiozero
+python3-spidev

requirements.txt

@@ -1,3 +1,4 @@
+# Test trigger
 # Library dependencies for the python code.  You need to install these with
 # `sudo python3 -m pip install --upgrade --force-reinstall -r requirements.txt` before you can run this.
 
@@ -6,7 +7,7 @@
 # related libraries.
 evdev
 git+https://github.com/lthiery/SPI-Py.git#egg=spi-py
-youtube_dl
+youtube-dl
 pyserial
 RPi.GPIO
 

scripts/helperscripts/setup_autohotspot.sh

@@ -14,6 +14,17 @@ if [[ "$#" -lt 2 || ( "${AUTOHOTSPOTconfig}" != "NO" && "${AUTOHOTSPOTconfig}" !
     exit 1
 fi
 
+# Reads a textfile and pipes all lines as args to the given command.
+# Does filter out comments.
+# Arguments:
+#   1    : textfile to read
+#   2... : command to receive args (e.g. 'echo', 'apt-get -y install', ...)
+call_with_args_from_file () {
+    local package_file="$1"
+    shift
+
+    sed 's/#.*//g' ${package_file} | xargs "$@"
+}
 
 apt_get="sudo apt-get -qq --yes"
 
@@ -37,7 +48,7 @@ if [ "${AUTOHOTSPOTconfig}" == "YES" ]; then
     sudo iwconfig wlan0 power off
 
     # required packages
-    ${apt_get} install dnsmasq hostapd iw
+    call_with_args_from_file "${JUKEBOX_HOME_DIR}"/packages-autohotspot.txt ${apt_get} install
     sudo systemctl unmask hostapd
     sudo systemctl disable hostapd
     sudo systemctl stop hostapd