Merge pull request #43 from coecms/update_payu_again

Move container build to github actions. Update payu again

.github/workflows/build_and_test.yml

@@ -3,10 +3,24 @@ on: pull_request
 jobs:
   build:
     runs-on: ubuntu-latest
+    container:
+      image: quay.io/singularity/singularity:v3.11.4
+      options: --privileged
     steps:
       - name: Checkout repository
         ### Latest at time of writing
         uses: actions/[email protected]
+      - name: Check if container definition has changed
+        id: changed-container-def
+        uses: tj-actions/changed-files@v37
+        with:
+          files_yaml: |
+            containerdef:
+              - container/container.def
+      - name: Build container if definition has changed
+        if: steps.changed-container-def.outputs.containerdef_any_changed == 'true'
+        run: |
+          sudo -E singularity build container/base.sif container/container.def
       - name: Sync repository to Gadi
         ### Latest at time of writing
         uses: up9cloud/[email protected]

container/container.def

@@ -1,38 +1,45 @@
-%setup
-        mkdir -p ${SINGULARITY_ROOTFS}/usr/bin
-        ln -s usr/bin ${SINGULARITY_ROOTFS}/bin
-        ln -s half-root/boot ${SINGULARITY_ROOTFS}/boot
-        ln -s half-root/etc ${SINGULARITY_ROOTFS}/etc
-        ln -s half-root/images ${SINGULARITY_ROOTFS}/images
-        ln -s half-root/initrd ${SINGULARITY_ROOTFS}/initrd
-        ln -s usr/lib ${SINGULARITY_ROOTFS}/lib
-        ln -s usr/lib64 ${SINGULARITY_ROOTFS}/lib64
-        ln -s usr/sbin ${SINGULARITY_ROOTFS}/sbin
-
-        mkdir -p ${SINGULARITY_ROOTFS}/opt
-        ln -s ../half-root/opt/Modules ${SINGULARITY_ROOTFS}/opt/Modules
-        ln -s ../half-root/opt/beegfs ${SINGULARITY_ROOTFS}/opt/beegfs
-        ln -s ../half-root/opt/bin ${SINGULARITY_ROOTFS}/opt/bin
-        ln -s ../half-root/opt/gdrcopy ${SINGULARITY_ROOTFS}/opt/gdrcopy
-        ln -s ../half-root/opt/intel ${SINGULARITY_ROOTFS}/opt/intel
-        ln -s ../half-root/opt/knem-1.1.4.90mlnx1 ${SINGULARITY_ROOTFS}/opt/knem-1.1.4.90mlnx1
-        ln -s ../half-root/opt/mellanox ${SINGULARITY_ROOTFS}/opt/mellanox
-        ln -s ../half-root/opt/pbs ${SINGULARITY_ROOTFS}/opt/pbs
-        ln -s ../half-root/opt/rash ${SINGULARITY_ROOTFS}/opt/rash
-        ln -s ../half-root/opt/rh ${SINGULARITY_ROOTFS}/opt/rh
-
-        ### Mount points for various bits of the image
-        mkdir -p ${SINGULARITY_ROOTFS}/etc
-        mkdir -p ${SINGULARITY_ROOTFS}/half-root
-        mkdir -p ${SINGULARITY_ROOTFS}/local
-        mkdir -p ${SINGULARITY_ROOTFS}/ram
-        mkdir -p ${SINGULARITY_ROOTFS}/run
-        mkdir -p ${SINGULARITY_ROOTFS}/system
-        mkdir -p ${SINGULARITY_ROOTFS}/usr
-
-        ### Mount points for service sockets
-        mkdir -p ${SINGULARITY_ROOTFS}/var/lib/sss
-        mkdir -p ${SINGULARITY_ROOTFS}/var/run/munge
-
-%runscript
+Bootstrap: scratch
+
+%setup
+        mkdir -p ${SINGULARITY_ROOTFS}/usr/bin
+        ln -s usr/bin ${SINGULARITY_ROOTFS}/bin
+        ln -s half-root/boot ${SINGULARITY_ROOTFS}/boot
+        ln -s half-root/etc ${SINGULARITY_ROOTFS}/etc
+        ln -s half-root/images ${SINGULARITY_ROOTFS}/images
+        ln -s half-root/initrd ${SINGULARITY_ROOTFS}/initrd
+        ln -s usr/lib ${SINGULARITY_ROOTFS}/lib
+        ln -s usr/lib64 ${SINGULARITY_ROOTFS}/lib64
+        ln -s usr/sbin ${SINGULARITY_ROOTFS}/sbin
+
+        mkdir -p ${SINGULARITY_ROOTFS}/opt
+        ln -s ../half-root/opt/Modules ${SINGULARITY_ROOTFS}/opt/Modules
+        ln -s ../half-root/opt/beegfs ${SINGULARITY_ROOTFS}/opt/beegfs
+        ln -s ../half-root/opt/bin ${SINGULARITY_ROOTFS}/opt/bin
+        ln -s ../half-root/opt/gdrcopy ${SINGULARITY_ROOTFS}/opt/gdrcopy
+        ln -s ../half-root/opt/intel ${SINGULARITY_ROOTFS}/opt/intel
+        ln -s ../half-root/opt/knem-1.1.4.90mlnx1 ${SINGULARITY_ROOTFS}/opt/knem-1.1.4.90mlnx1
+        ln -s ../half-root/opt/mellanox ${SINGULARITY_ROOTFS}/opt/mellanox
+        ln -s ../half-root/opt/pbs ${SINGULARITY_ROOTFS}/opt/pbs
+        ln -s ../half-root/opt/rash ${SINGULARITY_ROOTFS}/opt/rash
+        ln -s ../half-root/opt/rh ${SINGULARITY_ROOTFS}/opt/rh
+
+        ### Mount points for various bits of the image
+        mkdir -p ${SINGULARITY_ROOTFS}/etc
+        mkdir -p ${SINGULARITY_ROOTFS}/half-root
+        mkdir -p ${SINGULARITY_ROOTFS}/local
+        mkdir -p ${SINGULARITY_ROOTFS}/ram
+        mkdir -p ${SINGULARITY_ROOTFS}/run
+        mkdir -p ${SINGULARITY_ROOTFS}/system
+        mkdir -p ${SINGULARITY_ROOTFS}/usr
+
+        ### Mount points for service sockets
+        mkdir -p ${SINGULARITY_ROOTFS}/var/lib/sss
+        mkdir -p ${SINGULARITY_ROOTFS}/var/run/munge
+
+        mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-22.07
+        mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-22.10
+        mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-23.01
+        mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-23.04
+
+%runscript
         /usr/bin/bash -l

scripts/build.sh

@@ -154,15 +154,23 @@ fi
 
 ln -sf "${ENV_INSTALLATION_PATH}" "${CONDA_OUTER_BASE}"/"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}"/envs/
 
-"${SINGULARITY_BINARY_PATH}" -s exec --bind /etc,/half-root,/local,/ram,/run,/system,/usr,/var/lib/sss,/var/run/munge,/var/lib/rpm,"${OVERLAY_BASE}":/g "${CONTAINER_PATH}" $( realpath $0 ) --inner "${DO_UPDATE}"
+if [[ -e "${CONTAINER_PATH}" ]]; then
+    ### New container, use that
+    my_container="${CONTAINER_PATH}"
+else
+    my_container="${CONDA_OUTER_BASE}"/"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}"/etc/"${CONTAINER_PATH##*/}"
+fi
+
+"${SINGULARITY_BINARY_PATH}" -s exec --bind /etc,/half-root,/local,/ram,/run,/system,/usr,/var/lib/sss,/var/run/munge,/var/lib/rpm,"${OVERLAY_BASE}":/g "${my_container}" $( realpath $0 ) --inner "${DO_UPDATE}"
 if [[ $? -ne 0 ]]; then
     exit 1
 fi
 
 ### See if the container has been updated
-read newhash fn < <( md5sum "${CONTAINER_PATH}" )
-read oldhash fn < <( md5sum "${CONDA_OUTER_BASE}"/"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}"/etc/"${CONTAINER_PATH##*/}" )
-if [[ "${oldhash}" != "${newhash}" ]]; then
+### The container will only exist on ${CONTAINER_PATH} if it was built by the github action
+#read newhash fn < <( md5sum "${CONTAINER_PATH}" )
+#read oldhash fn < <( md5sum "${CONDA_OUTER_BASE}"/"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}"/etc/"${CONTAINER_PATH##*/}" )
+if [[ -e "${CONTAINER_PATH}" ]]; then
     echo "Container update detected. Copying in new container"
     cp "${CONTAINER_PATH}" "${CONDA_OUTER_BASE}"/"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}"/etc/"${CONTAINER_PATH##*/}"
 fi

scripts/environment.yml

@@ -121,7 +121,7 @@ dependencies:
 - coecms-util
 - yamanifest
 - cmocean
-- accessnri::payu==1.0.27
+- accessnri::payu
 - splitvar
 - marineHeatWaves
 - holoviews

scripts/install_config.sh

@@ -30,7 +30,7 @@ export FULLENV="${ENVIRONMENT}-${VERSION_TO_MODIFY}"
 ### Other settings
 export TEST_OUT_FILE=test_results.xml
 export PYTHONNOUSERSITE=true
-export CONTAINER_PATH=$( realpath "${SCRIPT_DIR}"/../container/base.sif )
+export CONTAINER_PATH="${SCRIPT_DIR}"/../container/base.sif
 export SINGULARITY_BINARY_PATH="/opt/singularity/bin/singularity"
 
 declare -a rpms_to_remove=( "openssh-clients" "openssh-server" "openssh" )

scripts/test.sh

@@ -58,7 +58,14 @@ for f in "${outside_files_to_copy[@]}"; do
     cp "${f}" "${OVERLAY_BASE}"/"${f#/g/}"
 done
 
-"${SINGULARITY_BINARY_PATH}" -s exec --bind /etc,/half-root,/local,/ram,/run,/system,/usr,/var/lib/sss,/var/run/munge,/var/lib/rpm,"${OVERLAY_BASE}":/g --overlay="${BUILD_STAGE_DIR}"/"${FULLENV}".sqsh.tmp "${CONTAINER_PATH}" $( realpath $0 ) --inner
+if [[ -e "${CONTAINER_PATH}" ]]; then
+    ### New container, use that
+    my_container="${CONTAINER_PATH}"
+else
+    my_container="${CONDA_OUTER_BASE}"/"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}"/etc/"${CONTAINER_PATH##*/}"
+fi
+
+"${SINGULARITY_BINARY_PATH}" -s exec --bind /etc,/half-root,/local,/ram,/run,/system,/usr,/var/lib/sss,/var/run/munge,/var/lib/rpm,"${OVERLAY_BASE}":/g --overlay="${BUILD_STAGE_DIR}"/"${FULLENV}".sqsh.tmp "${my_container}" $( realpath $0 ) --inner
 
 if [[ ! -e "${TEST_OUT_FILE}" ]]; then
     echo "TESTS FILE MISSING - assuming tests have failed"