Merge pull request #26 from ACCESS-NRI/22-automatic-dev-build

Add payu dev environment deployment workflow - this creates an environment in prerelease that has the latest changes from payu
ACCESS-NRI · Jun 17, 2024 · bb1d164 · bb1d164
2 parents c26c5bb + 43247a7
commit bb1d164
Show file tree

Hide file tree

Showing 6 changed files with 239 additions and 4 deletions.
diff --git a/.github/workflows/check-valid-env-dev.yml b/.github/workflows/check-valid-env-dev.yml
@@ -0,0 +1,19 @@
+name: env-dev.yml
+on:
+  pull_request:
+    paths:
+      - 'env-dev.yml'
+jobs:
+  check:
+    name: Check Env Dev
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Env-dev Valid
+        uses: mamba-org/setup-micromamba@422500192359a097648154e8db4e39bdb6c6eed7 #v1.8.1
+        with:
+          micromamba-version: '1.5.3-0'
+          environment-file: env-dev.yml
+          environment-name: test-payu-environment-dev
+          generate-run-shell: false
diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml
@@ -0,0 +1,178 @@
+name: Deploy Dev Enviroment
+on:
+  schedule:
+   - cron: '0 21 * * *' # Runs every morning at 7AM AEST
+  push:
+    branches:
+      - main
+    paths:
+      - 'env-dev.yml'
+  workflow_dispatch: # Allows manual triggering
+jobs:
+  check-for-payu-updates:
+    runs-on: ubuntu-latest
+    outputs:
+      commits-count: ${{ steps.check-payu-commits.outputs.commits-count }}
+      last-commit-hash: ${{ steps.check-payu-commits.outputs.latest-commit-hash }}
+    steps:
+      - name: Checkout current repository
+        uses: actions/checkout@v4
+
+      - name: Get last workflow run time
+        id: last-run-time
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN }}
+        run: |
+          # Fetch the last successful workflow run time
+          last_run_time=$(gh run list --status success --workflow deploy-dev.yml --json updatedAt --jq .[0].updatedAt)
+
+          echo "Last successful workflow run time: $last_run_time"
+          echo "last-run-time=$last_run_time" >> $GITHUB_OUTPUT
+
+      - name: Checkout payu repository
+        uses: actions/checkout@v4
+        with:
+          repository: payu-org/payu
+          path: payu
+          ref: master
+
+      - name: Check commits in payu repository
+        id: check-payu-commits
+        run: |
+          # Check for any commits since last successful runtime
+          last_run_time="${{ steps.last-run-time.outputs.last-run-time }}"
+          commits_count=$(git -C ./payu rev-list --count --since="$last_run_time" master)
+
+          # Get latest commit hash
+          latest_commit_hash=$(git -C ./payu rev-parse --short HEAD)
+
+          echo "Number of new commits since last run: $commits_count, latest commit hash: $latest_commit_hash"
+
+          echo "commits-count=$commits_count" >> $GITHUB_OUTPUT
+          echo "latest-commit-hash=$latest_commit_hash" >> $GITHUB_OUTPUT
+
+  pack:
+    name: Pack Payu
+    runs-on: ubuntu-latest
+    needs:
+      - check-for-payu-updates
+    # Deploy payu if manually triggered, env-dev.yml has been updated, or if there's new commits to payu repository
+    if: >
+      needs.check-for-payu-updates.outputs.commits-count > 0 ||
+      github.event_name == 'push' ||
+      github.event_name == 'workflow_dispatch'
+    outputs:
+      name: ${{ steps.payu.outputs.name }}
+      version: ${{ steps.payu.outputs.version }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get Payu Version
+        id: payu
+        run: |
+          # Set version to datetime and last short commit hash of payu
+          NOW=$(date -u +"%Y%m%dT%H%M%SZ")
+          COMMIT_HASH=${{ needs.check-for-payu-updates.outputs.last-commit-hash }}
+          VERSION="dev-$NOW-$COMMIT_HASH"
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo "name=payu-$VERSION" >> $GITHUB_OUTPUT
+
+      - name: Setup Micromamba
+        uses: mamba-org/setup-micromamba@422500192359a097648154e8db4e39bdb6c6eed7 #v1.8.1
+        with:
+          micromamba-version: '1.5.3-0'
+          environment-file: env-dev.yml
+          environment-name: ${{ steps.payu.outputs.name }}
+          generate-run-shell: true
+
+      - name: Create Pack
+        shell: micromamba-shell {0}
+        run: conda pack
+
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: payu-dev
+          if-no-files-found: error
+          path: |
+            ${{ steps.payu.outputs.name }}.tar.gz
+
+  deploy:
+    runs-on: ubuntu-latest
+    needs:
+      - pack
+    environment: Gadi
+    env:
+      NAME: ${{ needs.pack.outputs.name }}
+      VERSION: ${{ needs.pack.outputs.version }}
+    steps:
+      - uses: actions/[email protected]
+        with:
+          name: payu-dev
+
+      - uses: access-nri/actions/.github/actions/setup-ssh@main
+        id: ssh
+        with:
+          hosts: |
+            ${{ secrets.HOST_DATA }}
+            ${{ secrets.HOST }}
+          private-key: ${{ secrets.SSH_KEY }}
+
+      - name: Copy to Gadi 
+        run: |
+          rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \
+            ${{ env.NAME }}.tar.gz \
+            ${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ vars.PRERELEASE_PACK_LOCATION }}
+
+      - name: Deploy to Gadi
+        env:
+          ENVIRONMENT_LOCATION: ${{ vars.PRERELEASE_DEPLOYMENT_LOCATION }}/${{ env.VERSION }}
+          ENVIRONMENT_SYMLINK: ${{ vars.PRERELEASE_DEPLOYMENT_LOCATION }}/dev
+          MODULE_SYMLINK: ${{ vars.PRERELEASE_MODULE_LOCATION }}/dev
+          PACKED_ENVIRONMENT: ${{ vars.PRERELEASE_PACK_LOCATION }}/${{ env.NAME }}.tar.gz
+        run: |
+          ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT'
+
+          # Create list of previous payu/dev environments to remove later
+          old_versions=$(ls "${{ vars.PRERELEASE_DEPLOYMENT_LOCATION }}" | grep -E '^dev-[0-9]{8}T[0-9]{6}Z-.*')
+        
+          # Unpack conda enviroment
+          mkdir ${{ env.ENVIRONMENT_LOCATION }} || exit $?
+          tar -xzf ${{ env.PACKED_ENVIRONMENT }} -C ${{ env.ENVIRONMENT_LOCATION }}
+          source ${{ env.ENVIRONMENT_LOCATION }}/bin/activate
+          conda-unpack
+          payu --version
+          source ${{ env.ENVIRONMENT_LOCATION }}/bin/deactivate
+          echo "::notice::New payu/dev environment unpacked to ${{ env.ENVIRONMENT_LOCATION }}"
+
+          # Check if payu/dev symlink already exists
+          if [ -L "${{ env.ENVIRONMENT_SYMLINK}}" ]; then
+            # Get previous version  of payu/dev
+            previous_env_path=$(readlink -f "${{ env.ENVIRONMENT_SYMLINK}}")
+            previous_ver=$(basename "$previous_env_path")
+            echo "Previous payu/dev version $previous_ver"
+
+            # Remove previous version from old versions - to ensure active
+            # environments at time of deployment are not deleted
+            old_versions=$(echo "$old_versions" | grep -v "$previous_ver")
+
+            # Unlink the symlink
+            unlink ${{ env.ENVIRONMENT_SYMLINK}}
+          fi
+
+          # Create the payu/dev symlink in pre-release apps
+          ln -s ${{ env.ENVIRONMENT_LOCATION }} ${{ env.ENVIRONMENT_SYMLINK }}
+
+          # Setup modulefile symlink
+          ln -sf ${{ vars.PRERELEASE_MODULE_LOCATION }}/.common ${{ env.MODULE_SYMLINK }}
+
+          # Remove old versions of environments
+          for version in $old_versions; do
+              echo "Removing old payu/dev environment version $version"
+              rm -rf "${{ vars.PRERELEASE_DEPLOYMENT_LOCATION }}/$version"
+          done
+
+          # Remove packed environment file
+          echo "Removing conda-packed environment file ${{ env.PACKED_ENVIRONMENT }}"
+          rm -rf ${{ env.PACKED_ENVIRONMENT }}
+          EOT
diff --git a/.github/workflows/update-module.yml b/.github/workflows/update-module.yml
@@ -20,8 +20,28 @@ jobs:
             ${{ secrets.HOST_DATA }}
           private-key: ${{ secrets.SSH_KEY }}
 
+      - name: Generate release modulesfiles
+        run: |
+          mkdir -p release/modules
+          sed 's|{{MODULE_LOCATION}}|'"${{ vars.MODULE_LOCATION }}"'|g' modules/.common > release/modules/.common
+          sed 's|{{APPS_LOCATION}}|'"${{ vars.APPS_LOCATION }}"'|g' modules/env.sh > release/modules/env.sh
+          chmod +x release/modules/env.sh
+
+      - name: Generate pre-release modulesfiles
+        run: |
+          mkdir -p prerelease/modules
+          sed 's|{{MODULE_LOCATION}}|'"${{ vars.PRERELEASE_MODULE_LOCATION }}"'|g' modules/.common > prerelease/modules/.common
+          sed 's|{{APPS_LOCATION}}|'"${{ vars.PRERELEASE_APPS_LOCATION }}"'|g' modules/env.sh > prerelease/modules/env.sh
+          chmod +x prerelease/modules/env.sh
+
       - name: Copy modulefiles to Gadi
         run: |
+          # Rsync release modulefiles
           rsync -rvp -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \
-            modules/ \
+            release/modules/ \
             ${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ vars.MODULE_LOCATION }}
+
+          # Rsync development modulefiles
+          rsync -rvp -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \
+            prerelease/modules/ \
+            ${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ vars.PRERELEASE_MODULE_LOCATION }}
diff --git a/env-dev.yml b/env-dev.yml
@@ -0,0 +1,17 @@
+name: payu-dev
+channels:
+  - accessnri
+  - coecms
+  - conda-forge
+dependencies:
+  # Use latest changes from payu's master branch
+  - pip:
+    - git+https://github.com/payu-org/payu.git@master
+  - coecms::f90nml
+  - conda-lock
+  - conda-pack
+  - gh
+  - git
+  - nco
+  - pytest
+  - openssh>=8.3
diff --git a/modules/.common b/modules/.common
@@ -10,7 +10,7 @@ conflict payu
 set payuname [module-info name]
 
 # Get the environment variables from activate script
-set payuenv [exec /bin/env -i /g/data/vk83/modules/payu/env.sh $payuname]
+set payuenv [exec /bin/env -i {{MODULE_LOCATION}}/env.sh $payuname]
 
 # Convert the environment into module commands
 set lines [split $payuenv '\n']

diff --git a/modules/env.sh b/modules/env.sh
@@ -1,5 +1,6 @@
 #!/bin/bash
-# Prints the environment variables when activating un-packed payu environment
+# Prints the environment variables when activating un-packed environment
 export PATH=/usr/bin:/bin
-source /g/data/vk83/apps/$1/bin/activate
+
+source {{APPS_LOCATION}}/$1/bin/activate
 /bin/env