From 014c283670b52d691aedba1161ad6907096f7d83 Mon Sep 17 00:00:00 2001 From: Jo Basevi Date: Wed, 10 Apr 2024 13:44:10 +1000 Subject: [PATCH 01/10] Update modules: check for pre-release modulefiles location --- modules/.common | 10 +++++++++- modules/env.sh | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/modules/.common b/modules/.common index b9bc100..9e2cb78 100644 --- a/modules/.common +++ b/modules/.common @@ -9,8 +9,16 @@ conflict payu # Name of this module's environment, e.g. payu/VERSION set payuname [module-info name] +# Check if modulefile is in prerelease folder +if {[string match "/g/data/vk83/prerelease/*" "$ModulesCurrentModulefile"]} { + set is_prerelease true +} else { + set is_prerelease false +} + # Get the environment variables from activate script -set payuenv [exec /bin/env -i /g/data/vk83/modules/payu/env.sh $payuname] +set payuenv [exec /bin/env -i \ + /g/data/vk83/modules/payu/env.sh $payuname $is_prerelease] # Convert the environment into module commands set lines [split $payuenv '\n'] diff --git a/modules/env.sh b/modules/env.sh index da0cf22..8705e73 100755 --- a/modules/env.sh +++ b/modules/env.sh @@ -1,5 +1,15 @@ #!/bin/bash -# Prints the environment variables when activating un-packed payu environment +# Prints the environment variables when activating un-packed environment export PATH=/usr/bin:/bin -source /g/data/vk83/apps/$1/bin/activate + +name=$1 +is_prerelease=$2 + +if [ "$is_prerelease" = true ]; then + # Use prerelease location for apps + source /g/data/vk83/prerelease/apps/$name/bin/activate +else + # Use release location for apps + source /g/data/vk83/apps/$name/bin/activate +fi /bin/env \ No newline at end of file From f2afeb4e08d26adb5ccecbdfc8c73491bd36bed9 Mon Sep 17 00:00:00 2001 From: Jo Basevi Date: Wed, 10 Apr 2024 13:44:54 +1000 Subject: [PATCH 02/10] Add env-dev.yml for dev payu environments --- env-dev.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 env-dev.yml diff --git a/env-dev.yml b/env-dev.yml new file mode 100644 index 0000000..434ef7b --- /dev/null +++ b/env-dev.yml @@ -0,0 +1,17 @@ +name: payu +channels: + - accessnri + - coecms + - conda-forge +dependencies: + # Use latest changes from payu's master branch + - pip + - git+git://github.com/payu-org/payu@master + - coecms::f90nml + - conda-lock + - conda-pack + - gh + - git + - nco + - pytest + - openssh>=8.3 \ No newline at end of file From 0b6bce94a5185ce7daa59d17b0505ec10690f915 Mon Sep 17 00:00:00 2001 From: Jo Basevi Date: Wed, 10 Apr 2024 13:46:14 +1000 Subject: [PATCH 03/10] Add workflow to deploy dev environment to prerelease --- .github/workflows/check-valid-env-dev.yml | 19 +++++ .github/workflows/deploy-dev.yml | 90 +++++++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 .github/workflows/check-valid-env-dev.yml create mode 100644 .github/workflows/deploy-dev.yml diff --git a/.github/workflows/check-valid-env-dev.yml b/.github/workflows/check-valid-env-dev.yml new file mode 100644 index 0000000..aa39812 --- /dev/null +++ b/.github/workflows/check-valid-env-dev.yml @@ -0,0 +1,19 @@ +name: env-dev.yml +on: + pull_request: + paths: + - 'env-dev.yml' +jobs: + check: + name: Check Env Dev + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Env-dev Valid + uses: mamba-org/setup-micromamba@v1.7.0 + with: + micromamba-version: '1.5.3-0' + environment-file: env.yml + environment-name: test-payu-environment-dev + generate-run-shell: false diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml new file mode 100644 index 0000000..3884f5c --- /dev/null +++ b/.github/workflows/deploy-dev.yml @@ -0,0 +1,90 @@ +name: Deploy Dev Enviroment +on: + push: + branches: + - main + paths: + - 'env-dev.yml' + # TODO: Figure out an automatic trigger. Using manual trigger for now + workflow_dispatch: +env: + NAME: payu-dev + VERSION: dev +jobs: + pack: + name: Pack Payu + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Micromamba + uses: mamba-org/setup-micromamba@v1.7.0 + with: + micromamba-version: '1.5.3-0' + environment-file: env.yml + environment-name: ${{ env.NAME }} + generate-run-shell: true + + - name: Create Pack and Lockfile + shell: micromamba-shell {0} + run: | + conda pack + conda-lock lock --file env.yml --platform linux-64 --micromamba --lockfile ${{ env.NAME }}.conda-lock.yml + + - name: Upload Artifact + uses: actions/upload-artifact@v3 + with: + name: ${{ env.NAME }} + if-no-files-found: error + path: | + ${{ env.NAME }}.tar.gz + ${{ env.NAME }}.conda-lock.yml + + deploy: + runs-on: ubuntu-latest + needs: + - pack + environment: Gadi + permissions: + contents: write + steps: + - uses: actions/download-artifact@v3.0.2 + with: + name: ${{ env.NAME }} + + - uses: access-nri/actions/.github/actions/setup-ssh@main + id: ssh + with: + hosts: | + ${{ secrets.HOST_DATA }} + ${{ secrets.HOST }} + private-key: ${{ secrets.SSH_KEY }} + + - name: Copy to Gadi + run: | + rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \ + ${{ env.NAME }}.tar.gz \ + ${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ vars.PACK_LOCATION }} + + - name: Deploy to Gadi + env: + PAYU_ENVIRONMENT_LOCATION: ${{ vars.PRERELEASE_DEPLOYMENT_LOCATION }}/${{ env.VERSION }} + run: | + ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' + + # Remove previous enviroment if it exists + if [ -d ${{ env.PAYU_ENVIRONMENT_LOCATION }} ]; then + rm -rf ${{ env.PAYU_ENVIRONMENT_LOCATION }} + fi + + # Unpack conda enviroment + mkdir ${{ env.PAYU_ENVIRONMENT_LOCATION }} + tar -xzf ${{ vars.PACK_LOCATION }}/${{ env.NAME }}.tar.gz -C ${{ env.PAYU_ENVIRONMENT_LOCATION }} + source ${{ env.PAYU_ENVIRONMENT_LOCATION }}/bin/activate + conda-unpack + payu --version + source ${{ env.PAYU_ENVIRONMENT_LOCATION }}/bin/deactivate + + # Setup modulefile symlink + ln -s ${{ vars.MODULE_LOCATION }}/.common ${{ vars.PRERELEASE_MODULE_LOCATION }}/${{ env.VERSION }} + EOT From 10f7438d87a2751d9b4c376080852908612a8d4a Mon Sep 17 00:00:00 2001 From: Jo Basevi Date: Mon, 22 Apr 2024 10:01:23 +1000 Subject: [PATCH 04/10] Add commit for third-party actions and update env-dev.yml --- .github/workflows/check-valid-env-dev.yml | 4 ++-- .github/workflows/deploy-dev.yml | 2 +- env-dev.yml | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/check-valid-env-dev.yml b/.github/workflows/check-valid-env-dev.yml index aa39812..878e2d6 100644 --- a/.github/workflows/check-valid-env-dev.yml +++ b/.github/workflows/check-valid-env-dev.yml @@ -11,9 +11,9 @@ jobs: - uses: actions/checkout@v4 - name: Env-dev Valid - uses: mamba-org/setup-micromamba@v1.7.0 + uses: mamba-org/setup-micromamba@422500192359a097648154e8db4e39bdb6c6eed7 #v1.8.1 with: micromamba-version: '1.5.3-0' - environment-file: env.yml + environment-file: env-dev.yml environment-name: test-payu-environment-dev generate-run-shell: false diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index 3884f5c..bbff354 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v4 - name: Setup Micromamba - uses: mamba-org/setup-micromamba@v1.7.0 + uses: mamba-org/setup-micromamba@422500192359a097648154e8db4e39bdb6c6eed7 #v1.8.1 with: micromamba-version: '1.5.3-0' environment-file: env.yml diff --git a/env-dev.yml b/env-dev.yml index 434ef7b..6067869 100644 --- a/env-dev.yml +++ b/env-dev.yml @@ -5,11 +5,12 @@ channels: - conda-forge dependencies: # Use latest changes from payu's master branch - - pip - - git+git://github.com/payu-org/payu@master + - pip: + - git+https://github.com/payu-org/payu.git - coecms::f90nml - conda-lock - conda-pack + - squashfs-tools - gh - git - nco From 235b8d34ef0be79fd3f1e3201d137af4af003b59 Mon Sep 17 00:00:00 2001 From: Jo Basevi Date: Thu, 13 Jun 2024 15:11:37 +1000 Subject: [PATCH 05/10] Update deploy-dev.yml: Add automatic trigger for deployment workflow Add checks if there has been any commits to payu's main branch since last successful workflow run - it was done this way rather than a CD workflow on payu-org/payu to prevent having to store repository secrets of payu-condaenv on payu-org/payu --- .github/workflows/deploy-dev.yml | 100 +++++++++++++++++++++++++------ env-dev.yml | 5 +- 2 files changed, 83 insertions(+), 22 deletions(-) diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index bbff354..bc5ec25 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -1,56 +1,118 @@ name: Deploy Dev Enviroment on: +# schedule: +# - cron: '0 * * * *' # Runs every hour push: branches: - main paths: - 'env-dev.yml' - # TODO: Figure out an automatic trigger. Using manual trigger for now - workflow_dispatch: -env: - NAME: payu-dev - VERSION: dev + workflow_dispatch: # Allows manual triggering jobs: + check-for-payu-updates: + runs-on: ubuntu-latest + outputs: + commits-count: ${{ steps.check-payu-commits.outputs.commits-count }} + last-commit-hash: ${{ steps.check-payu-commits.outputs.latest-commit-hash }} + steps: + - name: Checkout current repository + uses: actions/checkout@v4 + + - name: Get last workflow run time + id: last-run-time + env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} + run: | + # Fetch the last successful workflow run time + response=$(gh api \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/access-nri/payu-condaenv/actions/workflows/deploy-dev.yml/runs?status=success&per_page=1) + last_run_time=$(echo $response | jq -r '.workflow_runs[0].updated_at') + + echo "Last successful workflow run time: $last_run_time" + echo "last-run-time=$last_run_time" >> $GITHUB_OUTPUT + + - name: Checkout payu repository + uses: actions/checkout@v4 + with: + repository: payu-org/payu + path: payu + ref: master + + - name: Check commits in payu repository + id: check-payu-commits + run: | + # Check for any commits since last successful runtime + last_run_time="${{ steps.last-run-time.outputs.last-run-time }}" + commits_count=$(git -C ./payu rev-list --count --since="$last_run_time" master) + + # Get latest commit hash + latest_commit_hash=$(git -C ./payu rev-parse --short HEAD) + + echo "Number of new commits since last run: $commits_count, latest commit hash: $latest_commit_hash" + + echo "commits-count=$commits_count" >> $GITHUB_OUTPUT + echo "latest-commit-hash=$latest_commit_hash" >> $GITHUB_OUTPUT + pack: name: Pack Payu runs-on: ubuntu-latest + needs: + - check-for-payu-updates + # Deploy payu if manually triggered, env-dev.yml has been updated, or if there's new commits to payu repository + if: > + needs.check-for-payu-updates.outputs.commits-count > 0 || + github.event_name == 'push' || + github.event_name == 'workflow_dispatch' + outputs: + name: ${{ steps.payu.outputs.name }} + version: ${{ steps.payu.outputs.version }} steps: - uses: actions/checkout@v4 + - name: Get Payu Version + id: payu + run: | + # Set version to datetime and last short commit hash of payu + NOW=$(date -u +"%Y%m%dT%H%M%SZ") + COMMIT_HASH=${{ needs.check-for-payu-updates.outputs.last-commit-hash }} + VERSION="$NOW-$COMMIT_HASH" + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "name=payu-dev-$VERSION" >> $GITHUB_OUTPUT + - name: Setup Micromamba uses: mamba-org/setup-micromamba@422500192359a097648154e8db4e39bdb6c6eed7 #v1.8.1 with: micromamba-version: '1.5.3-0' - environment-file: env.yml - environment-name: ${{ env.NAME }} + environment-file: env-dev.yml + environment-name: ${{ steps.payu.outputs.name }} generate-run-shell: true - - - name: Create Pack and Lockfile + + - name: Create Pack shell: micromamba-shell {0} - run: | - conda pack - conda-lock lock --file env.yml --platform linux-64 --micromamba --lockfile ${{ env.NAME }}.conda-lock.yml + run: conda pack - name: Upload Artifact uses: actions/upload-artifact@v3 with: - name: ${{ env.NAME }} + name: payu-dev if-no-files-found: error path: | - ${{ env.NAME }}.tar.gz - ${{ env.NAME }}.conda-lock.yml + ${{ steps.payu.outputs.name }}.tar.gz deploy: runs-on: ubuntu-latest needs: - pack environment: Gadi - permissions: - contents: write + env: + NAME: ${{ needs.pack.outputs.name }} + VERSION: ${{ needs.pack.outputs.version }} steps: - uses: actions/download-artifact@v3.0.2 with: - name: ${{ env.NAME }} + name: payu-dev - uses: access-nri/actions/.github/actions/setup-ssh@main id: ssh @@ -64,7 +126,7 @@ jobs: run: | rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \ ${{ env.NAME }}.tar.gz \ - ${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ vars.PACK_LOCATION }} + ${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ vars.PRERELEASE_PACK_LOCATION }} - name: Deploy to Gadi env: diff --git a/env-dev.yml b/env-dev.yml index 6067869..e8d7c26 100644 --- a/env-dev.yml +++ b/env-dev.yml @@ -1,4 +1,4 @@ -name: payu +name: payu-dev channels: - accessnri - coecms @@ -6,11 +6,10 @@ channels: dependencies: # Use latest changes from payu's master branch - pip: - - git+https://github.com/payu-org/payu.git + - git+https://github.com/payu-org/payu.git@master - coecms::f90nml - conda-lock - conda-pack - - squashfs-tools - gh - git - nco From d5b30d2e850e5f4304ea95eea2bd0640c4954493 Mon Sep 17 00:00:00 2001 From: Jo Basevi Date: Fri, 14 Jun 2024 13:12:48 +1000 Subject: [PATCH 06/10] Update deploy-dev.yml: Install to unique location, add payu/dev symlinks and remove old environments --- .github/workflows/deploy-dev.yml | 53 ++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index bc5ec25..2502c79 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -1,7 +1,7 @@ name: Deploy Dev Enviroment on: # schedule: -# - cron: '0 * * * *' # Runs every hour +# - cron: '0 * * * *' # Runs every hour #TODO: Figure out cron push: branches: - main @@ -77,9 +77,9 @@ jobs: # Set version to datetime and last short commit hash of payu NOW=$(date -u +"%Y%m%dT%H%M%SZ") COMMIT_HASH=${{ needs.check-for-payu-updates.outputs.last-commit-hash }} - VERSION="$NOW-$COMMIT_HASH" + VERSION="dev-$NOW-$COMMIT_HASH" echo "version=$VERSION" >> $GITHUB_OUTPUT - echo "name=payu-dev-$VERSION" >> $GITHUB_OUTPUT + echo "name=payu-$VERSION" >> $GITHUB_OUTPUT - name: Setup Micromamba uses: mamba-org/setup-micromamba@422500192359a097648154e8db4e39bdb6c6eed7 #v1.8.1 @@ -131,22 +131,55 @@ jobs: - name: Deploy to Gadi env: PAYU_ENVIRONMENT_LOCATION: ${{ vars.PRERELEASE_DEPLOYMENT_LOCATION }}/${{ env.VERSION }} + PAYU_ENVIRONMENT_SYMLINK: ${{ vars.PRERELEASE_DEPLOYMENT_LOCATION }}/dev + PAYU_MODULE_SYMLINK: ${{ vars.PRERELEASE_MODULE_LOCATION }}/dev + PAYU_PACKED_ENVIRONMENT: ${{ vars.PRERELEASE_PACK_LOCATION }}/${{ env.NAME }}.tar.gz run: | ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' - - # Remove previous enviroment if it exists - if [ -d ${{ env.PAYU_ENVIRONMENT_LOCATION }} ]; then - rm -rf ${{ env.PAYU_ENVIRONMENT_LOCATION }} - fi + + # Create list of previous payu/dev environments to remove later + old_versions=$(ls "${{ vars.PRERELEASE_DEPLOYMENT_LOCATION }}" | grep -E '^dev-[0-9]{8}T[0-9]{6}Z-.*') # Unpack conda enviroment mkdir ${{ env.PAYU_ENVIRONMENT_LOCATION }} - tar -xzf ${{ vars.PACK_LOCATION }}/${{ env.NAME }}.tar.gz -C ${{ env.PAYU_ENVIRONMENT_LOCATION }} + if [ $? -ne 0 ]; then + exit $? + fi + tar -xzf ${{ env.PAYU_PACKED_ENVIRONMENT }} -C ${{ env.PAYU_ENVIRONMENT_LOCATION }} source ${{ env.PAYU_ENVIRONMENT_LOCATION }}/bin/activate conda-unpack payu --version source ${{ env.PAYU_ENVIRONMENT_LOCATION }}/bin/deactivate + echo "New payu/dev environment unpacked to ${{ env.PAYU_ENVIRONMENT_LOCATION }}" + + # Check if payu/dev symlink already exists + if [ -L "${{ env.PAYU_ENVIRONMENT_SYMLINK}}"]; then + # Get previous version of payu/dev + previous_env_path=$(readlink -f "${{ env.PAYU_ENVIRONMENT_SYMLINK}}") + previous_ver=$(basename "$previous_env_path") + echo "Previous payu/dev version $previous_ver" + + # Remove previous version from old versions - to ensure active + # environments at time of deployment are not deleted + old_versions=$(echo "$old_versions" | grep -v "$previous_ver") + + # Unlink the symlink + unlink ${{ env.PAYU_ENVIRONMENT_SYMLINK}} + fi + + # Create the payu/dev symlink in pre-release apps + ln -s ${{ env.PAYU_ENVIRONMENT_LOCATION }} ${{ env.PAYU_ENVIRONMENT_SYMLINK }} # Setup modulefile symlink - ln -s ${{ vars.MODULE_LOCATION }}/.common ${{ vars.PRERELEASE_MODULE_LOCATION }}/${{ env.VERSION }} + ln -sf ${{ vars.MODULE_LOCATION }}/.common ${{ env.PAYU_MODULE_SYMLINK }} + + # Remove old versions of environments + for version in $old_versions; do + echo "Removing old payu/dev environment version $version" + rm -rf "${{ vars.PRERELEASE_DEPLOYMENT_LOCATION }}/$version" + done + + # Remove packed environment file + echo "Removing conda-packed environment file ${{ env.PAYU_PACKED_ENVIRONMENT }}" + rm -rf ${{ env.PAYU_PACKED_ENVIRONMENT }} EOT From 2d5f2290375935f3cc9c23c215fd8e281a770742 Mon Sep 17 00:00:00 2001 From: Jo Basevi Date: Fri, 14 Jun 2024 14:53:27 +1000 Subject: [PATCH 07/10] Update update-module.yml: Generate different modulefiles for release/pre-release modules --- .github/workflows/deploy-dev.yml | 4 ++-- .github/workflows/update-module.yml | 22 +++++++++++++++++++++- modules/.common | 10 +--------- modules/env.sh | 11 +---------- 4 files changed, 25 insertions(+), 22 deletions(-) diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index 2502c79..837c5aa 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -153,7 +153,7 @@ jobs: echo "New payu/dev environment unpacked to ${{ env.PAYU_ENVIRONMENT_LOCATION }}" # Check if payu/dev symlink already exists - if [ -L "${{ env.PAYU_ENVIRONMENT_SYMLINK}}"]; then + if [ -L "${{ env.PAYU_ENVIRONMENT_SYMLINK}}" ]; then # Get previous version of payu/dev previous_env_path=$(readlink -f "${{ env.PAYU_ENVIRONMENT_SYMLINK}}") previous_ver=$(basename "$previous_env_path") @@ -171,7 +171,7 @@ jobs: ln -s ${{ env.PAYU_ENVIRONMENT_LOCATION }} ${{ env.PAYU_ENVIRONMENT_SYMLINK }} # Setup modulefile symlink - ln -sf ${{ vars.MODULE_LOCATION }}/.common ${{ env.PAYU_MODULE_SYMLINK }} + ln -sf ${{ vars.PRERELEASE_MODULE_LOCATION }}/.common ${{ env.PAYU_MODULE_SYMLINK }} # Remove old versions of environments for version in $old_versions; do diff --git a/.github/workflows/update-module.yml b/.github/workflows/update-module.yml index 7709f48..c754c78 100644 --- a/.github/workflows/update-module.yml +++ b/.github/workflows/update-module.yml @@ -20,8 +20,28 @@ jobs: ${{ secrets.HOST_DATA }} private-key: ${{ secrets.SSH_KEY }} + - name: Generate release modulesfiles + run: | + mkdir -p release/modules + sed 's|{{MODULE_LOCATION}}|'"${{ vars.MODULE_LOCATION }}"'|g' modules/.common > release/modules/.common + sed 's|{{APPS_LOCATION}}|'"${{ vars.APPS_LOCATION }}"'|g' modules/env.sh > release/modules/env.sh + chmod +x release/modules/env.sh + + - name: Generate pre-release modulesfiles + run: | + mkdir -p prerelease/modules + sed 's|{{MODULE_LOCATION}}|'"${{ vars.PRERELEASE_MODULE_LOCATION }}"'|g' modules/.common > prerelease/modules/.common + sed 's|{{APPS_LOCATION}}|'"${{ vars.PRERELEASE_APPS_LOCATION }}"'|g' modules/env.sh > prerelease/modules/env.sh + chmod +x prerelease/modules/env.sh + - name: Copy modulefiles to Gadi run: | + # Rsync release modulefiles rsync -rvp -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \ - modules/ \ + release/modules/ \ ${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ vars.MODULE_LOCATION }} + + # Rsync development modulefiles + rsync -rvp -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \ + prerelease/modules/ \ + ${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ vars.PRERELEASE_MODULE_LOCATION }} diff --git a/modules/.common b/modules/.common index 9e2cb78..e8fe84e 100644 --- a/modules/.common +++ b/modules/.common @@ -9,16 +9,8 @@ conflict payu # Name of this module's environment, e.g. payu/VERSION set payuname [module-info name] -# Check if modulefile is in prerelease folder -if {[string match "/g/data/vk83/prerelease/*" "$ModulesCurrentModulefile"]} { - set is_prerelease true -} else { - set is_prerelease false -} - # Get the environment variables from activate script -set payuenv [exec /bin/env -i \ - /g/data/vk83/modules/payu/env.sh $payuname $is_prerelease] +set payuenv [exec /bin/env -i {{MODULE_LOCATION}}/env.sh $payuname] # Convert the environment into module commands set lines [split $payuenv '\n'] diff --git a/modules/env.sh b/modules/env.sh index 8705e73..0045d4a 100755 --- a/modules/env.sh +++ b/modules/env.sh @@ -2,14 +2,5 @@ # Prints the environment variables when activating un-packed environment export PATH=/usr/bin:/bin -name=$1 -is_prerelease=$2 - -if [ "$is_prerelease" = true ]; then - # Use prerelease location for apps - source /g/data/vk83/prerelease/apps/$name/bin/activate -else - # Use release location for apps - source /g/data/vk83/apps/$name/bin/activate -fi +source {{APPS_LOCATION}}/$1/bin/activate /bin/env \ No newline at end of file From f0831d51f45e25a24d8ab23f036e092aaceca0e1 Mon Sep 17 00:00:00 2001 From: Jo Basevi Date: Fri, 14 Jun 2024 15:27:31 +1000 Subject: [PATCH 08/10] deploy-dev.yml: Use more consise names for environment vars --- .github/workflows/deploy-dev.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index 837c5aa..565e0b7 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -130,10 +130,10 @@ jobs: - name: Deploy to Gadi env: - PAYU_ENVIRONMENT_LOCATION: ${{ vars.PRERELEASE_DEPLOYMENT_LOCATION }}/${{ env.VERSION }} - PAYU_ENVIRONMENT_SYMLINK: ${{ vars.PRERELEASE_DEPLOYMENT_LOCATION }}/dev - PAYU_MODULE_SYMLINK: ${{ vars.PRERELEASE_MODULE_LOCATION }}/dev - PAYU_PACKED_ENVIRONMENT: ${{ vars.PRERELEASE_PACK_LOCATION }}/${{ env.NAME }}.tar.gz + ENVIRONMENT_LOCATION: ${{ vars.PRERELEASE_DEPLOYMENT_LOCATION }}/${{ env.VERSION }} + ENVIRONMENT_SYMLINK: ${{ vars.PRERELEASE_DEPLOYMENT_LOCATION }}/dev + MODULE_SYMLINK: ${{ vars.PRERELEASE_MODULE_LOCATION }}/dev + PACKED_ENVIRONMENT: ${{ vars.PRERELEASE_PACK_LOCATION }}/${{ env.NAME }}.tar.gz run: | ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' @@ -141,21 +141,21 @@ jobs: old_versions=$(ls "${{ vars.PRERELEASE_DEPLOYMENT_LOCATION }}" | grep -E '^dev-[0-9]{8}T[0-9]{6}Z-.*') # Unpack conda enviroment - mkdir ${{ env.PAYU_ENVIRONMENT_LOCATION }} + mkdir ${{ env.ENVIRONMENT_LOCATION }} if [ $? -ne 0 ]; then exit $? fi - tar -xzf ${{ env.PAYU_PACKED_ENVIRONMENT }} -C ${{ env.PAYU_ENVIRONMENT_LOCATION }} - source ${{ env.PAYU_ENVIRONMENT_LOCATION }}/bin/activate + tar -xzf ${{ env.PACKED_ENVIRONMENT }} -C ${{ env.ENVIRONMENT_LOCATION }} + source ${{ env.ENVIRONMENT_LOCATION }}/bin/activate conda-unpack payu --version - source ${{ env.PAYU_ENVIRONMENT_LOCATION }}/bin/deactivate - echo "New payu/dev environment unpacked to ${{ env.PAYU_ENVIRONMENT_LOCATION }}" + source ${{ env.ENVIRONMENT_LOCATION }}/bin/deactivate + echo "New payu/dev environment unpacked to ${{ env.ENVIRONMENT_LOCATION }}" # Check if payu/dev symlink already exists - if [ -L "${{ env.PAYU_ENVIRONMENT_SYMLINK}}" ]; then + if [ -L "${{ env.ENVIRONMENT_SYMLINK}}" ]; then # Get previous version of payu/dev - previous_env_path=$(readlink -f "${{ env.PAYU_ENVIRONMENT_SYMLINK}}") + previous_env_path=$(readlink -f "${{ env.ENVIRONMENT_SYMLINK}}") previous_ver=$(basename "$previous_env_path") echo "Previous payu/dev version $previous_ver" @@ -164,14 +164,14 @@ jobs: old_versions=$(echo "$old_versions" | grep -v "$previous_ver") # Unlink the symlink - unlink ${{ env.PAYU_ENVIRONMENT_SYMLINK}} + unlink ${{ env.ENVIRONMENT_SYMLINK}} fi # Create the payu/dev symlink in pre-release apps - ln -s ${{ env.PAYU_ENVIRONMENT_LOCATION }} ${{ env.PAYU_ENVIRONMENT_SYMLINK }} + ln -s ${{ env.ENVIRONMENT_LOCATION }} ${{ env.ENVIRONMENT_SYMLINK }} # Setup modulefile symlink - ln -sf ${{ vars.PRERELEASE_MODULE_LOCATION }}/.common ${{ env.PAYU_MODULE_SYMLINK }} + ln -sf ${{ vars.PRERELEASE_MODULE_LOCATION }}/.common ${{ env.MODULE_SYMLINK }} # Remove old versions of environments for version in $old_versions; do @@ -180,6 +180,6 @@ jobs: done # Remove packed environment file - echo "Removing conda-packed environment file ${{ env.PAYU_PACKED_ENVIRONMENT }}" - rm -rf ${{ env.PAYU_PACKED_ENVIRONMENT }} + echo "Removing conda-packed environment file ${{ env.PACKED_ENVIRONMENT }}" + rm -rf ${{ env.PACKED_ENVIRONMENT }} EOT From c36f1d0048ca1bb31b21e3fe9193a35cbde27859 Mon Sep 17 00:00:00 2001 From: Jo Basevi Date: Mon, 17 Jun 2024 09:30:40 +1000 Subject: [PATCH 09/10] Update deploy-dev.yml: Add in review suggestions --- .github/workflows/deploy-dev.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index 565e0b7..e840ce7 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -24,11 +24,7 @@ jobs: GH_TOKEN: ${{ secrets.GH_TOKEN }} run: | # Fetch the last successful workflow run time - response=$(gh api \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - /repos/access-nri/payu-condaenv/actions/workflows/deploy-dev.yml/runs?status=success&per_page=1) - last_run_time=$(echo $response | jq -r '.workflow_runs[0].updated_at') + last_run_time=$(gh run list --status success --workflow deploy-dev.yml --json updatedAt --jq .[0].updatedAt) echo "Last successful workflow run time: $last_run_time" echo "last-run-time=$last_run_time" >> $GITHUB_OUTPUT @@ -141,16 +137,13 @@ jobs: old_versions=$(ls "${{ vars.PRERELEASE_DEPLOYMENT_LOCATION }}" | grep -E '^dev-[0-9]{8}T[0-9]{6}Z-.*') # Unpack conda enviroment - mkdir ${{ env.ENVIRONMENT_LOCATION }} - if [ $? -ne 0 ]; then - exit $? - fi + mkdir ${{ env.ENVIRONMENT_LOCATION }} || exit $? tar -xzf ${{ env.PACKED_ENVIRONMENT }} -C ${{ env.ENVIRONMENT_LOCATION }} source ${{ env.ENVIRONMENT_LOCATION }}/bin/activate conda-unpack payu --version source ${{ env.ENVIRONMENT_LOCATION }}/bin/deactivate - echo "New payu/dev environment unpacked to ${{ env.ENVIRONMENT_LOCATION }}" + echo "::notice::New payu/dev environment unpacked to ${{ env.ENVIRONMENT_LOCATION }}" # Check if payu/dev symlink already exists if [ -L "${{ env.ENVIRONMENT_SYMLINK}}" ]; then From 43247a71dec86a7eb8b5b3eecd4c3c102d1ce429 Mon Sep 17 00:00:00 2001 From: Jo Basevi Date: Mon, 17 Jun 2024 09:58:41 +1000 Subject: [PATCH 10/10] Update deploy-dev.yml: Update cron to run every morning --- .github/workflows/deploy-dev.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index e840ce7..5746d8b 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -1,7 +1,7 @@ name: Deploy Dev Enviroment on: -# schedule: -# - cron: '0 * * * *' # Runs every hour #TODO: Figure out cron + schedule: + - cron: '0 21 * * *' # Runs every morning at 7AM AEST push: branches: - main