generated from ACCESS-NRI/template
-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #26 from ACCESS-NRI/22-automatic-dev-build
Add payu dev environment deployment workflow - this creates an environment in prerelease that has the latest changes from payu
- Loading branch information
Showing
6 changed files
with
239 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
name: env-dev.yml | ||
on: | ||
pull_request: | ||
paths: | ||
- 'env-dev.yml' | ||
jobs: | ||
check: | ||
name: Check Env Dev | ||
runs-on: ubuntu-latest | ||
steps: | ||
- uses: actions/checkout@v4 | ||
|
||
- name: Env-dev Valid | ||
uses: mamba-org/setup-micromamba@422500192359a097648154e8db4e39bdb6c6eed7 #v1.8.1 | ||
with: | ||
micromamba-version: '1.5.3-0' | ||
environment-file: env-dev.yml | ||
environment-name: test-payu-environment-dev | ||
generate-run-shell: false |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,178 @@ | ||
name: Deploy Dev Enviroment | ||
on: | ||
schedule: | ||
- cron: '0 21 * * *' # Runs every morning at 7AM AEST | ||
push: | ||
branches: | ||
- main | ||
paths: | ||
- 'env-dev.yml' | ||
workflow_dispatch: # Allows manual triggering | ||
jobs: | ||
check-for-payu-updates: | ||
runs-on: ubuntu-latest | ||
outputs: | ||
commits-count: ${{ steps.check-payu-commits.outputs.commits-count }} | ||
last-commit-hash: ${{ steps.check-payu-commits.outputs.latest-commit-hash }} | ||
steps: | ||
- name: Checkout current repository | ||
uses: actions/checkout@v4 | ||
|
||
- name: Get last workflow run time | ||
id: last-run-time | ||
env: | ||
GH_TOKEN: ${{ secrets.GH_TOKEN }} | ||
run: | | ||
# Fetch the last successful workflow run time | ||
last_run_time=$(gh run list --status success --workflow deploy-dev.yml --json updatedAt --jq .[0].updatedAt) | ||
echo "Last successful workflow run time: $last_run_time" | ||
echo "last-run-time=$last_run_time" >> $GITHUB_OUTPUT | ||
- name: Checkout payu repository | ||
uses: actions/checkout@v4 | ||
with: | ||
repository: payu-org/payu | ||
path: payu | ||
ref: master | ||
|
||
- name: Check commits in payu repository | ||
id: check-payu-commits | ||
run: | | ||
# Check for any commits since last successful runtime | ||
last_run_time="${{ steps.last-run-time.outputs.last-run-time }}" | ||
commits_count=$(git -C ./payu rev-list --count --since="$last_run_time" master) | ||
# Get latest commit hash | ||
latest_commit_hash=$(git -C ./payu rev-parse --short HEAD) | ||
echo "Number of new commits since last run: $commits_count, latest commit hash: $latest_commit_hash" | ||
echo "commits-count=$commits_count" >> $GITHUB_OUTPUT | ||
echo "latest-commit-hash=$latest_commit_hash" >> $GITHUB_OUTPUT | ||
pack: | ||
name: Pack Payu | ||
runs-on: ubuntu-latest | ||
needs: | ||
- check-for-payu-updates | ||
# Deploy payu if manually triggered, env-dev.yml has been updated, or if there's new commits to payu repository | ||
if: > | ||
needs.check-for-payu-updates.outputs.commits-count > 0 || | ||
github.event_name == 'push' || | ||
github.event_name == 'workflow_dispatch' | ||
outputs: | ||
name: ${{ steps.payu.outputs.name }} | ||
version: ${{ steps.payu.outputs.version }} | ||
steps: | ||
- uses: actions/checkout@v4 | ||
|
||
- name: Get Payu Version | ||
id: payu | ||
run: | | ||
# Set version to datetime and last short commit hash of payu | ||
NOW=$(date -u +"%Y%m%dT%H%M%SZ") | ||
COMMIT_HASH=${{ needs.check-for-payu-updates.outputs.last-commit-hash }} | ||
VERSION="dev-$NOW-$COMMIT_HASH" | ||
echo "version=$VERSION" >> $GITHUB_OUTPUT | ||
echo "name=payu-$VERSION" >> $GITHUB_OUTPUT | ||
- name: Setup Micromamba | ||
uses: mamba-org/setup-micromamba@422500192359a097648154e8db4e39bdb6c6eed7 #v1.8.1 | ||
with: | ||
micromamba-version: '1.5.3-0' | ||
environment-file: env-dev.yml | ||
environment-name: ${{ steps.payu.outputs.name }} | ||
generate-run-shell: true | ||
|
||
- name: Create Pack | ||
shell: micromamba-shell {0} | ||
run: conda pack | ||
|
||
- name: Upload Artifact | ||
uses: actions/upload-artifact@v3 | ||
with: | ||
name: payu-dev | ||
if-no-files-found: error | ||
path: | | ||
${{ steps.payu.outputs.name }}.tar.gz | ||
deploy: | ||
runs-on: ubuntu-latest | ||
needs: | ||
- pack | ||
environment: Gadi | ||
env: | ||
NAME: ${{ needs.pack.outputs.name }} | ||
VERSION: ${{ needs.pack.outputs.version }} | ||
steps: | ||
- uses: actions/[email protected] | ||
with: | ||
name: payu-dev | ||
|
||
- uses: access-nri/actions/.github/actions/setup-ssh@main | ||
id: ssh | ||
with: | ||
hosts: | | ||
${{ secrets.HOST_DATA }} | ||
${{ secrets.HOST }} | ||
private-key: ${{ secrets.SSH_KEY }} | ||
|
||
- name: Copy to Gadi | ||
run: | | ||
rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \ | ||
${{ env.NAME }}.tar.gz \ | ||
${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ vars.PRERELEASE_PACK_LOCATION }} | ||
- name: Deploy to Gadi | ||
env: | ||
ENVIRONMENT_LOCATION: ${{ vars.PRERELEASE_DEPLOYMENT_LOCATION }}/${{ env.VERSION }} | ||
ENVIRONMENT_SYMLINK: ${{ vars.PRERELEASE_DEPLOYMENT_LOCATION }}/dev | ||
MODULE_SYMLINK: ${{ vars.PRERELEASE_MODULE_LOCATION }}/dev | ||
PACKED_ENVIRONMENT: ${{ vars.PRERELEASE_PACK_LOCATION }}/${{ env.NAME }}.tar.gz | ||
run: | | ||
ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' | ||
# Create list of previous payu/dev environments to remove later | ||
old_versions=$(ls "${{ vars.PRERELEASE_DEPLOYMENT_LOCATION }}" | grep -E '^dev-[0-9]{8}T[0-9]{6}Z-.*') | ||
# Unpack conda enviroment | ||
mkdir ${{ env.ENVIRONMENT_LOCATION }} || exit $? | ||
tar -xzf ${{ env.PACKED_ENVIRONMENT }} -C ${{ env.ENVIRONMENT_LOCATION }} | ||
source ${{ env.ENVIRONMENT_LOCATION }}/bin/activate | ||
conda-unpack | ||
payu --version | ||
source ${{ env.ENVIRONMENT_LOCATION }}/bin/deactivate | ||
echo "::notice::New payu/dev environment unpacked to ${{ env.ENVIRONMENT_LOCATION }}" | ||
# Check if payu/dev symlink already exists | ||
if [ -L "${{ env.ENVIRONMENT_SYMLINK}}" ]; then | ||
# Get previous version of payu/dev | ||
previous_env_path=$(readlink -f "${{ env.ENVIRONMENT_SYMLINK}}") | ||
previous_ver=$(basename "$previous_env_path") | ||
echo "Previous payu/dev version $previous_ver" | ||
# Remove previous version from old versions - to ensure active | ||
# environments at time of deployment are not deleted | ||
old_versions=$(echo "$old_versions" | grep -v "$previous_ver") | ||
# Unlink the symlink | ||
unlink ${{ env.ENVIRONMENT_SYMLINK}} | ||
fi | ||
# Create the payu/dev symlink in pre-release apps | ||
ln -s ${{ env.ENVIRONMENT_LOCATION }} ${{ env.ENVIRONMENT_SYMLINK }} | ||
# Setup modulefile symlink | ||
ln -sf ${{ vars.PRERELEASE_MODULE_LOCATION }}/.common ${{ env.MODULE_SYMLINK }} | ||
# Remove old versions of environments | ||
for version in $old_versions; do | ||
echo "Removing old payu/dev environment version $version" | ||
rm -rf "${{ vars.PRERELEASE_DEPLOYMENT_LOCATION }}/$version" | ||
done | ||
# Remove packed environment file | ||
echo "Removing conda-packed environment file ${{ env.PACKED_ENVIRONMENT }}" | ||
rm -rf ${{ env.PACKED_ENVIRONMENT }} | ||
EOT |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
name: payu-dev | ||
channels: | ||
- accessnri | ||
- coecms | ||
- conda-forge | ||
dependencies: | ||
# Use latest changes from payu's master branch | ||
- pip: | ||
- git+https://github.com/payu-org/payu.git@master | ||
- coecms::f90nml | ||
- conda-lock | ||
- conda-pack | ||
- gh | ||
- git | ||
- nco | ||
- pytest | ||
- openssh>=8.3 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,6 @@ | ||
#!/bin/bash | ||
# Prints the environment variables when activating un-packed payu environment | ||
# Prints the environment variables when activating un-packed environment | ||
export PATH=/usr/bin:/bin | ||
source /g/data/vk83/apps/$1/bin/activate | ||
|
||
source {{APPS_LOCATION}}/$1/bin/activate | ||
/bin/env |