diff --git a/.github/actions/parse-ci-config/README.md b/.github/actions/parse-ci-config/README.md new file mode 100644 index 0000000..00a20d7 --- /dev/null +++ b/.github/actions/parse-ci-config/README.md @@ -0,0 +1,40 @@ +# parse-ci-config + +This action parses the CI testing configuration file. The caller of the action needs to checkout the branch where the config file is defined before running this action. + +## Inputs + +| Name | Type | Description | Required | Example | +| ---- | ---- | ----------- | -------- | ------- | +| check | `string` | The type of check/test to run | true | `scheduled` | +| branch-or-tag | `string` | The name of git branch or tag | true | `release-1deg_jra55_ryf-2.0` | +| config-filepath | `string` | Path to configuration file | true | `config/ci.json` | + +## Outputs + +| Name | Type | Description | Example | +| ---- | ---- | ----------- | -------- | +| markers | `string` | Markers used for the pytest checks, in the python format | `checksum` | +| model-config-tests-version | `string` | The version of the model-config-tests | `0.0.1` | +| python-version | `string` | The python version used to create test virtual environment | `3.11.0` | + +## Example usage + +```yaml +# --------- + steps: + - name: Checkout main + uses: actions/checkout@v4 + with: + ref: main + + - name: Read scheduled test config + id: scheduled-config + uses: access-nri/model-config-tests/.github/actions/parse-ci-config@main + with: + check: scheduled + branch-or-tag: "release-1deg_jra55_ryf-2.0" + config-filepath: "config/ci.json" +``` + + diff --git a/.github/actions/parse-ci-config/action.yml b/.github/actions/parse-ci-config/action.yml new file mode 100644 index 0000000..6948320 --- /dev/null +++ b/.github/actions/parse-ci-config/action.yml @@ -0,0 +1,53 @@ +name: Parse CI Config File +description: Action to parse model-config-tests configurations for CI tests +inputs: + check: + required: true + description: Type of check/test to run (e.g. "reproducibility", "qa" or "scheduled") + branch-or-tag: + required: true + description: Name of Git branch or tag to run CI testing on + config-filepath: + required: true + description: Path to CI configuration file +outputs: + model-config-tests-version: + value: ${{ steps.read-config.outputs.model-config-tests-version }} + description: A version of the model-config-tests package + python-version: + value: ${{ steps.read-config.outputs.python-version }} + description: The python version used to create test virtual environment + markers: + value: ${{ steps.read-config.outputs.markers }} + description: A python expression of markers to pass to model-config-tests pytests +runs: + using: "composite" + steps: + - name: Read Configuration File + shell: bash + id: read-config + run: | + # Fall back to default config values if not defined for a given branch or tag + output=$(jq --arg branch "${{ inputs.branch-or-tag }}" --arg check "${{ inputs.check }}" ' + { + "model-config-tests-version": ( + .[$check][$branch]["model-config-tests-version"] // + .[$check].default["model-config-tests-version"] // + .default["model-config-tests-version"] + ), + "python-version": ( + .[$check][$branch]["python-version"] // + .[$check].default["python-version"] // + .default["python-version"] + ), + "markers": ( + .[$check][$branch].markers // + .[$check].default.markers // + .default.markers + ), + } + ' "${{ inputs.config-filepath }}") + + echo "markers=$(echo "$output" | jq -r '.["markers"]')" >> $GITHUB_OUTPUT + echo "python-version=$(echo "$output" | jq -r '.["python-version"]')" >> $GITHUB_OUTPUT + echo "model-config-tests-version=$(echo "$output" | jq -r '.["model-config-tests-version"]')" >> $GITHUB_OUTPUT diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index a511714..3e44258 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -4,6 +4,12 @@ on: pull_request: branches: - 'main' + paths-ignore: + - .github/** + - .* + - '**.md' + - COPYRIGHT.txt + - LICENSE jobs: pre-commit: diff --git a/.github/workflows/config-pr-1-ci.yml b/.github/workflows/config-pr-1-ci.yml new file mode 100644 index 0000000..da00e07 --- /dev/null +++ b/.github/workflows/config-pr-1-ci.yml @@ -0,0 +1,305 @@ +name: PR Checks +on: + workflow_call: + # Workflows that call this workflow use the following triggers: + # pull_request: + # branches: + # - 'release-*' + # - 'dev-*' + # paths-ignore: + # # These are ignored because they don't have anything to do with the model itself + # - .github/** + # - tools/** + # - doc/** + # - .* + # - README.md +jobs: + commit-check: + name: Commit Check + # We run this job to check if the current commit was done during a workflow run. + # Such as when 'github-actions' bumps the metadata.yaml file or updates the checksums + # in the `testing` directory. + # This is so we don't recursively commit and check infinitely during this workflow. + runs-on: ubuntu-latest + outputs: + authorship: ${{ steps.head-commit.outputs.authorship }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + + - name: HEAD Commit Authorship + id: head-commit + run: echo "authorship=$(git log -1 --pretty=format:'%an')" >> $GITHUB_OUTPUT + + branch-check: + name: PR Source Branch Check + # This check is used as a precursor to any repro-ci checks - which are only fired + # on dev-* -> release-* PRs. + # This check is run to confirm that the source branch is of the form `dev-` + # and the target branch is of the form `release-`. We are being especially + # concerned with branch names because deployment to GitHub Environments can only + # be done on source branches with a certain pattern. See ACCESS-NRI/access-om2-configs#20. + if: needs.commit-check.outputs.authorship != vars.GH_ACTIONS_BOT_GIT_USER_NAME && startsWith(github.base_ref, 'release-') && startsWith(github.head_ref, 'dev-') + needs: + - commit-check + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + + - name: Compare Source and Target Config Names + # In this step, we cut the 'dev-' and 'release-' to compare config names directly. + run: | + source=$(cut --delimiter '-' --field 2- <<< "${{ github.head_ref }}") + target=$(cut --delimiter '-' --field 2- <<< "${{ github.base_ref }}") + if [[ "${source}" != "${target}" ]]; then + echo "::error::Config name of Source branch '${source}' does not match Target branch '${target}'" + exit 1 + fi + + - name: Failure Notifier + if: failure() + uses: access-nri/actions/.github/actions/pr-comment@main + with: + comment: | + :x: Automated Reproducibility testing cannot be run on this branch :x: + Source and Target branches must be of the form `dev-` and `release-` respectively, and `` must match between them. + Rename the Source branch or check the Target branch, and try again. + + config: + name: Read CI Testing Configuration + runs-on: ubuntu-latest + outputs: + qa-markers: ${{ steps.qa-config.outputs.markers }} + qa-python-version: ${{ steps.qa-config.outputs.python-version }} + qa-model-config-tests-version: ${{ steps.qa-config.outputs.model-config-tests-version }} + repro-markers: ${{ steps.repro-config.outputs.markers }} + repro-python-version: ${{ steps.repro-config.outputs.python-version }} + repro-model-config-tests-version: ${{ steps.repro-config.outputs.model-config-tests-version }} + steps: + - name: Checkout main + uses: actions/checkout@v4 + with: + ref: main + + - name: Validate `config/ci.json` + uses: access-nri/schema/.github/actions/validate-with-schema@main + with: + schema-version: ${{ vars.CI_JSON_SCHEMA_VERSION }} + meta-schema-version: draft-2020-12 + schema-location: au.org.access-nri/model/configuration/ci + data-location: config/ci.json + + - name: Read QA tests config + id: qa-config + uses: access-nri/model-config-tests/.github/actions/parse-ci-config@main + with: + check: qa + branch-or-tag: ${{ github.base_ref }} + config-filepath: "config/ci.json" + + - name: Read reproducibility tests config + id: repro-config + uses: access-nri/model-config-tests/.github/actions/parse-ci-config@main + with: + check: reproducibility + branch-or-tag: ${{ github.base_ref }} + config-filepath: "config/ci.json" + + qa-ci: + # Run quick, non-HPC tests on the runner. + name: QA CI Checks + needs: + - commit-check + - config + if: needs.commit-check.outputs.authorship != vars.GH_ACTIONS_BOT_GIT_USER_NAME + runs-on: ubuntu-latest + permissions: + checks: write + steps: + - name: Checkout PR ${{ github.event.pull_request.number }} + uses: actions/checkout@v4 + + - name: Checkout model-config-tests requirements + # This step checks out model-config-tests/pyproject.toml because it is + # used as a cache key for model-config-tests in the next step + uses: actions/checkout@v4 + with: + repository: access-nri/model-config-tests + path: model-config-tests + sparse-checkout: pyproject.toml + sparse-checkout-cone-mode: false + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ needs.config.outputs.qa-python-version }} + # We can use cache:pip to cache model-config-tests despite the actual + # config repository not containing any python, because we checkout + # model-config-tests/pyproject.toml, which contains data on the tests + cache: pip + cache-dependency-path: model-config-tests/pyproject.toml + + - name: Install model-config-tests + run: pip install model-config-tests=='${{ needs.config.outputs.qa-model-config-tests-version }}' + + - name: Invoke Simple CI Pytests + # We continue on error because we will let the checks generated in + # the next step speak to the state of the testing + continue-on-error: true + run: | + echo "Running pytest using '-m ${{ needs.config.outputs.qa-markers }}'" + model-config-tests \ + -m '${{ needs.config.outputs.qa-markers }}' \ + --target-branch '${{ github.base_ref }}' \ + --junitxml=./test_report.xml + + - name: Parse Test Report + id: tests + uses: EnricoMi/publish-unit-test-result-action/composite@e780361cd1fc1b1a170624547b3ffda64787d365 #v2.12.0 + with: + files: ./test_report.xml + comment_mode: off + check_run: true + check_name: QA Test Results + compare_to_earlier_commit: false + report_individual_runs: true + report_suite_logs: any + + repro-ci: + # Run the given config on the deployment GitHub Environment (`environment-name`) and + # upload the checksums and test details + needs: + - commit-check + - branch-check + - config + if: needs.commit-check.outputs.authorship != vars.GH_ACTIONS_BOT_GIT_USER_NAME && needs.branch-check.result == 'success' + uses: access-nri/model-config-tests/.github/workflows/test-repro.yml@main + with: + environment-name: Gadi + config-tag: ${{ github.head_ref }} + test-markers: ${{ needs.config.outputs.repro-markers }} + model-config-tests-version: ${{ needs.config.outputs.repro-model-config-tests-version }} + python-version: ${{ needs.config.outputs.repro-python-version }} + secrets: inherit + permissions: + contents: write + + check-checksum: + # Parse the test report and return pass/fail result + name: Check and Update Checksum + needs: + - repro-ci + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + checks: write + env: + TESTING_LOCAL_LOCATION: /opt/testing + outputs: + # URL for the parsed test results + check-run-url: ${{ steps.results.outputs.check-url }} + # Overall result of the checksum repro CI - `pass` (if reproducible), `fail` otherwise + result: ${{ steps.results.outputs.result }} + # Version of the checksum compared against the newly generated one + compared-checksum-version: ${{ steps.results.outputs.compared-checksum-version }} + steps: + - name: Download Newly Created Checksum + uses: actions/download-artifact@v3 + with: + name: ${{ needs.repro-ci.outputs.artifact-name }} + path: ${{ env.TESTING_LOCAL_LOCATION }} + + - uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + fetch-depth: 0 + fetch-tags: true + + - name: Parse Test Report + id: tests + uses: EnricoMi/publish-unit-test-result-action/composite@e780361cd1fc1b1a170624547b3ffda64787d365 #v2.12.0 + with: + files: ${{ env.TESTING_LOCAL_LOCATION }}/checksum/test_report.xml + comment_mode: off + check_run: true + check_name: Repro Test Results + compare_to_earlier_commit: false + report_individual_runs: true + report_suite_logs: any + + - name: Checksum Tests Results + id: results + run: | + echo "check-url=${{ fromJson(steps.tests.outputs.json).check_url }}" >> $GITHUB_OUTPUT + echo "compared-checksum-version=$(git describe --tags --abbrev=0)" >> $GITHUB_OUTPUT + if [ "${{ fromJson(steps.tests.outputs.json).stats.tests_fail }}" > 0 ]; then + echo "result=fail" >> $GITHUB_OUTPUT + else + echo "result=pass" >> $GITHUB_OUTPUT + fi + + bump-check: + name: Version Bump Check + # Check that the `.version` in the metadata.yaml has been modified in + # this PR. + needs: + - repro-ci + runs-on: ubuntu-latest + steps: + - name: Checkout PR Target + uses: actions/checkout@v4 + with: + ref: ${{ github.base_ref }} + path: target + + - name: Checkout PR Source + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + path: source + + - name: Modification Check + run: | + target=$(yq e '.version' ./target/metadata.yaml) + source=$(yq e '.version' ./source/metadata.yaml) + + if [[ "${source}" != "${target}" && "${source}" != "null" ]]; then + echo "::notice::The version has been modified to ${target}. Merging is now availible" + else + echo "::error::The version has not been modified in this PR. Merging is disallowed until an appropriate '!bump' is issued" + exit 1 + fi + + result: + name: Repro Result Notifier + # Notify the PR of the result of the Repro check + needs: + - repro-ci + - check-checksum + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Successful Release Comment + if: needs.check-checksum.outputs.result == 'pass' + uses: access-nri/actions/.github/actions/pr-comment@main + with: + comment: | + :white_check_mark: The Bitwise Reproducibility check succeeded when comparing against `${{ needs.check-checksum.outputs.compared-checksum-version }}` for this `Release` config. :white_check_mark: + For further information, the experiment can be found on Gadi at ${{ needs.repro-ci.outputs.experiment-location }}, and the test results at ${{ needs.check-checksum.outputs.check-run-url }}. + You must bump the minor version of this configuration - to bump the version, comment `!bump minor` or modify the `version` in `metadata.yaml`. The meaning of these version bumps is explained in the README.md, under `Config Tags`. + + - name: Failed Release Comment + if: needs.check-checksum.outputs.result == 'fail' + uses: access-nri/actions/.github/actions/pr-comment@main + with: + comment: | + :x: The Bitwise Reproducibility check failed when comparing against `${{ needs.check-checksum.outputs.compared-checksum-version }}` for this `Release` config. :x: + For further information, the experiment can be found on Gadi at ${{ needs.repro-ci.outputs.experiment-location }}, and the test results at ${{ needs.check-checksum.outputs.check-run-url }}. + You must bump the major version of this configuration before this PR is merged to account for this - to bump the version, comment `!bump major`or modify the `version` in `metadata.yaml`. The meaning of these version bumps is explained in the README.md, under `Config Tags`. diff --git a/.github/workflows/config-pr-2-confirm.yml b/.github/workflows/config-pr-2-confirm.yml new file mode 100644 index 0000000..20f1169 --- /dev/null +++ b/.github/workflows/config-pr-2-confirm.yml @@ -0,0 +1,164 @@ +# This workflow is used to do a major/minor version bump to the `metadata.yaml` file, +# through a comment on the PR. It also commits and pushes the checksum file, +# as this is the last stage before merging. +# This is not done automatically because users may want to modify their config +# based on the result of the reproducibility check. +name: Confirm +on: + workflow_call: + # Workflows that call this workflow use the following triggers: + # issue_comment: + # types: + # - created + # - edited +env: + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} +jobs: + bump-version: + name: Bump metadata.yaml + # Bump the `metadata.yaml` file if the comment is made on a PR and starts with '!bump' + if: github.event.issue.pull_request && startsWith(github.event.comment.body, '!bump') + runs-on: ubuntu-latest + permissions: + pull-requests: write + outputs: + # metadata.yaml version before being bumped + before: ${{ steps.bump.outputs.before }} + # metadata.yaml version after being bumped + after: ${{ steps.bump.outputs.after }} + # The type of bump - 'major' or 'minor' + type: ${{ steps.type.outputs.bump }} + steps: + - uses: access-nri/actions/.github/actions/react-to-comment@main + with: + reaction: rocket + token: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GH_COMMIT_CHECK_TOKEN }} + + - name: Checkout Associated PR ${{ github.event.issue.number }} + # Since the trigger for this workflow was on.issue_comment, we need + # to do a bit more wrangling to checkout the pull request + id: pr + run: gh pr checkout ${{ github.event.issue.number }} + + - name: Get Type of Bump + id: type + run: | + if [[ "${{ contains(github.event.comment.body, 'minor') }}" == "true" ]]; then + echo "bump=minor" >> $GITHUB_OUTPUT + elif [[ ${{ contains(github.event.comment.body, 'major') }} == "true" ]]; then + echo "bump=major" >> $GITHUB_OUTPUT + else + echo "::error::Comment was not of the form: '!bump [major|minor]'" + exit 1 + fi + + - name: Bump + # Regarding the regex in the script: `([0-9]+)\.([0-9]+)` is broken down into: + # `([0-9]+)`: Major version (eg. `12`) + # `\.`: Version separator (eg. `.`) + # `([0-9]+)`: Minor version (eg. `1`) + # which would give `12.1` + id: bump + run: | + version=$(yq '.version' metadata.yaml) + + if [[ "${version}" == "null" ]]; then + echo "before=null" >> $GITHUB_OUTPUT + echo "after=1.0" >> $GITHUB_OUTPUT + exit 0 + fi + + regex="([0-9]+)\.([0-9]+)" + if [[ $version =~ $regex ]]; then + major_version="${BASH_REMATCH[1]}" + minor_version="${BASH_REMATCH[2]}" + else + echo "::error::Invalid version format in metadata.yaml file!" + exit 1 + fi + + if [[ "${{ steps.type.outputs.bump }}" == "minor" ]]; then + minor_version=$((minor_version + 1)) + elif [[ "${{ steps.type.outputs.bump }}" == "major" ]]; then + major_version=$((major_version + 1)) + minor_version=0 + fi + new_version="${major_version}.${minor_version}" + echo "before=$version" >> $GITHUB_OUTPUT + echo "after=$new_version" >> $GITHUB_OUTPUT + + commit: + name: Commit metadata.yaml and Checksum + needs: + - bump-version + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + env: + ARTIFACT_LOCAL_LOCATION: /opt/artifact + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GH_COMMIT_CHECK_TOKEN }} + + - name: Checkout Associated PR ${{ github.event.issue.number }} + # Since the trigger for this workflow was on.issue_comment, we need + # to do a bit more wrangling to checkout the pull request and get the branch name + id: pr + run: | + gh pr checkout ${{ github.event.issue.number }} + echo "branch=$(git rev-parse --abbrev-ref HEAD)" >> $GITHUB_OUTPUT + + - name: Download Newly Created Checksum + # Given the PR branch, we need to find the latest associated workflow run + # on this branch we can then download the associated artifact + run: | + associated_run=$(gh run list \ + --json='databaseId,headBranch,updatedAt,status' \ + --jq='[.[] | select(.headBranch == "${{ steps.pr.outputs.branch }}" and .status == "completed")] | sort_by(.updatedAt) | last | .databaseId') + gh run download $associated_run -D ${{ env.ARTIFACT_LOCAL_LOCATION }} + + - name: Update metadata.yaml and Checksum files + run: | + yq -i '.version = "${{ needs.bump-version.outputs.after }}"' metadata.yaml + cp --recursive --verbose ${{ env.ARTIFACT_LOCAL_LOCATION }}/*/* testing + + - name: Commit and Push Updates + run: | + git config user.name ${{ vars.GH_ACTIONS_BOT_GIT_USER_NAME }} + git config user.email ${{ vars.GH_ACTIONS_BOT_GIT_USER_EMAIL }} + + if [[ "${{ needs.bump-version.outputs.type }}" == "minor" ]]; then + git commit -am "Bumped version to ${{ needs.bump-version.outputs.after }} as part of ${{ env.RUN_URL }}" + elif [[ "${{ needs.bump-version.outputs.type }}" == "major" ]]; then + git commit -am "Updated checksums and bumped version to ${{ needs.bump-version.outputs.after }} as part of ${{ env.RUN_URL }}" + fi + git push + + - name: Comment Success + uses: access-nri/actions/.github/actions/pr-comment@main + with: + comment: | + :white_check_mark: Version bumped from `${{ needs.bump-version.outputs.before }}` to `${{ needs.bump-version.outputs.after }}` :white_check_mark: + + failure-notifier: + name: Failure Notifier + if: failure() + needs: + - commit + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Comment Failure + uses: access-nri/actions/.github/actions/pr-comment@main + with: + comment: | + :x: Failed to bump VERSION or commit changes, see ${{ env.RUN_URL }} :x: diff --git a/.github/workflows/config-pr-3-bump-tag.yml b/.github/workflows/config-pr-3-bump-tag.yml new file mode 100644 index 0000000..7fcf8a4 --- /dev/null +++ b/.github/workflows/config-pr-3-bump-tag.yml @@ -0,0 +1,61 @@ +# This workflow is used to convert the `.version` in the `metadata.yaml` file into a valid `git tag` on push to `main`. +# We use the `.version` field in that file to denote the version of the config once a PR is merged. +name: Bump Tag +on: + workflow_call: + # Workflows that call this workflow use the following triggers: + # push: + # branches: + # - 'release-*' + # paths: + # - 'metadata.yaml' +jobs: + tag-update: + name: Check and Update Tag + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + with: + fetch-tags: true + + - name: Existing Tag Check + # Check if the tag already exists, if it does, we don't want to move it. + id: tag + run: | + VERSION=$(yq '.version' metadata.yaml) + VERSION_TAG=${{ github.ref_name }}-$VERSION + VERSION_TAG_ON_GIT=$(git tag -l $VERSION_TAG) + if [[ "$VERSION" == "null" ]]; then + echo "::warning::Version is null. Skipping." + exit 1 + elif [ -n "$VERSION_TAG_ON_GIT" ]; then + echo "::warning::Tag $VERSION_TAG already exists. Skipping." + echo "exists=true" >> $GITHUB_OUTPUT + else + echo "exists=false" >> $GITHUB_OUTPUT + echo "version=$VERSION_TAG" >> $GITHUB_OUTPUT + fi + + - name: Update Tag + if: steps.tag.outputs.exists == 'false' + # NOTE: Regarding the config user.name/user.email, see https://github.com/actions/checkout/pull/1184 + run: | + git config user.name ${{ vars.GH_ACTIONS_BOT_GIT_USER_NAME }} + git config user.email ${{ vars.GH_ACTIONS_BOT_GIT_USER_EMAIL }} + git tag ${{ steps.tag.outputs.version }} + git push --tags + + - name: Create Release + if: steps.tag.outputs.exists == 'false' + env: + TAG: ${{ steps.tag.outputs.version }} + IS_REPRO_BREAK: ${{ endsWith(steps.tag.outputs.version, '.0') && 'DOES' || 'does not' }} + uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 #v0.1.15 + with: + tag_name: ${{ env.TAG }} + name: Configuration ${{ env.TAG }} + body: | + This released configuration ${{ env.IS_REPRO_BREAK }} break reproducibility with released configurations before it. See the 'Config Tags' section in the `README.md` for more information. + generate_release_notes: true diff --git a/.github/workflows/config-schedule-1-ci.yml b/.github/workflows/config-schedule-1-ci.yml new file mode 100644 index 0000000..b23f117 --- /dev/null +++ b/.github/workflows/config-schedule-1-ci.yml @@ -0,0 +1,39 @@ +name: Scheduled Checks +on: + workflow_call: + # Workflows that call this workflow use the following triggers: + # workflow_dispatch: + # schedule: + # - cron: '0 0 1 * *' # once a month +jobs: + setup: + name: Setup Tasks + runs-on: ubuntu-latest + outputs: + tags: ${{ steps.get-released-config.outputs.tags }} + steps: + - uses: actions/checkout@v4 + with: + ref: main + + - name: Get all released configs + id: get-released-config + run: echo "tags=$(jq --compact-output --raw-output '.scheduled | del(.default) | keys[]' config/ci.json)" >> $GITHUB_OUTPUT + + repro-ci: + # We use this reusable workflow with a matrix strategy rather than calling repro-ci.yml, as + # we may want to do config-branch-specific tasks after the matrixed repro-ci.yml has completed. + needs: + - setup + strategy: + fail-fast: false + matrix: + config-tag: ${{ fromJson(needs.setup.outputs.tags) }} + uses: ./.github/workflows/config-schedule-2-start.yml + with: + config-tag: ${{ matrix.config-tag }} + secrets: inherit + permissions: + checks: write + contents: write + issues: write diff --git a/.github/workflows/config-schedule-2-start.yml b/.github/workflows/config-schedule-2-start.yml new file mode 100644 index 0000000..593cb21 --- /dev/null +++ b/.github/workflows/config-schedule-2-start.yml @@ -0,0 +1,145 @@ +name: Scheduled Checks - Branch Specific +on: + workflow_call: + inputs: + config-tag: + type: string + required: true + description: Tag associated with a config branch that is used in the reproducibility run +jobs: + config: + name: Read CI Testing Configuration + runs-on: ubuntu-latest + outputs: + markers: ${{ steps.scheduled-config.outputs.markers }} + python-version: ${{ steps.scheduled-config.outputs.python-version }} + model-config-tests-version: ${{ steps.scheduled-config.outputs.model-config-tests-version }} + steps: + - name: Checkout main + uses: actions/checkout@v4 + with: + ref: main + + - name: Validate `config/ci.json` + uses: access-nri/schema/.github/actions/validate-with-schema@main + with: + schema-version: ${{ vars.CI_JSON_SCHEMA_VERSION }} + meta-schema-version: draft-2020-12 + schema-location: au.org.access-nri/model/configuration/ci + data-location: config/ci.json + + - name: Read scheduled tests config + id: scheduled-config + uses: access-nri/model-config-tests/.github/actions/parse-ci-config@main + with: + check: scheduled + branch-or-tag: ${{ inputs.config-tag }} + config-filepath: "config/ci.json" + + repro-ci: + # Run the given config on the deployment Github Environment (`environment-name`) and upload + # the test results and checksum. + needs: + - config + uses: access-nri/model-config-tests/.github/workflows/test-repro.yml@main + with: + environment-name: Gadi + config-tag: ${{ inputs.config-tag }} + test-markers: ${{ needs.config.outputs.markers }} + model-config-tests-version: ${{ needs.config.outputs.model-config-tests-version }} + python-version: ${{ needs.config.outputs.python-version }} + secrets: inherit + permissions: + contents: write + + check-checksum: + name: Check Repro Results + # Parse the test results + needs: + - repro-ci + runs-on: ubuntu-latest + permissions: + checks: write + env: + TESTING_LOCAL_LOCATION: /opt/testing + outputs: + # URL for the parsed test results + check-run-url: ${{ steps.results.outputs.check-url }} + # Overall result of the checksum repro CI - `pass` (if reproducible), `fail` otherwise + result: ${{ steps.results.outputs.result }} + # Version of the checksum compared against the newly generated one + compared-checksum-version: ${{ steps.results.outputs.compared-checksum-version }} + steps: + - name: Download Newly Created Checksum + uses: actions/download-artifact@v3 + with: + name: ${{ needs.repro-ci.outputs.artifact-name }} + path: ${{ env.TESTING_LOCAL_LOCATION }} + + - name: Parse Test Report + id: tests + uses: EnricoMi/publish-unit-test-result-action/composite@e780361cd1fc1b1a170624547b3ffda64787d365 #v2.12.0 + with: + files: ${{ env.TESTING_LOCAL_LOCATION }}/checksum/test_report.xml + comment_mode: off + check_run: true + compare_to_earlier_commit: false + report_individual_runs: true + report_suite_logs: any + + - name: Checksum Tests Results + id: results + run: | + echo "check-url=${{ fromJson(steps.tests.outputs.json).check_url }}" >> $GITHUB_OUTPUT + echo "compared-checksum-version=${{ inputs.config-tag }}" >> $GITHUB_OUTPUT + if [ "${{ fromJson(steps.tests.outputs.json).stats.tests_fail }}" -gt "0" ]; then + echo "result=fail" >> $GITHUB_OUTPUT + else + echo "result=pass" >> $GITHUB_OUTPUT + fi + + failed-repro: + name: Failed Reproduction Notifier + needs: + - repro-ci + - check-checksum + if: failure() || needs.check-checksum.outputs.result == 'fail' + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - name: Setup Issue Variables + id: variables + run: | + config=${{ github.event.repository.name }} + echo "config=$config" >> $GITHUB_OUTPUT + echo "config-url=https://github.com/ACCESS-NRI/$config" >> $GITHUB_OUTPUT + # model is just the config repo name without the '-configs' bit, so split it below with bash param expansion + model=${config%-*} + echo "model=$model" >> $GITHUB_OUTPUT + echo "model-url=https://github.com/ACCESS-NRI/$model" >> $GITHUB_OUTPUT + echo "tag-url=https://github.com/ACCESS-NRI/$config/releases/tag/${{ needs.check-checksum.outputs.compared-checksum-version }}" >> GITHUB_OUTPUT + echo "run-url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_OUTPUT + + cat $GITHUB_OUTPUT + + - name: Create issue + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + BODY: | + There was a failure of a monthly reproducibility check on `${{ github.repository }}`. + Logs, checksums and other artifacts can be found at the Failed Run Log link below. + + Model: `${{ steps.variables.outputs.model }}`, found here: ${{ steps.variables.outputs.model-url }} + Config Repo: `${{ steps.variables.outputs.config }}`, found here: ${{ steps.variables.outputs.config-url }} + Config Tag Tested for Reproducibility: `${{ needs.check-checksum.outputs.compared-checksum-version }}`, found here: ${{ steps.variables.outputs.tag-url }} + Failed Run Log: ${{ steps.variables.outputs.run-url }} + Experiment Location (Gadi): `${{ needs.repro-ci.outputs.experiment-location }}` + + Tagging @ACCESS-NRI/model-release + run: | + gh issue create \ + --title 'Scheduled Repro Check Failed for Config `${{ needs.check-checksum.outputs.compared-checksum-version }}`' \ + --label "type:repro-fail,priority:blocker" \ + --body '${{ env.BODY }}' diff --git a/.github/workflows/config-generate-checksums.yml b/.github/workflows/generate-checksums.yml similarity index 96% rename from .github/workflows/config-generate-checksums.yml rename to .github/workflows/generate-checksums.yml index 3c84731..05ca5fc 100644 --- a/.github/workflows/config-generate-checksums.yml +++ b/.github/workflows/generate-checksums.yml @@ -2,10 +2,6 @@ name: Initial Checksums on: workflow_call: inputs: - model-name: - type: string - required: true - description: Name of the model that is having it's checksums generated. config-branch-name: type: string required: true @@ -50,7 +46,7 @@ jobs: runs-on: ubuntu-latest environment: ${{ inputs.environment-name }} env: - EXPERIMENT_LOCATION: ${{ vars.EXPERIMENTS_LOCATION }}/${{ inputs.model-name }}/${{ inputs.config-branch-name }} + EXPERIMENT_LOCATION: ${{ vars.EXPERIMENTS_LOCATION }}/${{ github.event.repository.name }}/${{ inputs.config-branch-name }} outputs: artifact-name: ${{ steps.artifact.outputs.name }} checksum-location: ${{ steps.run.outputs.checksum-location }} @@ -122,7 +118,7 @@ jobs: - name: Generate Output Artifact Name id: artifact - run: echo "name=${{ inputs.model-name }}-${{ inputs.config-branch-name }}" >> $GITHUB_OUTPUT + run: echo "name=${{ github.event.repository.name }}-${{ inputs.config-branch-name }}" >> $GITHUB_OUTPUT - name: Upload Output uses: actions/upload-artifact@v4 diff --git a/.github/workflows/config-pr-checks.yml b/.github/workflows/test-repro.yml similarity index 93% rename from .github/workflows/config-pr-checks.yml rename to .github/workflows/test-repro.yml index 42f4bdb..0282109 100644 --- a/.github/workflows/config-pr-checks.yml +++ b/.github/workflows/test-repro.yml @@ -2,10 +2,6 @@ name: Repro Checks on: workflow_call: inputs: - model-name: - type: string - required: true - description: The name of the model to check for reproducibility config-tag: type: string required: true @@ -46,7 +42,7 @@ jobs: artifact-name: ${{ steps.artifact.outputs.name }} experiment-location: ${{ steps.run.outputs.experiment-location }} env: - EXPERIMENT_LOCATION: ${{ vars.EXPERIMENTS_LOCATION }}/${{ inputs.model-name }}/${{ inputs.config-tag }} + EXPERIMENT_LOCATION: ${{ vars.EXPERIMENTS_LOCATION }}/${{ github.event.repository.name }}/${{ inputs.config-tag }} steps: - name: Setup SSH id: ssh @@ -94,14 +90,14 @@ jobs: model-config-tests -s -m "${{ inputs.test-markers }}" \ --output-path ${{ env.EXPERIMENT_LOCATION }} \ --junitxml=${{ env.EXPERIMENT_LOCATION }}/checksum/test_report.xml - + # Deactivate and remove the test virtual environment deactivate rm -rf ${{ env.TEST_VENV_LOCATION }} # We want the exit code post-`pytest` to be 0 so the overall `ssh` call succeeeds # after a potential `pytest` error. - exit 0 + exit 0 EOT echo "experiment-location=${{ env.EXPERIMENT_LOCATION }}" >> $GITHUB_OUTPUT @@ -113,7 +109,7 @@ jobs: - name: Generate Test Output Artifact Name id: artifact - run: echo "name=${{ inputs.model-name }}-${{ inputs.config-tag }}" >> $GITHUB_OUTPUT + run: echo "name=${{ github.event.repository.name }}-${{ inputs.config-tag }}" >> $GITHUB_OUTPUT - name: Upload Test Output uses: actions/upload-artifact@v3 diff --git a/README.md b/README.md index f634bc9..5179aa3 100644 --- a/README.md +++ b/README.md @@ -1,48 +1,56 @@ -## Model Configuration Pytests +# Model Configuration Pytests and CI -These pytests are used as part CI checks for model configurations. +This repository houses pytests that are used as part CI checks for model configurations, as well as reusable components of model configuration CI. -The checksum pytests are used for reproducibility CI checks in the [ACCESS-NRI/reproducibility](https://github.com/ACCESS-NRI/reproducibility) repository. The quick configuration tests are used in -[ACCESS-NRI/access-om2-configs](https://github.com/ACCESS-NRI/access-om2-configs). +The checksum pytests are used for reproducibility CI checks in this repository. The quick configuration tests are used in any repository that calls `config-pr-1-ci.yml` or is templated by [`ACCESS-NRI/model-configs-template](https://github.com/ACCESS-NRI/model-configs-template). For example, [ACCESS-NRI/access-om2-configs](https://github.com/ACCESS-NRI/access-om2-configs). -Code from these tests is adapted from COSIMAS's ACCESS-OM2's [ +Code from these pytests is adapted from COSIMAS's ACCESS-OM2's [ bit reproducibility tests](https://github.com/COSIMA/access-om2/blob/master/test/test_bit_reproducibility.py). -### How to run tests manually (from a local install of model-config-tests) +## Pytests -1. First clone the pytest code into a separate directory. -```sh -git clone https://github.com/ACCESS-NRI/model-config-tests/ model-config-tests -``` +### How to run pytests manually on NCI -2. To create a local install of `model-config-tests` -```sh -cd model-config-tests -pip install --user -e . -model-config-tests --help # model-config-tests runs the pytest command for tests in the package -``` +1. Create and activate a python virtual environment for installing and running tests + ```sh + module load python3/3.11.0 + python3 -m venv + source /bin/activate + ``` -2. Checkout an experiment (in this case it is using an ACCESS-OM2 config) -```sh -git clone https://github.com/ACCESS-NRI/access-om2-configs/ -cd -git checkout -``` +2. Either pip install a released version of `model-config-tests`, + ```sh + pip install model-config-tests==0.0.1 + ``` + Or to install `model-config-tests` in "editable" mode, first clone the repository, and then run pip install from the repository. This means any changes to the code are reflected in the installed package. + ```sh + git clone https://github.com/ACCESS-NRI/model-config-tests/ + pip install -e + ``` -3. Setup payu -```sh -module use /g/data/vk83/modules -module load payu/1.1.3 -``` +3. Checkout an experiment (in this case it is using an ACCESS-OM2 config) + + ```sh + git clone https://github.com/ACCESS-NRI/access-om2-configs/ + cd + git checkout + ``` 4. Run the pytests -```sh -model-config-tests -``` + ```sh + model-config-tests + ``` + +5. Once done with testing, deactivate the virtual environment, and if the environment is no longer needed, remove the environment + ```sh + deactivate + rm -rf # Deletes the test environment + ``` ### Pytest Options The output directory for pytests defaults to `$TMPDIR/test-model-repro` and contains the following subdirectories: + - `control` - contains copies of the model configuration used for each experiment run in the tests. - `lab` - contains `payu` model output directories containing `work` and `archive` sub-directories. @@ -54,7 +62,7 @@ To specify a different folder for pytest outputs, use `--output-path` command fl model-config-tests --output-path /some/other/path/for/output ``` -By default, the control directory, e.g. the model configuration to test, is the current working directory. This can be set similarly to above by using the +By default, the control directory, e.g. the model configuration to test, is the current working directory. This can be set similarly to above by using the `--control-path` command flag. The path containing the checksum file to check against can also be set using @@ -74,3 +82,46 @@ e.g.: ```sh model-config-tests -m "config or access_om2" ``` + +## CI/CD + +The `.github` directory contains many different workflows and actions. This section describes how they are used. + +### CI/CD For This Repository + +`CI.yml` and `CD.yml` are used to test, package and upload the `model-config-tests` package that is used by `model-configs`-style repositories across the ACCESS-NRI. These are the only workflows that run on this repository. The others are reusable workflows called by `model-configs`-style repositories, among others. + +### Reusable CI + +The `config-*.yml`, `generate-checksums.yml` and `test-repro.yml` workflows are called by `model-configs`-style repositories to test model configurations. They are stored in this repository to allow a central place to update generic CI used by all model configuration repositories. + +Below is information on the use of these workflows. + +#### `config-pr-*.yml` Pipeline + +The `config-pr-*` Pipeline is a series of workflows that govern the testing, ChatOps and merging procedures of pull requests for model configuration repositories, such as [`ACCESS-NRI/access-om2-configs`](https://github.com/ACCESS-NRI/access-om2-configs). + +Essentially, these files work on two types of configuration branch pull requests in the model configuration repository. More information on the terminology used in model configuration repositories can be found in the `README.md` of the `ACCESS-NRI/model-configs-template` repository. The types of pull requests are: + +- Pull requests into `dev-*`: Allows quick checks of configuration metadata and common mistakes in configurations during PRs into the `dev-*` configuration branches. +- Pull requests from `dev-*` into `release-*`: Allows both quick checks, as well as a longer, more comprehensive check on the reproducibility of the changes being brought into the `release-*` configuration branch, compared to the previous `release-*` commit. It also acts on 'comment commands' run during the pull request, like `!bump` for updating the version of the configuration ([see the 'Config Tags' section](https://github.com/ACCESS-NRI/model-configs-template/blob/main/README.md) in the `ACCESS-NRI/model-configs-template` repository for more). It is also responsible for the creation of the final config tag and release, once merged. + +#### `config-schedule-*.yml` Pipeline + +The `config-schedule-*` Pipeline is a series of workflows used to check the reproducibility of certain config tags against themselves, every month. This is used as a kind of canary to make sure that we continue to get the same results on the same deployment targets. + +#### `generate-checksums` Reusable Workflow + +This workflow is used to easily generate the checksums used in the reproducibility checks, for a specific branch of a model configuration repository, if they don't already exist. This is most often used for the initial commit of a checksum to the `release-*` configuration branch. + +#### `test-repro` Reusable Workflow + +This workflow is used to test the reproducibility of a given model repository against historical checksums, and can be used as a standalone workflow. + +Using it has some requirements outside of just filling in the inputs: One must have a valid GitHub Environment (specified by the `environment-name` input) in the calling repository, that has the following `secrets` and `vars` defined: + +- `secrets.SSH_HOST` - hostname for the deployment target +- `secrets.SSH_HOST_DATA` - hostname for the data mover on the deployment target (if it exists) +- `secrets.SSH_KEY` - private key for access to the deployment target +- `secrets.SSH_USER` - username for access to the deployment target +- `vars.EXPERIMENTS_LOCATION` - directory on the deployment target that will contain all the experiments used during testing of reproducibility across multiple runs of this workflow (ex. `/scratch/some/directory/experiments`)