build-pipeline.yml

# This is the main build pipeline that verifies and publishes the software
name: Build
# Controls when the workflow will run
on:
  # Triggers the workflow on push events
  push:
    branches: [ develop, release/**, main, feature/** ]

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

env:
  POETRY_VERSION: "1.3.1"
  PYTHON_VERSION: "3.10"
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}

jobs:
  # First job in the workflow installs and verifies the software
  build:
    name: Build, Test, Verify, Publish
    # The type of runner that the job will run on
    runs-on: ubuntu-latest
    steps:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
      - name: Install Poetry
        uses: abatilo/actions-poetry@v2
        with:
          poetry-version: ${{ env.POETRY_VERSION }}
      - name: Get version
        id: get-version
        run: |
          echo "current_version=$(poetry version | awk '{print $2}')" >> $GITHUB_OUTPUT
          echo "pyproject_name=$(poetry version | awk '{print $1}')" >> $GITHUB_ENV
      - name: Bump pre-alpha version
        # If triggered by push to a feature branch
        if: ${{ startsWith(github.ref, 'refs/heads/feature/') }}
        run: |
          new_ver="${{ steps.get-version.outputs.current_version }}+$(git rev-parse --short ${GITHUB_SHA})"
          poetry version $new_ver
          echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV
      - name: Bump alpha version
        # If triggered by push to the develop branch
        if: ${{ github.ref == 'refs/heads/develop' }}
        run: |
          poetry version prerelease
          echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV
          echo "venue=sit" >> $GITHUB_ENV
      - name: Bump rc version
        # If triggered by push to a release branch
        if: ${{ startsWith(github.ref, 'refs/heads/release/') }}
        env:
          # True if the version already has a 'rc' pre-release identifier
          BUMP_RC: ${{ contains(steps.get-version.outputs.current_version, 'rc') }}
        run: |
          if [ "$BUMP_RC" = true ]; then
            poetry version prerelease
          else
            poetry version ${GITHUB_REF#refs/heads/release/}rc1
          fi
          echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV
          echo "venue=uat" >> $GITHUB_ENV
      - name: Release version
        # If triggered by push to the main branch
        if: ${{ startsWith(github.ref, 'refs/heads/main') }}
        env:
          CURRENT_VERSION: ${{ steps.get-version.outputs.current_version }}
        # Remove rc* from end of version string
        # The ${string%%substring} syntax below deletes the longest match of $substring from back of $string.
        run: |
          poetry version ${CURRENT_VERSION%%rc*}
          echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV
          echo "venue=ops" >> $GITHUB_ENV
      - name: Install concise
        run: poetry install
      - name: Lint
        run: |
          poetry run pylint podaac
          poetry run flake8 podaac
      - name: Test and coverage
        run: |
          poetry run pytest --junitxml=build/reports/pytest.xml --cov=podaac/ --cov-report=xml:build/reports/coverage.xml -m "not aws and not integration" tests/
      - name: SonarCloud Scan
        uses: sonarsource/sonarcloud-github-action@master
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
        with:
          args: >
            -Dsonar.organization=${{ github.repository_owner }}
            -Dsonar.projectKey=${{ github.repository_owner }}_concise
            -Dsonar.python.coverage.reportPaths=build/reports/coverage.xml
            -Dsonar.sources=podaac/
            -Dsonar.tests=tests/
            -Dsonar.projectName=podaac-concise
            -Dsonar.projectVersion=${{ env.software_version }}
            -Dsonar.python.version=${{ env.PYTHON_VERSION }}
      - name: Run Snyk as a blocking step
        uses: snyk/actions/python-3.10@master
        env:
            SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
        with:
          command: test
          args: >
              --org=${{ secrets.SNYK_ORG_ID }}
              --project-name=${{ github.repository }}
              --severity-threshold=high
              --fail-on=all
      - name: Run Snyk on Python
        uses: snyk/actions/python-3.10@master
        env:
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
        with:
          command: monitor
          args: >
            --org=${{ secrets.SNYK_ORG_ID }}
            --project-name=${{ github.repository }}
      - name: Commit Version Bump
        # If building develop, a release branch, or main then we commit the version bump back to the repo
        if: |
          github.ref == 'refs/heads/develop' ||
          github.ref == 'refs/heads/main'    ||
          startsWith(github.ref, 'refs/heads/release')
        run: |
          git config --global user.name 'concise bot'
          git config --global user.email 'concise@noreply.github.com'
          git commit -am "/version ${{ env.software_version }}"
          git push
      - name: Push Tag
        if: |
          github.ref == 'refs/heads/develop' ||
          github.ref == 'refs/heads/main'    ||
          startsWith(github.ref, 'refs/heads/release')
        run: |
          git config user.name "${GITHUB_ACTOR}"
          git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
          git tag -a "${{ env.software_version }}" -m "Version ${{ env.software_version }}"
          git push origin "${{ env.software_version }}"
      - name: Publish UMM-S with new version
        uses: podaac/cmr-umm-updater@develop
        if: |
          github.ref == 'refs/heads/main'    ||
          startsWith(github.ref, 'refs/heads/release')
        with:
          umm-json: 'cmr/concise_cmr_umm_s.json'
          provider: 'POCLOUD'
          env: ${{ env.venue }}
          version: ${{ env.software_version }}
          timeout: 60
          disable_removal: 'true'
          umm_type: 'umm-s'
          use_associations: 'false'
          umm_version: '1.5.2'
        env:
          LAUNCHPAD_TOKEN_SIT: ${{secrets.LAUNCHPAD_TOKEN_SIT}}
          LAUNCHPAD_TOKEN_UAT: ${{secrets.LAUNCHPAD_TOKEN_UAT}}
          LAUNCHPAD_TOKEN_OPS: ${{secrets.LAUNCHPAD_TOKEN_OPS}}
      - name: Publish L2ss Concise Chain UMM-S with new version
        uses: podaac/cmr-umm-updater@develop
        if: |
          github.ref == 'refs/heads/main'    ||
          startsWith(github.ref, 'refs/heads/release')
        with:
          umm-json: 'cmr/l2ss_concise_chain_cmr_umm_s.json'
          provider: 'POCLOUD'
          env: ${{ env.venue }}
          version: ${{ env.software_version }}
          timeout: 60
          disable_removal: 'true'
          umm_type: 'umm-s'
          use_associations: 'false'
          umm_version: '1.5.2'
        env:
          LAUNCHPAD_TOKEN_SIT: ${{secrets.LAUNCHPAD_TOKEN_SIT}}
          LAUNCHPAD_TOKEN_UAT: ${{secrets.LAUNCHPAD_TOKEN_UAT}}
          LAUNCHPAD_TOKEN_OPS: ${{secrets.LAUNCHPAD_TOKEN_OPS}}
      - name: Build Docs
        run: |
          poetry run sphinx-build -b html ./docs docs/_build/
      - name: Publish Docs
        uses: JamesIves/github-pages-deploy-action@v4
        with:
          branch: gh-pages # The branch the action should deploy to.
          folder: docs/_build/ # The folder the action should deploy.
          target-folder: ${{ env.software_version }}
      - name: Build Python Artifact
        run: |
          poetry build
      - uses: actions/upload-artifact@v4
        with:
          name: python-artifact
          path: dist/*
      - name: Publish to test.pypi.org
        if: |
          github.ref == 'refs/heads/develop' ||
          startsWith(github.ref, 'refs/heads/release')
        env:
          POETRY_PYPI_TOKEN_TESTPYPI: ${{secrets.POETRY_PYPI_TOKEN_TESTPYPI}}
        run: |
          poetry config repositories.testpypi https://test.pypi.org/legacy/
          poetry publish -r testpypi
      - name: Publish to pypi.org
        if: ${{ github.ref == 'refs/heads/main' }}
        env:
          POETRY_PYPI_TOKEN_PYPI: ${{secrets.POETRY_PYPI_TOKEN_PYPI}}
        run: |
          poetry publish
      - name: Log in to the Container registry
        if: ${{ !startsWith(github.ref, 'refs/heads/feature') }}
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Extract metadata (tags, labels) for Docker
        if: ${{ !startsWith(github.ref, 'refs/heads/feature') }}
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
          tags: |
            type=raw,pattern={{version}},value=${{ env.software_version }}
            type=raw,value=${{ env.venue }}
      - name: Wait for package
        if: ${{ !startsWith(github.ref, 'refs/heads/feature') }}
        run: |
          pip install tenacity
          ${GITHUB_WORKSPACE}/.github/workflows/wait-for-pypi.py ${{env.pyproject_name}}[harmony]==${{ env.software_version }}
      - name: Build and push Docker image
        if: ${{ !startsWith(github.ref, 'refs/heads/feature') }}
        id: docker-push
        uses: docker/build-push-action@v5
        with:
          context: .
          file: docker/Dockerfile
          build-args: |
            SOURCE=${{env.pyproject_name}}[harmony]==${{ env.software_version }}
          push: true
          pull: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          
      - name: Deploy Harmony
        env:
            ENV: ${{ env.venue }}
            CMR_USER: ${{ secrets.CMR_USER }}
            CMR_PASS: ${{ secrets.CMR_PASS }}
        if: |
          github.ref == 'refs/heads/main'    ||
          startsWith(github.ref, 'refs/heads/release')
        working-directory: deployment
        run: 
          poetry run python harmony_deploy.py --tag ${{ env.software_version }} 

# As of 2023/01/23 these steps below for scanning the Docker image with Snyk are failing. I've tried both the official Snyk
# action https://github.com/snyk/actions/tree/master/docker and this method below of manually calling the CLI.
# The error when using the official Snyk action is
#   "Could not detect package manager for file: ./docker/Dockerfile"
# The error when trying to run it manually is
#   "The image does not exist for the current platform"
# Commenting these steps out for now so they don't fail the build. Will try revisiting again in the future when Snyk is updated.
# 
#       - uses: snyk/actions/setup@master
#         if: |
#           steps.docker-push.conclusion == 'success'
#       - name: Run Snyk on Docker Image
#         if: |
#           steps.docker-push.conclusion == 'success'
#         # Snyk can be used to break the build when it detects vulnerabilities.
#         # In this case we want to upload the issues to GitHub Code Scanning
#         continue-on-error: true
#         run: |
#           snyk test --severity-threshold=high --file=./docker/Dockerfile --sarif-file-output=docker.sarif --docker ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.software_version }}
#         env:
#           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
#       - name: Upload result to GitHub Code Scanning
#         if: |
#           steps.docker-push.conclusion == 'success'
#         uses: github/codeql-action/upload-sarif@v2
#         with:
#           sarif_file: ./