tests_tox.yml

name: continuous-integration-tox

# Concurrency group that uses the workflow name and PR number if available
# or commit SHA as a fallback. If a new build is triggered under that
# concurrency group while a previous build is running it will be canceled.
# Repeated pushes to a PR will cancel all previous builds, while multiple
# merges to main will not cancel.
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

env:
  FORCE_COLOR: "1" # Make tools pretty
  DEFAULT_PYTHON_VERSION: "3.12" # keep in sync with tox.ini
  PIP_DISABLE_PIP_VERSION_CHECK: "1" # Don't check for pip updates

permissions: {}

on:
  push:
    branches:
      - main
  pull_request:
  workflow_call:
  # allow manual triggering of the workflow, while debugging
  workflow_dispatch:

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - name: "Checkout repository 🛎"
        uses: actions/checkout@v4
      - name: "Setup CI environment 🛠"
        uses: ./.github/actions/set-dev-env
        with:
          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
      - name: "Run lint checks 🧹"
        run: python -Im tox run -e lint

  # Run our test suite on various combinations of OS & Python versions
  run-pytest:
    needs: lint
    strategy:
      fail-fast: true
      matrix:
        os: ["ubuntu-latest", "macos-latest", "macos-14", "windows-latest"]
        python-version: ["3.9", "3.10", "3.11", "3.12"]
        sphinx-version: [""]
        include:
          # oldest Python version with the oldest Sphinx version
          - os: ubuntu-latest
            python-version: "3.9"
            sphinx-version: "61"
            # newest Python version with the newest Sphinx version
          - os: ubuntu-latest
            python-version: "3.12"
            # Sphinx HEAD
            sphinx-version: "dev"
        exclude:
          # Python 3.9 is not supported on macOS 14 - https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json
          - os: macos-14
            python-version: "3.9"
    runs-on: ${{ matrix.os }}
    steps:
      - name: "Checkout repository 🛎"
        uses: actions/checkout@v4
      - name: "Setup CI environment 🛠"
        uses: ./.github/actions/set-dev-env
        with:
          python-version: ${{ matrix.python-version }}
          pandoc: "True"
      - name: "Run tests ✅"
        shell: bash
        run: |
          # this will compile the assets then run the tests
          # check if there is a specific Sphinx version to test with
          # example substitution: tox run -e compile,py39-sphinx61-tests
          if [ -n "${{matrix.sphinx-version}}" ]; then
            python -Im tox run -e compile,py$(echo ${{ matrix.python-version }} | tr -d .)-sphinx${{matrix.sphinx-version}}-tests
          # if not we use the default version
          # example substitution: tox run -e compile,py39-tests
          else
            python -Im tox run -e compile,py$(echo ${{ matrix.python-version }} | tr -d .)-tests
          fi
      - name: "Upload coverage data to GH artifacts 📤"
        if: matrix.python-version == '3.12' && matrix.os == 'ubuntu-latest' && matrix.sphinx-version == 'dev'
        uses: actions/upload-artifact@v4
        with:
          name: coverage-data-${{ matrix.python-version }}
          path: .coverage
          if-no-files-found: ignore

  # Only run accessibility tests on the latest Python version (3.12) and Ubuntu
  a11y-tests:
    name: "a11y-tests (ubuntu-latest, 3.12)"
    needs: lint
    runs-on: ubuntu-latest
    steps:
      - name: "Checkout repository 🛎"
        uses: actions/checkout@v4
      - name: "Setup CI environment 🛠"
        uses: ./.github/actions/set-dev-env
        with:
          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
      - name: "Run accessibility tests with playwright 🎭"
        # build PST, build docs, then run a11y-tests
        run: python -Im tox run -e py312-docs,a11y-tests
        continue-on-error: true

  # Build our docs (PST) on major OSes and check for warnings
  build-site:
    name: "build PST docs"
    needs: lint
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ["3.12"]
        include:
          # oldest Python version with the oldest Sphinx version
          - os: ubuntu-latest
            python-version: "3.9"
            sphinx-version: "61"
    runs-on: ${{ matrix.os }}
    steps:
      - name: "Checkout repository 🛎"
        uses: actions/checkout@v4
      - name: "Setup CI environment 🛠"
        uses: ./.github/actions/set-dev-env
        with:
          python-version: ${{ matrix.python-version }}
          pandoc: "True"
      - name: "Build docs and check for warnings 📖"
        shell: bash
        run: |
          # check if there is a specific Sphinx version to build with
          # example substitution: tox run -e docs-py39-sphinx61-docs
          if [ -n "${{matrix.sphinx-version}}" ]; then
            python -Im tox run -e docs-py$(echo ${{ matrix.python-version }} | tr -d .)-sphinx${{matrix.sphinx-version}}-docs
          # build with the default Sphinx version 
          # example substitution: tox run -e docs-py312-docs
          else
            python -Im tox run -e docs-py$(echo ${{ matrix.python-version }} | tr -d .)-docs
          fi

  # Run Lighthouse audits on the built site (kitchen-sink only)
  lighthouse-audit:
    needs: build-site
    runs-on: ubuntu-latest
    env:
      DOCS_DIR: "audit"
    steps:
      - name: "Checkout repository 🛎"
        uses: actions/checkout@v4
      - name: "Setup CI environment 🛠"
        uses: ./.github/actions/set-dev-env
        with:
          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
      - name: "Copy kitchen sink to a tiny site"
        run: |
          mkdir -p ${{ env.DOCS_DIR }}/site
          cp -r docs/examples/kitchen-sink ${{ env.DOCS_DIR }}/site/kitchen-sink
          printf "Test\n====\n\n.. toctree::\n\n   kitchen-sink/index\n" > ${{ env.DOCS_DIR }}/site/index.rst
          echo 'html_theme = "pydata_sphinx_theme"' > ${{ env.DOCS_DIR }}/site/conf.py
          echo '.. toctree::\n   :glob:\n\n   *' >> ${{ env.DOCS_DIR }}/site/index.rst

          # build docs without checking for warnings
          python -Im tox run -e docs-no-checks

      - name: "Audit with Lighthouse 🔦"
        uses: treosh/lighthouse-ci-action@v11
        with:
          configPath: ".github/workflows/lighthouserc.json"
          temporaryPublicStorage: true
          uploadArtifacts: true
          runs: 3 # Multiple runs to reduce variance

  coverage:
    name: "check coverage"
    needs: run-pytest
    runs-on: ubuntu-latest
    steps:
      - name: "Checkout repository 🛎"
        uses: actions/checkout@v4
      - name: "Setup CI environment 🛠"
        uses: ./.github/actions/set-dev-env
        with:
          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
      - run: python -Im pip install --upgrade coverage[toml]
      - name: "Download coverage data 📥"
        uses: actions/download-artifact@v4
        with:
          pattern: coverage-data-*
          merge-multiple: true
      - name: "Get coverage data & fail if it's <80%"
        run: |
          # if we decide to check cov across versions and combine
          # python -Im coverage combine
          python -Im coverage html --skip-covered --skip-empty

          # report and write to summary.
          python -Im coverage report --format=markdown >> $GITHUB_STEP_SUMMARY

          # report again and fail if under 80%.
          python -Im coverage report --fail-under=80
      - name: "Upload HTML report if check failed 📤"
        uses: actions/upload-artifact@v4
        with:
          name: html-report
          path: htmlcov
        if: ${{ failure() }}

  profile:
    needs: [build-site, run-pytest]
    runs-on: ubuntu-latest
    steps:
      - name: "Checkout repository 🛎"
        uses: actions/checkout@v4
      - name: "Setup CI environment 🛠"
        uses: ./.github/actions/set-dev-env
        with:
          # 3.12 is not supported by py-spy yet
          python-version: "3.11"
      - name: "Run profiling with py-spy 🕵️‍♂️"
        # profiling needs to be run as sudo
        run: sudo python -Im tox run -e py311-profile-docs -- -o docbuild_profile.svg
        continue-on-error: true
      - name: "Upload profiling data to GH artifacts 📤"
        uses: actions/upload-artifact@v4
        with:
          name: profile-results
          path: docbuild_profile.svg
          if-no-files-found: ignore