tests.yml

name: continuous-integration
concurrency:
  group: ${{ github.workflow }}-${{ github.event.number }}-${{ github.event.ref }}
  cancel-in-progress: true

# README
# ======
#
# All the jobs are defined with `matrix` for OS and Python version, even if we
# only run them on one combination of OS/Python. The reason for this is you get
# a nice side-effect that the OS and Python version of the job are listed in
# parentheses next to the job name in the Actions UI.

# This prevents workflows from being run twice on PRs
# ref: https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662
on:
  push:
    branches:
      - main
  pull_request:
  workflow_call:

jobs:
  lint:
    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.12"]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"
          cache-dependency-path: "pyproject.toml"
      - uses: pre-commit/action@v3.0.1

  # run our test suite on various combinations of OS / Python / Sphinx version
  run-pytest:
    needs: [lint]
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.9", "3.10", "3.11", "3.12"]
        sphinx-version: ["rls"]
        include:
          # macos test
          - os: macos-latest
            python-version: "3.12"
            sphinx-version: "rls"
          # windows test
          - os: windows-latest
            python-version: "3.12"
            sphinx-version: "rls"
          # old Sphinx test
          - os: ubuntu-latest
            python-version: "3.9"
            sphinx-version: "old"
          # dev Sphinx test
          - os: ubuntu-latest
            python-version: "3.12"
            sphinx-version: "dev"
    # needed to cache the browsers for the accessibility tests
    env:
      PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/pw-browsers
      SPHINX_VERSION: ${{ matrix.sphinx-version }}
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"
          cache-dependency-path: "pyproject.toml"
      # waiting for https://github.com/nikeee/setup-pandoc/pull/8
      # using 12rambau fork until then
      - uses: 12rambau/setup-pandoc@test
      - name: Install dependencies
        # if Sphinx version not specified in matrix, the constraints in the
        # pyproject.toml file determine Sphinx version
        shell: bash
        # setting shell to BASH and using PYTHONUTF8 env var makes the editable
        # install work on Windows even though there are emoji in our README
        run: ./tools/github_actions_install.sh test
      - name: Show installed versions
        run: python -m pip list
      - name: Compile MO files
        run: |
          pip install nox
          nox -s compile
      - name: Run tests
        run: pytest -m "not a11y" --color=yes --cov --cov-report=xml
      - name: Upload to Codecov
        if: matrix.python-version == '3.12' && matrix.os == 'ubuntu-latest' && matrix.sphinx-version == 'dev'
        uses: codecov/codecov-action@v4
        with:
          fail_ci_if_error: true
          verbose: true
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

      # note I am setting this on top of the Python cache as I could not find
      # how to set the hash key on the python one
      - name: Set up browser cache (for accessibility tests)
        uses: actions/cache@v4
        with:
          path: |
            ${{ github.workspace }}/pw-browsers
          key: ${{ runner.os }}-pw-${{ hashFiles('pyproject.toml') }}
          restore-keys: |
            ${{ runner.os }}-pw-

      - name: Run accessibility tests with playwright
        if: matrix.python-version == '3.12' && matrix.os == 'ubuntu-latest'
        run: |
          nox -s a11y
        continue-on-error: true

  # Build our site on the 3 major OSes and check for Sphinx warnings
  build-site:
    needs: [lint]
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ["3.12"]
        sphinx-version: [""]
        include:
          - os: ubuntu-latest
            python-version: "3.9"
            sphinx-version: "old"
          # crash with myst parser
          # will be restored when myst will be dropped from our documentation
          # - os: ubuntu-latest
          #   python-version: "3.12"
          #   sphinx-version: "dev"
    env:
      SPHINX_VERSION: ${{ matrix.sphinx-version }}
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      # waiting for https://github.com/nikeee/setup-pandoc/pull/8
      # using 12rambau fork until then
      - uses: 12rambau/setup-pandoc@test
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"
          cache-dependency-path: "pyproject.toml"
      - name: Install dependencies
        shell: bash
        run: ./tools/github_actions_install.sh doc
      - name: Show installed versions
        run: python -m pip list
      - name: Build docs
        run: sphinx-build -b html docs/ docs/_build/html --keep-going -w warnings.txt
      - name: Check for unexpected Sphinx warnings
        run: python tests/utils/check_warnings.py

  # Run local Lighthouse audit against built site
  audit:
    needs: [build-site]
    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.12"]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"
          cache-dependency-path: "pyproject.toml"
      - name: Install dependencies
        run: ./tools/github_actions_install.sh doc
        shell: bash
      - name: Show installed versions
        run: python -m pip list
      # We want to run the audit on a simplified documentation build so that
      # the audit results aren't affected by non-theme things like extensions.
      # Here we copy over just the kitchen sink into an empty docs site with
      # only the theme loaded so extensions don't play a role in audit scores.
      - name: Copy kitchen sink to a tiny site and build it
        run: |
          mkdir audit/
          mkdir audit/site
          cp -r docs/examples/kitchen-sink audit/site/kitchen-sink
          printf "Test\n====\n\n.. toctree::\n\n   kitchen-sink/index\n" > audit/site/index.rst
          echo 'html_theme = "pydata_sphinx_theme"' > audit/site/conf.py
          echo '.. toctree::\n   :glob:\n\n   *' >> audit/site/index.rst
          sphinx-build audit/site audit/_build
      # The lighthouse audit runs directly on the HTML files, no serving needed
      - name: Audit with Lighthouse
        uses: treosh/lighthouse-ci-action@v11
        with:
          configPath: ".github/workflows/lighthouserc.json"
          temporaryPublicStorage: true
          uploadArtifacts: true
          runs: 3 # Multiple runs to reduce variance

  # Generate a profile of the code and upload as an artifact
  profile:
    needs: [build-site, run-pytest]
    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.12"]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"
          cache-dependency-path: "pyproject.toml"
      - name: Install dependencies
        run: ./tools/github_actions_install.sh test nox
        shell: bash
      - name: Show installed versions
        run: python -m pip list
      - name: Generate a profile
        run: nox -s profile
        continue-on-error: true
      - uses: actions/upload-artifact@v4
        with:
          name: profile-results
          path: profile.svg