DEV - Refactor CI and environment setup

.devcontainer/devcontainer.json

@@ -3,7 +3,7 @@
   "name": "Python 3",
   "image": "mcr.microsoft.com/devcontainers/python:1-3.10-bullseye",
   "features": {
-    "ghcr.io/devcontainers-contrib/features/nox:2": {},
+    "ghcr.io/devcontainers-contrib/features/tox:2": {},
     "ghcr.io/devcontainers-contrib/features/pre-commit:2": {},
     "ghcr.io/rocker-org/devcontainer-features/pandoc:1": {},
     "ghcr.io/devcontainers/features/node:1": {}

.github/actions/set-dev-env/action.yml

@@ -0,0 +1,32 @@
+name: Setup PST CI environment
+description: Create a PST dev environment
+
+inputs:
+  python-version:
+    description: Default Python version to use if none is specified
+    required: false
+    default: "3.12"
+  pandoc:
+    description: Whether this should install pandoc or not
+    required: false
+    default: "False"
+
+runs:
+  using: composite
+  steps:
+    - name: "Setup Python 🐍"
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
+        cache: "pip"
+        cache-dependency-path: "pyproject.toml"
+        allow-prereleases: true
+
+    - run: python -Im pip install tox-uv
+      shell: bash
+
+    # waiting for https://github.com/nikeee/setup-pandoc/pull/8
+    # using 12rambau fork until then
+    - name: "Install pandoc 📝"
+      uses: 12rambau/setup-pandoc@test
+      if: ${{ inputs.pandoc }} == "True"

.github/prerelease-template.md

@@ -4,6 +4,7 @@ assignees: choldgraf
 labels: bug, enhancement
 ---
 
-A prerelease of one of our dependencies failed. See the
+A prerelease of one of our dependencies failed.
+See the
 [action log](https://github.com/{{ env.GITHUB_ACTION_REPOSITORY }}/actions/runs/{{ env.GITHUB_RUN_ID }})
 for more details.

.github/workflows/CI.yml

@@ -0,0 +1,234 @@
+name: continuous-integration-tox
+
+# Concurrency group that uses the workflow name and PR number if available
+# or commit SHA as a fallback. If a new build is triggered under that
+# concurrency group while a previous build is running it will be canceled.
+# Repeated pushes to a PR will cancel all previous builds, while multiple
+# merges to main will not cancel.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
+env:
+  FORCE_COLOR: "1" # Make tools pretty
+  DEFAULT_PYTHON_VERSION: "3.12" # keep in sync with tox.ini
+  PIP_DISABLE_PIP_VERSION_CHECK: "1" # Don't check for pip updates
+
+permissions: {}
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_call:
+  # allow manual triggering of the workflow, while debugging
+  workflow_dispatch:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout repository 🛎"
+        uses: actions/checkout@v4
+      - name: "Setup CI environment 🛠"
+        uses: ./.github/actions/set-dev-env
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
+      - name: "Run lint checks 🧹"
+        run: python -Im tox run -e lint
+
+  # Run our test suite on various combinations of OS & Python versions
+  run-pytest:
+    needs: lint
+    strategy:
+      fail-fast: true
+      matrix:
+        os: ["ubuntu-latest", "macos-latest", "macos-14", "windows-latest"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        sphinx-version: [""]
+        include:
+          # oldest Python version with the oldest Sphinx version
+          - os: ubuntu-latest
+            python-version: "3.9"
+            sphinx-version: "61"
+            # newest Python version with the newest Sphinx version
+          - os: ubuntu-latest
+            python-version: "3.12"
+            # Sphinx HEAD
+            sphinx-version: "dev"
+        exclude:
+          # Python 3.9 is not supported on macOS 14 - https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json
+          - os: macos-14
+            python-version: "3.9"
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: "Checkout repository 🛎"
+        uses: actions/checkout@v4
+      - name: "Setup CI environment 🛠"
+        uses: ./.github/actions/set-dev-env
+        with:
+          python-version: ${{ matrix.python-version }}
+          pandoc: "True"
+      - name: "Run tests ✅"
+        shell: bash
+        run: |
+          # this will compile the assets then run the tests
+          # check if there is a specific Sphinx version to test with
+          # example substitution: tox run -e compile,py39-sphinx61-tests
+          if [ -n "${{matrix.sphinx-version}}" ]; then
+            python -Im tox run -e compile,py$(echo ${{ matrix.python-version }} | tr -d .)-sphinx${{matrix.sphinx-version}}-tests
+          # if not we use the default version
+          # example substitution: tox run -e compile,py39-tests
+          else
+            python -Im tox run -e compile,py$(echo ${{ matrix.python-version }} | tr -d .)-tests
+          fi
+      - name: "Upload coverage data to GH artifacts 📤"
+        if: matrix.python-version == '3.12' && matrix.os == 'ubuntu-latest' && matrix.sphinx-version == 'dev'
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-data-${{ matrix.python-version }}
+          path: .coverage
+          if-no-files-found: ignore
+
+  # Only run accessibility tests on the latest Python version (3.12) and Ubuntu
+  a11y-tests:
+    name: "a11y-tests (ubuntu-latest, 3.12)"
+    needs: lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout repository 🛎"
+        uses: actions/checkout@v4
+      - name: "Setup CI environment 🛠"
+        uses: ./.github/actions/set-dev-env
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
+      - name: "Run accessibility tests with playwright 🎭"
+        # build PST, build docs, then run a11y-tests
+        run: python -Im tox run -e py312-docs,a11y-tests
+        continue-on-error: true
+
+  # Build our docs (PST) on major OSes and check for warnings
+  build-site:
+    name: "build PST docs"
+    needs: lint
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.12"]
+        include:
+          # oldest Python version with the oldest Sphinx version
+          - os: ubuntu-latest
+            python-version: "3.9"
+            sphinx-version: "61"
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: "Checkout repository 🛎"
+        uses: actions/checkout@v4
+      - name: "Setup CI environment 🛠"
+        uses: ./.github/actions/set-dev-env
+        with:
+          python-version: ${{ matrix.python-version }}
+          pandoc: "True"
+      - name: "Build docs and check for warnings 📖"
+        shell: bash
+        run: |
+          # check if there is a specific Sphinx version to build with
+          # example substitution: tox run -e docs-py39-sphinx61-docs
+          if [ -n "${{matrix.sphinx-version}}" ]; then
+            python -Im tox run -e docs-py$(echo ${{ matrix.python-version }} | tr -d .)-sphinx${{matrix.sphinx-version}}-docs
+          # build with the default Sphinx version 
+          # example substitution: tox run -e docs-py312-docs
+          else
+            python -Im tox run -e docs-py$(echo ${{ matrix.python-version }} | tr -d .)-docs
+          fi
+
+  # Run Lighthouse audits on the built site (kitchen-sink only)
+  lighthouse-audit:
+    needs: build-site
+    runs-on: ubuntu-latest
+    env:
+      DOCS_DIR: "audit"
+    steps:
+      - name: "Checkout repository 🛎"
+        uses: actions/checkout@v4
+      - name: "Setup CI environment 🛠"
+        uses: ./.github/actions/set-dev-env
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
+      - name: "Copy kitchen sink to a tiny site"
+        run: |
+          mkdir -p ${{ env.DOCS_DIR }}/site
+          cp -r docs/examples/kitchen-sink ${{ env.DOCS_DIR }}/site/kitchen-sink
+          printf "Test\n====\n\n.. toctree::\n\n   kitchen-sink/index\n" > ${{ env.DOCS_DIR }}/site/index.rst
+          echo 'html_theme = "pydata_sphinx_theme"' > ${{ env.DOCS_DIR }}/site/conf.py
+          echo '.. toctree::\n   :glob:\n\n   *' >> ${{ env.DOCS_DIR }}/site/index.rst
+
+          # build docs without checking for warnings
+          python -Im tox run -e docs-no-checks
+
+      - name: "Audit with Lighthouse 🔦"
+        uses: treosh/lighthouse-ci-action@v11
+        with:
+          configPath: ".github/workflows/lighthouserc.json"
+          temporaryPublicStorage: true
+          uploadArtifacts: true
+          runs: 3 # Multiple runs to reduce variance
+
+  coverage:
+    name: "check coverage"
+    needs: run-pytest
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout repository 🛎"
+        uses: actions/checkout@v4
+      - name: "Setup CI environment 🛠"
+        uses: ./.github/actions/set-dev-env
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
+      - run: python -Im pip install --upgrade coverage[toml]
+      - name: "Download coverage data 📥"
+        uses: actions/download-artifact@v4
+        with:
+          pattern: coverage-data-*
+          merge-multiple: true
+      - name: "Get coverage data & fail if it's <80%"
+        run: |
+          # if we decide to check cov across versions and combine
+          # python -Im coverage combine
+          python -Im coverage html --skip-covered --skip-empty
+
+          # report and write to summary.
+          python -Im coverage report --format=markdown >> $GITHUB_STEP_SUMMARY
+
+          # report again and fail if under 80%.
+          python -Im coverage report --fail-under=80
+      - name: "Upload HTML report if check failed 📤"
+        uses: actions/upload-artifact@v4
+        with:
+          name: html-report
+          path: htmlcov
+        if: ${{ failure() }}
+
+  profiling:
+    needs: [build-site, run-pytest]
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout repository 🛎"
+        uses: actions/checkout@v4
+      - name: "Setup CI environment 🛠"
+        uses: ./.github/actions/set-dev-env
+        with:
+          # 3.12 is not supported by py-spy yet
+          python-version: "3.11"
+      - name: "Run profiling with py-spy 🕵️‍♂️"
+        # profiling needs to be run as sudo
+        run: python -m tox run -e py311-profile-docs -- -o docbuild_profile.svg
+        continue-on-error: true
+      - name: "Upload profiling data to GH artifacts 📤"
+        uses: actions/upload-artifact@v4
+        with:
+          name: profile-results
+          path: docbuild_profile.svg
+          if-no-files-found: ignore

.github/workflows/prerelease.yml

@@ -7,6 +7,10 @@ on:
     - cron: "0 5 * * 1,4"
   workflow_dispatch:
 
+env:
+  FORCE_COLOR: "1" # Make tools pretty
+  PIP_DISABLE_PIP_VERSION_CHECK: "1" # Don't check for pip updates
+
 jobs:
   prerelease:
     runs-on: ubuntu-latest
@@ -16,35 +20,28 @@ jobs:
         python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+      - name: "Checkout repository 🛎"
+        uses: actions/checkout@v4
+      - name: "Setup CI environment 🛠"
+        uses: ./.github/actions/set-dev-env
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "pyproject.toml"
-
-      - name: Install (prerelease) dependencies
+          pandoc: "True"
+      - name: "Install (prerelease) dependencies 📦"
         run: |
-          python -m pip install --upgrade pip wheel setuptools
-          python -m pip install --upgrade --pre -e .[doc,test]
-
-      - name: Build docs to store
+          python -Im pip install --upgrade pip wheel setuptools
+      - name: "Build PST docs and check for warnings 📖"
         run: |
-          sphinx-build -b html docs/ docs/_build/html --keep-going -w warnings.txt
-
-      - name: Check that there are no unexpected Sphinx warnings
-        if: matrix.python-version == '3.10'
-        run: python tests/utils/check_warnings.py
-
-      - name: Run the tests
+          # example substitution: tox run -e docs-py312-docs
+          python -Im tox run -e docs-py$(echo ${{ matrix.python-version }} | tr -d .)-docs -- --keep-going
+      - name: "Run tests ✅ (no coverage)"
         run: |
-          pytest --color=yes
+          # this will compile the assets then run the tests
+          python -Im tox run -e compile,py$(echo ${{ matrix.python-version }} | tr -d .)-tests-no-cov
           echo "PYTEST_ERRORS=$?" >> $GITHUB_ENV
 
       # If either the docs build or the tests resulted in an error, create an issue to note it
-      - name: Create an issue if failure
+      - name: "Create an issue if failure"
         uses: JasonEtco/create-an-issue@v2
         if: ${{ env.SPHINX_BUILD_UNEXPECTED_WARNINGS || !env.PYTEST_ERRORS }}
         env:

.github/workflows/publish.yml

@@ -17,7 +17,7 @@ permissions:
 jobs:
   # calls our tests workflow
   tests:
-    uses: ./.github/workflows/tests.yml
+    uses: ./.github/workflows/CI.yml
 
   build-package:
     name: "Build & verify PST package"
@@ -27,10 +27,11 @@ jobs:
       - name: "Checkout repository 🛎"
         uses: actions/checkout@v4
 
-      - name: "Set up Python 3.9"
-        uses: actions/setup-python@v5
+      - name: "Setup CI environment 🛠"
+        uses: ./.github/actions/set-dev-env
         with:
           python-version: "3.9"
+          pandoc: "False"
 
       - name: "Install gettext for translations 🌐"
         run: |