Try to implement coverage action instead of using codecov. (#1756)

* Try to implement coverage action instead of using codecov.

See #1753

* Update .github/workflows/tests.yml

* Apply suggestions from code review

Co-authored-by: Tania Allard <[email protected]>

---------

Co-authored-by: Tania Allard <[email protected]>
Co-authored-by: Daniel McCloy <[email protected]>

.github/workflows/coverage.yml

@@ -0,0 +1,23 @@
+name: Post coverage comment
+
+on:
+  workflow_run:
+    workflows: ["continuous-integration"]
+    types:
+      - completed
+
+jobs:
+  test:
+    name: Run tests & display coverage
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success'
+    permissions:
+      pull-requests: write
+      contents: write # needed to edit the comment vs opening multiple ones
+      actions: read
+    steps:
+      - name: Post comment
+        uses: py-cov-action/python-coverage-comment-action@v3
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_PR_RUN_ID: ${{ github.event.workflow_run.id }}

.github/workflows/tests.yml

@@ -94,6 +94,20 @@ jobs:
           nox -s compile
       - name: Run tests
         run: pytest -m "not a11y" --color=yes --cov --cov-report=xml
+        env:
+          COVERAGE_FILE: ".coverage.${{ matrix.python-version }}.${{ matrix.os }}.${{ matrix.sphinx-version }}"
+      - name: Store coverage file
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-${{ matrix.python-version }}-${{ matrix.os }}-${{ matrix.sphinx-version }}
+          path: .coverage.${{ matrix.python-version }}.${{ matrix.os }}.${{ matrix.sphinx-version }}
+      - name: Upload to Codecov
+        if: matrix.python-version == '3.12' && matrix.os == 'ubuntu-latest' && matrix.sphinx-version == 'dev'
+        uses: codecov/codecov-action@v4
+        with:
+          fail_ci_if_error: true
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
       # note I am setting this on top of the Python cache as I could not find
       # how to set the hash key on the python one
@@ -112,6 +126,38 @@ jobs:
           nox -s a11y
         continue-on-error: true
 
+  coverage:
+    name: Collect Coverage
+    runs-on: ubuntu-latest
+    needs: run-pytest
+    # run both on previous step success and failure
+    if: "!cancelled()"
+    permissions:
+      pull-requests: write
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/download-artifact@v4
+        id: download
+        with:
+          pattern: coverage-*
+          merge-multiple: true
+
+      - name: Coverage comment
+        id: coverage_comment
+        uses: py-cov-action/python-coverage-comment-action@v3
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          MERGE_COVERAGE_FILES: true
+
+      - name: Store Pull Request comment to be posted
+        uses: actions/upload-artifact@v4
+        if: steps.coverage_comment.outputs.COMMENT_FILE_WRITTEN == 'true'
+        with:
+          name: python-coverage-comment-action
+          path: python-coverage-comment-action.txt
+
   # Build our site on the 3 major OSes and check for Sphinx warnings
   build-site:
     needs: [lint]

pyproject.toml

@@ -120,6 +120,7 @@ markers = "a11y: mark a test as an accessibility test"
 [tool.coverage.run]
 source = ["pydata_sphinx_theme"]
 branch = true
+relative_files = true
 
 [tool.coverage.report]
 fail_under = 60