From efdb7280ead7ddeadab712bd541b7992703acc60 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Wed, 18 Oct 2023 14:56:23 +0100 Subject: [PATCH 01/14] Add GH_TOKEN --- action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/action.yml b/action.yml index 67d3ee3..9f1cf39 100644 --- a/action.yml +++ b/action.yml @@ -197,6 +197,7 @@ runs: INPUTS_FIXIT_VERSION: ${{ inputs.fixit-version }} INPUTS_PYRE_VERSION: ${{ inputs.pyre-version }} INPUTS_TYPESHED_VERSION: ${{ inputs.typeshed-version }} + GH_TOKEN: ${{ github.token }} shell: bash - name: Upload SARIF if: ${{ hashFiles(inputs.output) != '' }} From 4e3d9340779085d236467b41f265e6b4e1de2d73 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Wed, 18 Oct 2023 14:57:44 +0100 Subject: [PATCH 02/14] Handle typeshed_path being None --- python_lint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_lint.py b/python_lint.py index 47afd09..ac8c60c 100755 --- a/python_lint.py +++ b/python_lint.py @@ -751,7 +751,7 @@ def main() -> None: sarif_runs: List[dict] = [] target = Path(args.target).resolve().absolute() - typeshed_path = Path(args.typeshed_path).resolve().absolute() + typeshed_path = Path(args.typeshed_path).resolve().absolute() if args.typeshed_path is not None else None for linter in args.linter: LOG.debug("Running %s", linter) From 79436837b9e02a08eee477af97fd48182740d62f Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Wed, 18 Oct 2023 15:00:50 +0100 Subject: [PATCH 03/14] Pin commit --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 15b1981..507943c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -29,7 +29,7 @@ jobs: python3 -mpip install -q flake8 pylint ruff mypy pytype pyright fixit pyre-check python3 -mpip install -q flake8-sarif-formatter - name: Run Python Lint - uses: advanced-security/python-lint-code-scanning-action@main + uses: advanced-security/python-lint-code-scanning-action@4e3d9340779085d236467b41f265e6b4e1de2d73 with: linter: ${{ matrix.linter }} python-version: ${{ matrix.python-version }} From 2737939b5e0c7745040634c0b0e8e4baf7552cc3 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Wed, 18 Oct 2023 16:22:14 +0100 Subject: [PATCH 04/14] Swallow extra arg for fixit --- python_lint.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python_lint.py b/python_lint.py index ac8c60c..95d514f 100755 --- a/python_lint.py +++ b/python_lint.py @@ -55,7 +55,7 @@ def make_sarif_run(tool_name: str) -> dict: return sarif_run -def flake8_linter(target: Path, *args) -> None: +def flake8_linter(target: Path, *_args) -> None: """Run the flake8 linter. In contrast to the other linters, flake8 has plugin architecture. @@ -155,7 +155,7 @@ def ruff_format_sarif(results: List[Dict[str, Any]], target: Path) -> dict: return sarif_run -def ruff_linter(target: Path, *args) -> Optional[dict]: +def ruff_linter(target: Path, *_args) -> Optional[dict]: """Run the ruff linter.""" try: # pylint: disable=import-outside-toplevel @@ -257,7 +257,7 @@ def pylint_format_sarif(results: List[Dict[str, Any]], target: Path) -> dict: return sarif_run -def pylint_linter(target: Path, *args) -> Optional[dict]: +def pylint_linter(target: Path, *_args) -> Optional[dict]: """Run the pylint linter.""" process = run( ["pylint", "--output-format=json", "--recursive=y", target.absolute().as_posix()], @@ -680,7 +680,7 @@ def fixit_format_sarif(results: str, target: Path) -> dict: return sarif_run -def fixit_linter(target: Path) -> Optional[dict]: +def fixit_linter(target: Path, *_args) -> Optional[dict]: """Run the fixit linter, from Meta.""" process = run(["fixit", "lint", target.absolute().as_posix()], capture_output=True, check=False) From 4e84f1340935143efd1d1a0d01b33de6e5112ec7 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Wed, 18 Oct 2023 16:35:54 +0100 Subject: [PATCH 05/14] Change gh clone line --- action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 9f1cf39..8edfb5d 100644 --- a/action.yml +++ b/action.yml @@ -164,7 +164,7 @@ runs: if [[ "${install_typeshed_linters[*]}" =~ (^|[^[:alpha:]])${INPUTS_LINTER}([^[:alpha:]]|$) ]]; then echo "::debug::Installing typeshed for ${INPUTS_LINTER}" # clone from GitHub - gh repo clone python/typeshed -- --depth 1 --branch "${INPUTS_TYPESHED_VERSION}" "${GITHUB_WORKSPACE}/typeshed" || ( echo "::error::typeshed failed to install for Python ${INPUTS_PYTHON_VERSION}" && exit 1 ) + gh repo clone python/typeshed "${GITHUB_WORKSPACE}/typeshed" -- --depth 1 --branch "${INPUTS_TYPESHED_VERSION}" || ( echo "::error::typeshed failed to install for Python ${INPUTS_PYTHON_VERSION}" && exit 1 ) EXTRA_LINTER_SCRIPT_FLAGS+=" --typeshed-path=${GITHUB_WORKSPACE}/typeshed" fi From f140cdefdefe22a7707c02f3369dad6fab0b892c Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Wed, 18 Oct 2023 16:43:02 +0100 Subject: [PATCH 06/14] gh clone line --- action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 8edfb5d..f3bcc5e 100644 --- a/action.yml +++ b/action.yml @@ -164,7 +164,7 @@ runs: if [[ "${install_typeshed_linters[*]}" =~ (^|[^[:alpha:]])${INPUTS_LINTER}([^[:alpha:]]|$) ]]; then echo "::debug::Installing typeshed for ${INPUTS_LINTER}" # clone from GitHub - gh repo clone python/typeshed "${GITHUB_WORKSPACE}/typeshed" -- --depth 1 --branch "${INPUTS_TYPESHED_VERSION}" || ( echo "::error::typeshed failed to install for Python ${INPUTS_PYTHON_VERSION}" && exit 1 ) + gh repo clone python/typeshed -- --depth 1 --branch "${INPUTS_TYPESHED_VERSION}" || ( echo "::error::typeshed failed to install for Python ${INPUTS_PYTHON_VERSION}" && exit 1 ) EXTRA_LINTER_SCRIPT_FLAGS+=" --typeshed-path=${GITHUB_WORKSPACE}/typeshed" fi From 20102131aeacbb412e9983b8781f4b81bb88794b Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Wed, 18 Oct 2023 16:45:35 +0100 Subject: [PATCH 07/14] Bump --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 507943c..e1691be 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -29,7 +29,7 @@ jobs: python3 -mpip install -q flake8 pylint ruff mypy pytype pyright fixit pyre-check python3 -mpip install -q flake8-sarif-formatter - name: Run Python Lint - uses: advanced-security/python-lint-code-scanning-action@4e3d9340779085d236467b41f265e6b4e1de2d73 + uses: advanced-security/python-lint-code-scanning-action@fix-typeshed-and-token with: linter: ${{ matrix.linter }} python-version: ${{ matrix.python-version }} From 47d234112855e94c2723d6625ed927b0a8fcde32 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Wed, 18 Oct 2023 17:12:57 +0100 Subject: [PATCH 08/14] Debug SARIF output --- action.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/action.yml b/action.yml index f3bcc5e..af33268 100644 --- a/action.yml +++ b/action.yml @@ -204,3 +204,11 @@ runs: uses: github/codeql-action/upload-sarif@v2 with: sarif_file: ${{ inputs.output }} + - name: Upload SARIF as debug artefact + # if we're in debug mode and we got results + if: ${{ hashFiles(inputs.output) != '' && runner.debug == '1' }} + uses: actions/upload-artifact@v2 + with: + name: ${{ inputs.output }} + path: ${{ inputs.output }} + From f0af24acb7472da973c4b8d78a8fde8bb80febab Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Wed, 18 Oct 2023 17:21:43 +0100 Subject: [PATCH 09/14] Try debug again --- action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/action.yml b/action.yml index af33268..8e94fb4 100644 --- a/action.yml +++ b/action.yml @@ -206,8 +206,8 @@ runs: sarif_file: ${{ inputs.output }} - name: Upload SARIF as debug artefact # if we're in debug mode and we got results - if: ${{ hashFiles(inputs.output) != '' && runner.debug == '1' }} - uses: actions/upload-artifact@v2 + if: runner.debug == '1' + uses: actions/upload-artifact@v3 with: name: ${{ inputs.output }} path: ${{ inputs.output }} From 34c89eac9e770ec005653276a6e1e04d72c7ee37 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Wed, 18 Oct 2023 17:23:10 +0100 Subject: [PATCH 10/14] Reduce workload for testing --- .github/workflows/lint.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e1691be..dd42a79 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,9 +14,9 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - linter: ['flake8', 'pylint', 'ruff', 'mypy', 'pytype', 'pyright', 'fixit', 'pyre'] - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] - os: [ubuntu-latest, macos-latest] # doesn't yet work on Windows + linter: ['pyre'] # ['flake8', 'pylint', 'ruff', 'mypy', 'pytype', 'pyright', 'fixit', 'pyre'] + python-version: ['3.11'] # ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] + os: ['ubuntu-latest'] # [ubuntu-latest, macos-latest] # doesn't yet work on Windows fail-fast: false steps: From 8a416b4105f4bd8329c300db0d54e9b977a1d6f9 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Wed, 18 Oct 2023 18:37:45 +0100 Subject: [PATCH 11/14] Get debug results --- action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/action.yml b/action.yml index 8e94fb4..529c878 100644 --- a/action.yml +++ b/action.yml @@ -205,8 +205,7 @@ runs: with: sarif_file: ${{ inputs.output }} - name: Upload SARIF as debug artefact - # if we're in debug mode and we got results - if: runner.debug == '1' + if: ${{ always() && runner.debug == '1' && hashFiles(inputs.output) != '' }} uses: actions/upload-artifact@v3 with: name: ${{ inputs.output }} From ae5422d9f6af8cdfbb5e02a783b7c7889f6320ed Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Wed, 18 Oct 2023 18:44:00 +0100 Subject: [PATCH 12/14] Clone typeshed into RUNNER_TEMP instead --- action.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/action.yml b/action.yml index 529c878..00729ca 100644 --- a/action.yml +++ b/action.yml @@ -164,8 +164,11 @@ runs: if [[ "${install_typeshed_linters[*]}" =~ (^|[^[:alpha:]])${INPUTS_LINTER}([^[:alpha:]]|$) ]]; then echo "::debug::Installing typeshed for ${INPUTS_LINTER}" # clone from GitHub - gh repo clone python/typeshed -- --depth 1 --branch "${INPUTS_TYPESHED_VERSION}" || ( echo "::error::typeshed failed to install for Python ${INPUTS_PYTHON_VERSION}" && exit 1 ) - EXTRA_LINTER_SCRIPT_FLAGS+=" --typeshed-path=${GITHUB_WORKSPACE}/typeshed" + ( + cd ${RUNNER_TEMP} + gh repo clone python/typeshed -- --depth 1 --branch "${INPUTS_TYPESHED_VERSION}" || ( echo "::error::typeshed failed to install for Python ${INPUTS_PYTHON_VERSION}" && exit 1 ) + ) + EXTRA_LINTER_SCRIPT_FLAGS+=" --typeshed-path=${RUNNER_TEMP}/typeshed" fi # run linter From c2c374af0e158720a7c1ef73b3c605f138613908 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Wed, 18 Oct 2023 18:54:01 +0100 Subject: [PATCH 13/14] Fix up SARIF locations --- python_lint.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/python_lint.py b/python_lint.py index 95d514f..b8dbbf5 100755 --- a/python_lint.py +++ b/python_lint.py @@ -712,6 +712,31 @@ def make_paths_relative_to_target(runs: List[dict], target: Path) -> None: ) +def fix_sarif_locations(runs: List[dict]) -> None: + """Fix the SARIF locations. + + Normalise values less than 1 to 1, e.g. -1 or 0. + + Convert strings to ints. + + For anything that can't be converted to an int, set it to 1. + """ + for sarif_run in runs: + for result in sarif_run["results"]: + for location in result["locations"]: + region = location["physicalLocation"]["region"] + for key in ("startLine", "endLine", "startColumn", "endColumn"): + if key in region: + try: + region[key] = int(region[key]) + except ValueError: + LOG.error("Unable to convert %s to int", region[key]) + region[key] = 1 + continue + if region[key] < 1: + region[key] = 1 + + LINTERS = { "pylint": pylint_linter, "ruff": ruff_linter, From d064d54000eca56145cb3195a0804cfb3bdfc48e Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Wed, 18 Oct 2023 18:55:09 +0100 Subject: [PATCH 14/14] Reset to main branch linting, with all matrix values --- .github/workflows/lint.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index dd42a79..15b1981 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,9 +14,9 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - linter: ['pyre'] # ['flake8', 'pylint', 'ruff', 'mypy', 'pytype', 'pyright', 'fixit', 'pyre'] - python-version: ['3.11'] # ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] - os: ['ubuntu-latest'] # [ubuntu-latest, macos-latest] # doesn't yet work on Windows + linter: ['flake8', 'pylint', 'ruff', 'mypy', 'pytype', 'pyright', 'fixit', 'pyre'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] + os: [ubuntu-latest, macos-latest] # doesn't yet work on Windows fail-fast: false steps: @@ -29,7 +29,7 @@ jobs: python3 -mpip install -q flake8 pylint ruff mypy pytype pyright fixit pyre-check python3 -mpip install -q flake8-sarif-formatter - name: Run Python Lint - uses: advanced-security/python-lint-code-scanning-action@fix-typeshed-and-token + uses: advanced-security/python-lint-code-scanning-action@main with: linter: ${{ matrix.linter }} python-version: ${{ matrix.python-version }}