diff --git a/.github/linters/lychee.toml b/.github/linters/lychee.toml new file mode 100644 index 0000000000..5e7c88f80d --- /dev/null +++ b/.github/linters/lychee.toml @@ -0,0 +1,13 @@ +exclude = ["docs/api/javadoc/sql/org/apache/sedona/sql/utils/index.html$", + "docs/api/scaladoc/spark/org/apache/sedona/sql/utils/index.html$", + "docs/javadoc/spark", + "docs/scaladoc/spark/org/apache/sedona/sql/utils/index.html$", + "spark://localhost:.*", + "https://issues.apache.org/jira.*", + "https://keyserver.pgp.com/.*", + "http://localhost:.*", + "https://whimsy.apache.org/officers/acreq.*", + "https://whimsy.apache.org/roster/pmc/sedona.*", + "https://www.datasyslab.net/", + "https://mvnrepository.com/.*", + "https://www.researchgate.net/figure/Relation-between-the-cartesian-axes-x-y-and-i-j-axes-of-the-pixels_fig3_313860913"] diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8ee034f12c..797b3b70ac 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,3 +1,4 @@ +# https://pre-commit.com/ name: Lint on: [pull_request] @@ -6,8 +7,33 @@ permissions: contents: read jobs: + lychee: + name: Check links with lychee + runs-on: ubuntu-latest + steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.x' + architecture: 'x64' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pre-commit + curl -sSf 'https://sh.rustup.rs' | sh -s -- -y + sudo apt install gcc pkg-config libc6-dev libssl-dev + cargo install lychee + - name: set PY + run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV + - uses: actions/cache@v3 + with: + path: ~/.cache/pre-commit + key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} + - name: Run pre-commit + run: pre-commit run lychee --all-files pre-commit: - name: Run pre-commit # https://pre-commit.com/ + name: Run pre-commit runs-on: ubuntu-latest steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" @@ -27,4 +53,4 @@ jobs: path: ~/.cache/pre-commit key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} - name: Run pre-commit - run: pre-commit run --all-files + run: SKIP=lychee pre-commit run --all-files diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f84f612d54..7272c21d4f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,7 +28,7 @@ repos: rev: v2.3.0 hooks: - id: codespell - name: Run codespell + name: run codespell description: Check spelling with codespell args: [--ignore-words=.github/linters/codespell.txt] exclude: ^docs/image|^spark/common/src/test/resources|^docs/usecases|^tools/maven/scalafmt @@ -75,11 +75,18 @@ repos: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] exclude: ^docs-overrides/main\.html$|\.Rd$ + - repo: https://github.com/lycheeverse/lychee.git + rev: v0.15.0 + hooks: + - id: lychee + name: run lychee + args: [--config=.github/linters/lychee.toml] + files: \.md$ - repo: https://github.com/igorshubovych/markdownlint-cli rev: v0.42.0 hooks: - id: markdownlint - name: Run markdownlint + name: run markdownlint description: Check Markdown files with markdownlint args: [--config=.github/linters/.markdown-lint.yml] exclude: ^\.github/.*$ diff --git a/Makefile b/Makefile index 775d1d01c3..4d6c9488ca 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,15 @@ check : - pre-commit run --all-files + SKIP=lychee pre-commit run --all-files .PHONY : check checkinstall : pre-commit install .PHONY : checkinstall +checklinks : + pre-commit run lychee --all-files +.PHONY : checklinks + checkupdate : pre-commit autoupdate .PHONY : checkupdate diff --git a/docs/api/flink/Function.md b/docs/api/flink/Function.md index 2fcfaa8d0f..c8a781df17 100644 --- a/docs/api/flink/Function.md +++ b/docs/api/flink/Function.md @@ -3386,7 +3386,7 @@ SRID=4326;POLYGON ((0 0, -0.8390715290764524 0, -0.8390715290764524 1, 0 0)) ## ST_S2CellIDs Introduction: Cover the geometry with Google S2 Cells, return the corresponding cell IDs with the given level. -The level indicates the [size of cells](https://s2geometry.io/resources/s2cell_statistics.html). With a bigger level, +The level indicates the [size of cells](http://s2geometry.io/resources/s2cell_statistics.html). With a bigger level, the cells will be smaller, the coverage will be more accurate, but the result size will be exponentially increasing. Format: `ST_S2CellIDs(geom: Geometry, level: Integer)` diff --git a/docs/api/snowflake/vector-data/Function.md b/docs/api/snowflake/vector-data/Function.md index d171b478a3..51f74ff77d 100644 --- a/docs/api/snowflake/vector-data/Function.md +++ b/docs/api/snowflake/vector-data/Function.md @@ -2619,7 +2619,7 @@ SRID=4326;POLYGON ((0 0, -0.8390715290764524 0, -0.8390715290764524 1, 0 0)) ## ST_S2CellIDs Introduction: Cover the geometry with Google S2 Cells, return the corresponding cell IDs with the given level. -The level indicates the [size of cells](https://s2geometry.io/resources/s2cell_statistics.html). With a bigger level, +The level indicates the [size of cells](http://s2geometry.io/resources/s2cell_statistics.html). With a bigger level, the cells will be smaller, the coverage will be more accurate, but the result size will be exponentially increasing. Format: `ST_S2CellIDs(geom: geometry, level: Int)` diff --git a/docs/api/sql/Function.md b/docs/api/sql/Function.md index 6410112e32..2c776ce99c 100644 --- a/docs/api/sql/Function.md +++ b/docs/api/sql/Function.md @@ -3466,7 +3466,7 @@ SRID=4326;POLYGON ((0 0, -0.8390715290764524 0, -0.8390715290764524 1, 0 0)) ## ST_S2CellIDs Introduction: Cover the geometry with Google S2 Cells, return the corresponding cell IDs with the given level. -The level indicates the [size of cells](https://s2geometry.io/resources/s2cell_statistics.html). With a bigger level, +The level indicates the [size of cells](http://s2geometry.io/resources/s2cell_statistics.html). With a bigger level, the cells will be smaller, the coverage will be more accurate, but the result size will be exponentially increasing. Format: `ST_S2CellIDs(geom: Geometry, level: Integer)` diff --git a/docs/setup/compile.md b/docs/setup/compile.md index 5589874f56..714e0fee37 100644 --- a/docs/setup/compile.md +++ b/docs/setup/compile.md @@ -162,22 +162,39 @@ mike serve ## pre-commit -We run [pre-commit](https://pre-commit.com/) with GitHub Actions so installation on -your local machine is currently optional. +We run [pre-commit](https://pre-commit.com/) with GitHub Actions so installation on your local machine is currently +optional. -The pre-commit [configuration file](https://github.com/apache/sedona/blob/master/.pre-commit-config.yaml) -is in the repository root. Before you can run the hooks, you need to have pre-commit installed. +The pre-commit [configuration file](https://github.com/apache/sedona/blob/master/.pre-commit-config.yaml) is in the +repository root. Before you can run the hooks, you need to have pre-commit installed. -The hooks run when running `git commit`. Some of the hooks will auto fix the code after the hook fails -whilst most will print error messages from the linters. - -If you want to test all hooks against all files and when you are adding a new hook -you should always run: +The hooks run when running `git commit` and also from the command line with `pre-commit`. Some of the hooks will auto +fix the code after the hooks fail whilst most will print error messages from the linters. If a hook fails the overall +commit will fail, and you will need to fix the issues or problems and `git add` and git commit again. On git commit +the hooks will run mostly only against modified files so if you want to test all hooks against all files and when you +are adding a new hook you should always run: `pre-commit run --all-files` -Sometimes you might need to skip a hook to commit for example: +We are also using pre-commit to check our links with [lychee](https://github.com/lycheeverse/lychee). The lychee hook +is not self-contained and requires a local lychee installation. + +Sometimes you might need to skip a hook to commit because the hook is stopping you from committing or your computer +might not have all the installation requirements for all the hooks. The `SKIP` variable is comma separated for two or +more hooks: + +`SKIP=lychee git commit -m "foo"` + +The same applies when running pre-commit: + +`SKIP=lychee pre-commit run --all-files` + +If you just want to run one hook for example just run the `markdownlint` hook: + +`pre-commit run markdownlint --all-files` -`SKIP=markdownlint git commit -m "foo"` +We have a [Makefile](https://github.com/apache/sedona/blob/master/Makefile) in the repository root which has four +pre-commit convenience commands. For example to run [Make](https://en.wikipedia.org/wiki/Make_(software)) to check the +links with lychee run: -We have a [Makefile](https://github.com/apache/sedona/blob/master/Makefile) in the repository root which has three pre-commit convenience commands. +`make checklinks` diff --git a/docs/tutorial/sql.md b/docs/tutorial/sql.md index 4b71ee3cd4..4f11f0ebb1 100644 --- a/docs/tutorial/sql.md +++ b/docs/tutorial/sql.md @@ -851,7 +851,7 @@ Sedona provides `SedonaPyDeck` and `SedonaKepler` wrappers, both of which expose Spatial query results can be visualized in a Jupyter lab/notebook environment using SedonaPyDeck. -SedonaPyDeck exposes APIs to create interactive map visualizations using [pydeck](https://pydeck.gl/index.html#) based on [deck.gl](https://deck.gl/) +SedonaPyDeck exposes APIs to create interactive map visualizations using [pydeck](https://pydeck.gl/) based on [deck.gl](https://deck.gl/) !!!Note To use SedonaPyDeck, install sedona with the `pydeck-map` extra: