diff --git a/README.md b/README.md index c4135db2..bbfef57c 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,11 @@ After migrations and development database loading are in place, you can rebuild > ./scripts/update ``` +`pip` dependencies in `setup.py` are collected and installed through requirements files. +If you modify dependencies, run `./scripts/generate-requirements` to regenerate +`requirements-*.txt` used by Dockerfiles otherwise your dependency change will not +be realized. + #### Running the services There is a local proxy service that facilitates a local "managed identity" functionality, run as your local identity. Make sure to run diff --git a/docker-compose.yml b/docker-compose.yml index 8afa3bca..63790237 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -101,7 +101,7 @@ services: azurite: container_name: pcapis-azurite - image: mcr.microsoft.com/azure-storage/azurite:3.17.1 + image: mcr.microsoft.com/azure-storage/azurite:3.29.0 hostname: azurite command: "azurite --silent --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 -l /workspace" diff --git a/pccommon/requirements.txt b/pccommon/requirements.txt new file mode 100644 index 00000000..a4b533e9 --- /dev/null +++ b/pccommon/requirements.txt @@ -0,0 +1,165 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --extra=server --output-file=pccommon/requirements.txt ./pccommon/setup.py +# +anyio==4.3.0 + # via starlette +async-timeout==4.0.3 + # via redis +azure-core==1.30.1 + # via + # azure-data-tables + # azure-identity + # azure-storage-blob + # msrest +azure-data-tables==12.4.0 + # via pccommon (pccommon/setup.py) +azure-identity==1.7.1 + # via pccommon (pccommon/setup.py) +azure-storage-blob==12.19.1 + # via pccommon (pccommon/setup.py) +beautifulsoup4==4.12.3 + # via html-sanitizer +cachetools==5.3.3 + # via + # google-auth + # pccommon (pccommon/setup.py) +certifi==2024.2.2 + # via + # msrest + # requests +cffi==1.16.0 + # via cryptography +charset-normalizer==3.3.2 + # via requests +cryptography==42.0.5 + # via + # azure-identity + # azure-storage-blob + # msal + # pyjwt +deprecated==1.2.14 + # via redis +exceptiongroup==1.2.0 + # via anyio +fastapi==0.90.1 + # via pccommon (pccommon/setup.py) +google-api-core==2.18.0 + # via opencensus +google-auth==2.29.0 + # via google-api-core +googleapis-common-protos==1.63.0 + # via google-api-core +html-sanitizer==2.4.0 + # via pccommon (pccommon/setup.py) +idna==3.6 + # via + # anyio + # requests +isodate==0.6.1 + # via + # azure-storage-blob + # msrest +lxml[html-clean]==5.2.1 + # via + # html-sanitizer + # lxml-html-clean +lxml-html-clean==0.1.0 + # via pccommon (pccommon/setup.py) +msal==1.28.0 + # via + # azure-identity + # msal-extensions +msal-extensions==0.3.1 + # via azure-identity +msrest==0.7.1 + # via azure-data-tables +oauthlib==3.2.2 + # via requests-oauthlib +opencensus==0.11.4 + # via + # opencensus-ext-azure + # opencensus-ext-logging +opencensus-context==0.1.3 + # via opencensus +opencensus-ext-azure==1.0.8 + # via pccommon (pccommon/setup.py) +opencensus-ext-logging==0.1.0 + # via pccommon (pccommon/setup.py) +orjson==3.9.15 + # via pccommon (pccommon/setup.py) +packaging==24.0 + # via redis +portalocker==2.8.2 + # via msal-extensions +proto-plus==1.23.0 + # via google-api-core +protobuf==4.25.3 + # via + # google-api-core + # googleapis-common-protos + # proto-plus +psutil==5.9.8 + # via opencensus-ext-azure +pyasn1==0.5.1 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth +pycparser==2.21 + # via cffi +pydantic==1.10.14 + # via + # fastapi + # pccommon (pccommon/setup.py) +pyhumps==3.5.3 + # via pccommon (pccommon/setup.py) +pyjwt[crypto]==2.8.0 + # via msal +redis==4.2.0rc1 + # via pccommon (pccommon/setup.py) +requests==2.31.0 + # via + # azure-core + # google-api-core + # msal + # msrest + # opencensus-ext-azure + # requests-oauthlib +requests-oauthlib==2.0.0 + # via msrest +rsa==4.9 + # via google-auth +six==1.16.0 + # via + # azure-core + # azure-identity + # isodate + # opencensus +sniffio==1.3.1 + # via anyio +soupsieve==2.5 + # via beautifulsoup4 +starlette==0.22.0 + # via + # fastapi + # pccommon (pccommon/setup.py) +types-cachetools==4.2.9 + # via pccommon (pccommon/setup.py) +typing-extensions==4.10.0 + # via + # anyio + # azure-core + # azure-storage-blob + # pydantic + # redis + # starlette +urllib3==2.2.1 + # via + # pccommon (pccommon/setup.py) + # requests +wrapt==1.16.0 + # via deprecated diff --git a/pccommon/setup.py b/pccommon/setup.py index 60cc7e8e..b49207f9 100644 --- a/pccommon/setup.py +++ b/pccommon/setup.py @@ -8,10 +8,10 @@ "starlette>=0.22.0,<0.23.0", "opencensus-ext-azure==1.0.8", "opencensus-ext-logging==0.1.0", - "orjson==3.5.2", + "orjson>=3.9.15", "azure-identity==1.7.1", "azure-data-tables==12.4.0", - "azure-storage-blob==12.12.0", + "azure-storage-blob>=12.19.1", "pydantic>=1.9, <2.0.0", "cachetools~=5.3", "types-cachetools==4.2.9", @@ -20,6 +20,7 @@ "html-sanitizer==2.4", # Soon available as lxml[html_clean] "lxml_html_clean==0.1.0", + "urllib3>=1.26.18", ] extra_reqs = { diff --git a/pcfuncs/Dockerfile b/pcfuncs/Dockerfile index 9ffc6bc0..3beb133f 100644 --- a/pcfuncs/Dockerfile +++ b/pcfuncs/Dockerfile @@ -11,10 +11,12 @@ COPY pccommon /home/site/pccommon RUN mkdir -p /home/site/wwwroot WORKDIR /home/site/wwwroot -COPY pcfuncs/requirements.txt /home/site/wwwroot/requirements.txt -RUN pip install -r /home/site/wwwroot/requirements.txt +RUN --mount=type=cache,target=/root/.cache \ + --mount=type=bind,source=pcfuncs/requirements.txt,target=/home/site/wwwroot/requirements.txt \ + pip install -r /home/site/wwwroot/requirements.txt -COPY requirements-dev.txt /home/site/ -RUN pip install -r /home/site/requirements-dev.txt +RUN --mount=type=cache,target=/root/.cache \ + --mount=type=bind,source=requirements-dev.txt,target=/home/site/requirements-dev.txt \ + pip install -r /home/site/requirements-dev.txt ENV PYTHONPATH=/home/site/wwwroot diff --git a/pcfuncs/requirements-deploy.txt b/pcfuncs/requirements-deploy.txt index 5d81fac2..a5b547b5 100644 --- a/pcfuncs/requirements-deploy.txt +++ b/pcfuncs/requirements-deploy.txt @@ -6,7 +6,7 @@ azure-functions -requests==2.28.1 +requests==2.31.0 aiohttp==3.8.1 dateutils==0.6.12 mercantile==1.2.1 diff --git a/pcfuncs/requirements.txt b/pcfuncs/requirements.txt index a5650fd2..dc854675 100644 --- a/pcfuncs/requirements.txt +++ b/pcfuncs/requirements.txt @@ -6,7 +6,7 @@ azure-functions -requests==2.28.1 +requests==2.31.0 aiohttp==3.8.1 dateutils==0.6.12 mercantile==1.2.1 diff --git a/pcstac/Dockerfile b/pcstac/Dockerfile index 14cdc4a1..a206280c 100644 --- a/pcstac/Dockerfile +++ b/pcstac/Dockerfile @@ -9,7 +9,15 @@ WORKDIR /opt/src COPY pcstac /opt/src/pcstac COPY pccommon /opt/src/pccommon -RUN pip install -r ./pcstac/requirements.txt -e ./pccommon -e ./pcstac[server] +RUN --mount=type=cache,target=/root/.cache \ + pip install -U "setuptools>=65.5.1" +# The order of these pip installs is important :( +RUN --mount=type=cache,target=/root/.cache \ + pip install -r ./pccommon/requirements.txt +RUN --mount=type=cache,target=/root/.cache \ + pip install -r ./pcstac/requirements-server.txt +RUN --mount=type=cache,target=/root/.cache \ + pip install --no-deps -e ./pccommon -e ./pcstac[server] ENV APP_HOST=0.0.0.0 ENV APP_PORT=81 diff --git a/pcstac/Dockerfile.dev b/pcstac/Dockerfile.dev index b61d44a3..380d98eb 100644 --- a/pcstac/Dockerfile.dev +++ b/pcstac/Dockerfile.dev @@ -1,6 +1,9 @@ FROM pc-apis-stac -COPY requirements-dev.txt requirements-dev.txt -RUN pip install -r requirements-dev.txt -RUN pip install -e ./pccommon[dev] -e ./pcstac +RUN --mount=type=cache,target=/root/.cache \ + --mount=type=bind,source=requirements-dev.txt,target=requirements-dev.txt \ + pip install -r requirements-dev.txt + +RUN --mount=type=cache,target=/root/.cache \ + pip install -e ./pccommon[dev] -e ./pcstac diff --git a/pcstac/requirements-server.txt b/pcstac/requirements-server.txt new file mode 100644 index 00000000..4f5bedfc --- /dev/null +++ b/pcstac/requirements-server.txt @@ -0,0 +1,166 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --extra=server --output-file=pcstac/requirements-server.txt ./pcstac/setup.py +# +anyio==3.7.1 + # via + # starlette + # watchgod +asgiref==3.8.1 + # via uvicorn +async-timeout==4.0.3 + # via asyncpg +asyncpg==0.29.0 + # via stac-fastapi-pgstac +attrs==23.2.0 + # via + # stac-fastapi-api + # stac-fastapi-extensions + # stac-fastapi-pgstac + # stac-fastapi-types +brotli==1.1.0 + # via brotli-asgi +brotli-asgi==1.4.0 + # via + # stac-fastapi-api + # stac-fastapi-pgstac +buildpg==0.4 + # via stac-fastapi-pgstac +cachetools==5.3.3 + # via pypgstac +click==8.1.7 + # via + # stac-pydantic + # uvicorn +dateparser==1.2.0 + # via pygeofilter +exceptiongroup==1.2.0 + # via anyio +fastapi==0.110.0 + # via stac-fastapi-types +fire==0.4.0 + # via pypgstac +geojson-pydantic==0.6.3 + # via stac-pydantic +h11==0.14.0 + # via uvicorn +httptools==0.6.1 + # via uvicorn +idna==3.6 + # via anyio +iso8601==1.1.0 + # via stac-fastapi-types +lark==0.12.0 + # via pygeofilter +orjson==3.9.15 + # via + # pcstac (pcstac/setup.py) + # pypgstac + # stac-fastapi-pgstac +plpygis==0.2.2 + # via pypgstac +psycopg[binary]==3.1.18 + # via pypgstac +psycopg-binary==3.1.18 + # via psycopg +psycopg-pool==3.1.9 + # via pypgstac +pydantic[dotenv]==1.10.14 + # via + # fastapi + # geojson-pydantic + # pypgstac + # stac-fastapi-api + # stac-fastapi-extensions + # stac-fastapi-pgstac + # stac-fastapi-types + # stac-pydantic +pygeofilter==0.2.1 + # via stac-fastapi-pgstac +pygeoif==1.4.0 + # via pygeofilter +pypgstac[psycopg]==0.7.10 + # via + # pcstac (pcstac/setup.py) + # stac-fastapi-pgstac +pystac==1.9.0 + # via + # pcstac (pcstac/setup.py) + # stac-fastapi-types +python-dateutil==2.8.2 + # via + # dateparser + # pypgstac + # pystac +python-dotenv==1.0.1 + # via + # pydantic + # uvicorn +pytz==2024.1 + # via dateparser +pyyaml==6.0.1 + # via uvicorn +regex==2023.12.25 + # via dateparser +six==1.16.0 + # via + # fire + # python-dateutil +smart-open==6.4.0 + # via pypgstac +sniffio==1.3.1 + # via anyio +stac-fastapi-api==2.4.8 + # via + # pcstac (pcstac/setup.py) + # stac-fastapi-extensions + # stac-fastapi-pgstac +stac-fastapi-extensions==2.4.8 + # via + # pcstac (pcstac/setup.py) + # stac-fastapi-pgstac +stac-fastapi-pgstac==2.4.9 + # via pcstac (pcstac/setup.py) +stac-fastapi-types==2.4.8 + # via + # pcstac (pcstac/setup.py) + # stac-fastapi-api + # stac-fastapi-extensions + # stac-fastapi-pgstac +stac-pydantic==2.0.3 + # via + # stac-fastapi-api + # stac-fastapi-extensions + # stac-fastapi-pgstac + # stac-fastapi-types +starlette==0.36.3 + # via + # brotli-asgi + # fastapi +tenacity==8.1.0 + # via pypgstac +termcolor==2.4.0 + # via fire +typing-extensions==4.10.0 + # via + # asgiref + # fastapi + # psycopg + # psycopg-pool + # pydantic + # pygeoif + # starlette +tzlocal==5.2 + # via dateparser +uvicorn[standard]==0.17.6 + # via pcstac (pcstac/setup.py) +uvloop==0.19.0 + # via uvicorn +version-parser==1.0.1 + # via pypgstac +watchgod==0.8.2 + # via uvicorn +websockets==12.0 + # via uvicorn diff --git a/pcstac/requirements.txt b/pcstac/requirements.txt deleted file mode 100644 index 48f38069..00000000 --- a/pcstac/requirements.txt +++ /dev/null @@ -1,295 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile --extra=server --output-file=pcstac/requirements.txt pccommon/setup.py pcstac/setup.py -# -anyio==3.7.0 - # via - # starlette - # watchgod -asgiref==3.7.2 - # via uvicorn -async-timeout==4.0.2 - # via redis -asyncpg==0.27.0 - # via stac-fastapi-pgstac -attrs==23.1.0 - # via - # stac-fastapi-api - # stac-fastapi-extensions - # stac-fastapi-pgstac - # stac-fastapi-types -azure-core==1.27.0 - # via - # azure-data-tables - # azure-identity - # azure-storage-blob - # msrest -azure-data-tables==12.4.0 - # via pccommon (pccommon/setup.py) -azure-identity==1.7.1 - # via pccommon (pccommon/setup.py) -azure-storage-blob==12.12.0 - # via pccommon (pccommon/setup.py) -beautifulsoup4==4.12.2 - # via html-sanitizer -brotli==1.0.9 - # via brotli-asgi -brotli-asgi==1.3.0 - # via - # stac-fastapi-api - # stac-fastapi-pgstac -buildpg==0.4 - # via stac-fastapi-pgstac -cachetools==5.3.1 - # via - # google-auth - # pccommon (pccommon/setup.py) - # pypgstac -certifi==2023.5.7 - # via - # msrest - # requests -cffi==1.15.1 - # via cryptography -charset-normalizer==3.1.0 - # via requests -click==8.1.3 - # via - # stac-pydantic - # uvicorn -cryptography==41.0.1 - # via - # azure-identity - # azure-storage-blob - # msal - # pyjwt -dateparser==1.1.8 - # via pygeofilter -deprecated==1.2.14 - # via redis -exceptiongroup==1.1.1 - # via anyio -fastapi==0.90.1 - # via - # pccommon (pccommon/setup.py) - # stac-fastapi-types -fire==0.4.0 - # via pypgstac -geojson-pydantic==0.6.2 - # via stac-pydantic -google-api-core==2.11.0 - # via opencensus -google-auth==2.19.1 - # via google-api-core -googleapis-common-protos==1.59.1 - # via google-api-core -h11==0.14.0 - # via uvicorn -html-sanitizer==2.4.0 - # via pccommon (pccommon/setup.py) -httptools==0.5.0 - # via uvicorn -idna==3.4 - # via - # anyio - # requests -iso8601==1.1.0 - # via stac-fastapi-types -isodate==0.6.1 - # via msrest -lark==0.12.0 - # via pygeofilter -lxml[html-clean]==5.2.0 - # via - # html-sanitizer - # lxml-html-clean -lxml-html-clean==0.1.0 - # via pccommon (pccommon/setup.py) -msal==1.22.0 - # via - # azure-identity - # msal-extensions -msal-extensions==0.3.1 - # via azure-identity -msrest==0.7.1 - # via - # azure-data-tables - # azure-storage-blob -oauthlib==3.2.2 - # via requests-oauthlib -opencensus==0.11.2 - # via - # opencensus-ext-azure - # opencensus-ext-logging -opencensus-context==0.1.3 - # via opencensus -opencensus-ext-azure==1.0.8 - # via pccommon (pccommon/setup.py) -opencensus-ext-logging==0.1.0 - # via pccommon (pccommon/setup.py) -orjson==3.5.2 - # via - # pccommon (pccommon/setup.py) - # pypgstac - # stac-fastapi-pgstac -packaging==23.1 - # via redis -plpygis==0.2.1 - # via pypgstac -portalocker==2.7.0 - # via msal-extensions -protobuf==4.23.2 - # via - # google-api-core - # googleapis-common-protos -psutil==5.9.5 - # via opencensus-ext-azure -psycopg[binary]==3.1.9 - # via pypgstac -psycopg-binary==3.1.9 - # via psycopg -psycopg-pool==3.1.7 - # via pypgstac -pyasn1==0.5.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.3.0 - # via google-auth -pycparser==2.21 - # via cffi -pydantic[dotenv]==1.10.9 - # via - # fastapi - # geojson-pydantic - # pccommon (pccommon/setup.py) - # pypgstac - # stac-fastapi-api - # stac-fastapi-extensions - # stac-fastapi-pgstac - # stac-fastapi-types - # stac-pydantic -pygeofilter==0.2.1 - # via stac-fastapi-pgstac -pygeoif==1.0.0 - # via pygeofilter -pyhumps==3.5.3 - # via pccommon (pccommon/setup.py) -pyjwt[crypto]==2.7.0 - # via - # msal - # pyjwt -pypgstac[psycopg]==0.7.9 - # via - # pcstac (pcstac/setup.py) - # stac-fastapi-pgstac -pystac==1.9.0 - # via - # pcstac (pcstac/setup.py) - # stac-fastapi-types -python-dateutil==2.8.2 - # via - # dateparser - # pypgstac - # pystac -python-dotenv==1.0.0 - # via - # pydantic - # uvicorn -pytz==2023.3 - # via dateparser -pyyaml==6.0 - # via uvicorn -redis==4.2.0rc1 - # via pccommon (pccommon/setup.py) -regex==2023.6.3 - # via dateparser -requests==2.31.0 - # via - # azure-core - # google-api-core - # msal - # msrest - # opencensus-ext-azure - # requests-oauthlib -requests-oauthlib==1.3.1 - # via msrest -rsa==4.9 - # via google-auth -six==1.16.0 - # via - # azure-core - # azure-identity - # fire - # google-auth - # isodate - # python-dateutil -smart-open==6.3.0 - # via pypgstac -sniffio==1.3.0 - # via anyio -soupsieve==2.4.1 - # via beautifulsoup4 -stac-fastapi-api==2.4.8 - # via - # pcstac (pcstac/setup.py) - # stac-fastapi-extensions - # stac-fastapi-pgstac -stac-fastapi-extensions==2.4.8 - # via - # pcstac (pcstac/setup.py) - # stac-fastapi-pgstac -stac-fastapi-pgstac==2.4.9 - # via pcstac (pcstac/setup.py) -stac-fastapi-types==2.4.8 - # via - # pcstac (pcstac/setup.py) - # stac-fastapi-api - # stac-fastapi-extensions - # stac-fastapi-pgstac -stac-pydantic==2.0.3 - # via - # stac-fastapi-api - # stac-fastapi-extensions - # stac-fastapi-pgstac - # stac-fastapi-types -starlette==0.22.0 - # via - # brotli-asgi - # fastapi - # pccommon (pccommon/setup.py) -tenacity==8.1.0 - # via pypgstac -termcolor==2.3.0 - # via fire -types-cachetools==4.2.9 - # via pccommon (pccommon/setup.py) -typing-extensions==4.6.3 - # via - # asgiref - # azure-core - # psycopg - # psycopg-pool - # pydantic - # pygeoif - # redis -tzlocal==5.0.1 - # via dateparser -urllib3==1.26.16 - # via - # google-auth - # requests -uvicorn[standard]==0.17.6 - # via pcstac (pcstac/setup.py) -uvloop==0.17.0 - # via uvicorn -version-parser==1.0.1 - # via pypgstac -watchgod==0.8.2 - # via uvicorn -websockets==11.0.3 - # via uvicorn -wrapt==1.15.0 - # via deprecated diff --git a/pcstac/setup.py b/pcstac/setup.py index 3e04d0ea..0ad71ca0 100644 --- a/pcstac/setup.py +++ b/pcstac/setup.py @@ -8,6 +8,7 @@ "stac-fastapi.extensions==2.4.8", "stac-fastapi.pgstac==2.4.9", "stac-fastapi.types==2.4.8", + "orjson>=3.9.15", # Required due to some imports related to pypgstac CLI usage in startup script "pypgstac[psycopg]~=0.7", "pystac>=1.9", diff --git a/pctiler/Dockerfile b/pctiler/Dockerfile index 428a7587..caca68f5 100644 --- a/pctiler/Dockerfile +++ b/pctiler/Dockerfile @@ -1,29 +1,4 @@ -FROM ubuntu:latest as build - -# Install Micromamba -RUN apt-get update && apt-get install -y curl bzip2 -RUN curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba - -COPY pctiler/environment.yaml /tmp/env.yaml - -# Create a separate environment for the packages and remove unnecessary files -RUN micromamba create -p /opt/conda/envs/myenv -f /tmp/env.yaml && \ - micromamba clean --all --yes && \ - rm -rf /root/.cache /tmp/env.yaml - -# Start from a new stage to avoid copying the build environment, and -# copy the environment from the build stage -# Remove unnecessary files -RUN find /opt/conda/envs/myenv -type f -name "*.pyc" -exec rm -f {} \; -RUN rm -rf /opt/conda/envs/myenv/share/doc -RUN rm -rf /opt/conda/envs/myenv/share/locale -RUN rm -rf /opt/conda/envs/myenv/man -RUN rm -rf /opt/conda/envs/myenv/include - -FROM ubuntu:latest - -COPY --from=build /opt/conda/envs/myenv /opt/conda/envs/myenv -ENV PATH="/opt/conda/envs/myenv/bin:$PATH" +FROM python:3.9-slim WORKDIR /opt/src @@ -31,7 +6,15 @@ COPY pccommon /opt/src/pccommon COPY pctiler /opt/src/pctiler # Install the local modules in the new environment -RUN /bin/sh -c "python -m pip install --no-cache-dir -e ./pccommon -e ./pctiler[server]" +RUN --mount=type=cache,target=/root/.cache \ + /bin/sh -c "python -m pip install -U 'setuptools>=65.5.1'" +# The order of these pip installs is important :( +RUN --mount=type=cache,target=/root/.cache \ + /bin/sh -c "python -m pip install -r ./pccommon/requirements.txt" +RUN --mount=type=cache,target=/root/.cache \ + /bin/sh -c "python -m pip install -r ./pctiler/requirements-server.txt" +RUN --mount=type=cache,target=/root/.cache \ + /bin/sh -c "python -m pip install --no-deps -e ./pccommon -e ./pctiler[server]" # GDAL config ENV GDAL_CACHEMAX 200 diff --git a/pctiler/Dockerfile.dev b/pctiler/Dockerfile.dev index d0bba244..12f95fbe 100644 --- a/pctiler/Dockerfile.dev +++ b/pctiler/Dockerfile.dev @@ -4,5 +4,9 @@ RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash COPY requirements-dev.txt requirements-dev.txt -RUN python3 -m pip install -r requirements-dev.txt -RUN python3 -m pip install -e ./pccommon -e ./pctiler[dev] +RUN --mount=type=cache,target=/root/.cache \ + --mount=type=bind,source=requirements-dev.txt,target=requirements-dev.txt \ + python3 -m pip install -r requirements-dev.txt + +RUN --mount=type=cache,target=/root/.cache \ + python3 -m pip install -r pctiler/requirements-dev.txt diff --git a/pctiler/environment.yaml b/pctiler/environment.yaml deleted file mode 100644 index 8684e678..00000000 --- a/pctiler/environment.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: base -channels: - - conda-forge -dependencies: - - python=3.9.16 - - pystac==1.7.1 - - jinja2==3.0.3 - - rasterio==1.3.6 - - pyproj==3.4.1 - - planetary-computer==0.4.9 - - geojson-pydantic==0.4.2 - # colormap dependencies - - matplotlib-base==3.4.3 - - pip - - pip: - - "titiler.core==0.10.2" - - "titiler.mosaic==0.10.2" - - - # titiler-pgstac - - "psycopg[binary,pool]" - - "titiler.pgstac==0.2.4" diff --git a/pctiler/requirements-dev.txt b/pctiler/requirements-dev.txt new file mode 100644 index 00000000..e9c9e154 --- /dev/null +++ b/pctiler/requirements-dev.txt @@ -0,0 +1,239 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --extra=dev --output-file=pctiler/requirements-dev.txt ./pctiler/setup.py +# +affine==2.4.0 + # via + # rasterio + # supermercado +anyio==4.3.0 + # via + # httpx + # starlette +attrs==23.2.0 + # via + # cogeo-mosaic + # morecantile + # rasterio + # rio-tiler +boto3==1.34.71 + # via + # pctiler (pctiler/setup.py) + # rio-tiler +botocore==1.34.71 + # via + # boto3 + # pctiler (pctiler/setup.py) + # s3transfer +cachetools==5.3.3 + # via + # cogeo-mosaic + # morecantile + # rio-tiler +certifi==2024.2.2 + # via + # httpcore + # httpx + # pyproj + # rasterio + # requests +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via + # click-plugins + # cligj + # mercantile + # planetary-computer + # rasterio + # stac-pydantic + # supermercado +click-plugins==1.1.1 + # via + # rasterio + # supermercado +cligj==0.7.2 + # via + # rasterio + # supermercado +cogeo-mosaic==5.0.0 + # via titiler-mosaic +color-operations==0.1.3 + # via rio-tiler +cycler==0.12.1 + # via matplotlib +exceptiongroup==1.2.0 + # via anyio +fastapi==0.91.0 + # via + # titiler-core + # titiler-pgstac +geojson-pydantic==0.4.2 + # via + # pctiler (pctiler/setup.py) + # stac-pydantic + # titiler-core + # titiler-pgstac +h11==0.14.0 + # via httpcore +httpcore==1.0.5 + # via httpx +httpx==0.27.0 + # via + # cogeo-mosaic + # rio-tiler +idna==3.6 + # via + # anyio + # httpx + # requests +jinja2==3.0.3 + # via + # pctiler (pctiler/setup.py) + # titiler-core +jmespath==1.0.1 + # via + # boto3 + # botocore +kiwisolver==1.4.5 + # via matplotlib +markupsafe==2.1.5 + # via jinja2 +matplotlib==3.4.3 + # via pctiler (pctiler/setup.py) +mercantile==1.2.1 + # via supermercado +morecantile==3.4.0 + # via + # cogeo-mosaic + # rio-tiler +numexpr==2.9.0 + # via rio-tiler +numpy==1.26.4 + # via + # color-operations + # matplotlib + # numexpr + # rasterio + # rio-tiler + # shapely + # snuggs + # supermercado + # titiler-core +orjson==3.9.15 + # via pctiler (pctiler/setup.py) +pillow==10.2.0 + # via + # matplotlib + # pctiler (pctiler/setup.py) +planetary-computer==0.4.9 + # via pctiler (pctiler/setup.py) +psycopg[binary,pool]==3.1.18 + # via pctiler (pctiler/setup.py) +psycopg-binary==3.1.18 + # via psycopg +psycopg-pool==3.2.1 + # via psycopg +pydantic[dotenv]==1.10.14 + # via + # cogeo-mosaic + # fastapi + # geojson-pydantic + # morecantile + # pctiler (pctiler/setup.py) + # planetary-computer + # rio-tiler + # stac-pydantic + # titiler-core +pyparsing==3.1.2 + # via + # matplotlib + # snuggs +pyproj==3.6.1 + # via morecantile +pystac==1.7.1 + # via + # pctiler (pctiler/setup.py) + # planetary-computer + # pystac-client + # rio-tiler +pystac-client==0.6.1 + # via planetary-computer +python-dateutil==2.9.0.post0 + # via + # botocore + # matplotlib + # pystac + # pystac-client +python-dotenv==1.0.1 + # via pydantic +pytz==2024.1 + # via planetary-computer +rasterio==1.3.9 + # via + # cogeo-mosaic + # pctiler (pctiler/setup.py) + # rio-tiler + # supermercado + # titiler-core +requests==2.31.0 + # via + # planetary-computer + # pystac-client +rio-tiler==4.1.13 + # via + # cogeo-mosaic + # titiler-core +s3transfer==0.10.1 + # via boto3 +shapely==2.0.3 + # via cogeo-mosaic +simplejson==3.19.2 + # via titiler-core +six==1.16.0 + # via python-dateutil +sniffio==1.3.1 + # via + # anyio + # httpx +snuggs==1.4.7 + # via rasterio +stac-pydantic==2.0.3 + # via titiler-pgstac +starlette==0.24.0 + # via + # fastapi + # titiler-pgstac +supermercado==0.2.0 + # via cogeo-mosaic +titiler-core==0.10.2 + # via + # pctiler (pctiler/setup.py) + # titiler-mosaic + # titiler-pgstac +titiler-mosaic==0.10.2 + # via + # pctiler (pctiler/setup.py) + # titiler-pgstac +titiler-pgstac==0.2.4 + # via pctiler (pctiler/setup.py) +types-requests==2.31.0.6 + # via pctiler (pctiler/setup.py) +types-urllib3==1.26.25.14 + # via types-requests +typing-extensions==4.10.0 + # via + # anyio + # psycopg + # psycopg-pool + # pydantic + # starlette +urllib3==1.26.18 + # via + # botocore + # requests + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/pctiler/requirements-server.txt b/pctiler/requirements-server.txt new file mode 100644 index 00000000..e93d6da3 --- /dev/null +++ b/pctiler/requirements-server.txt @@ -0,0 +1,255 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --extra=server --output-file=pctiler/requirements-server.txt ./pctiler/setup.py +# +affine==2.4.0 + # via + # rasterio + # supermercado +anyio==3.7.1 + # via + # httpx + # starlette + # watchgod +asgiref==3.8.1 + # via uvicorn +attrs==23.2.0 + # via + # cogeo-mosaic + # morecantile + # rasterio + # rio-tiler +boto3==1.34.71 + # via + # pctiler (pctiler/setup.py) + # rio-tiler +botocore==1.34.71 + # via + # boto3 + # pctiler (pctiler/setup.py) + # s3transfer +cachetools==5.3.3 + # via + # cogeo-mosaic + # morecantile + # rio-tiler +certifi==2024.2.2 + # via + # httpcore + # httpx + # pyproj + # rasterio + # requests +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via + # click-plugins + # cligj + # mercantile + # planetary-computer + # rasterio + # stac-pydantic + # supermercado + # uvicorn +click-plugins==1.1.1 + # via + # rasterio + # supermercado +cligj==0.7.2 + # via + # rasterio + # supermercado +cogeo-mosaic==5.0.0 + # via titiler-mosaic +color-operations==0.1.3 + # via rio-tiler +cycler==0.12.1 + # via matplotlib +exceptiongroup==1.2.0 + # via anyio +fastapi==0.91.0 + # via + # titiler-core + # titiler-pgstac +geojson-pydantic==0.4.2 + # via + # pctiler (pctiler/setup.py) + # stac-pydantic + # titiler-core + # titiler-pgstac +h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==1.0.4 + # via httpx +httptools==0.6.1 + # via uvicorn +httpx==0.27.0 + # via + # cogeo-mosaic + # rio-tiler +idna==3.6 + # via + # anyio + # httpx + # requests +jinja2==3.0.3 + # via + # pctiler (pctiler/setup.py) + # titiler-core +jmespath==1.0.1 + # via + # boto3 + # botocore +kiwisolver==1.4.5 + # via matplotlib +markupsafe==2.1.5 + # via jinja2 +matplotlib==3.4.3 + # via pctiler (pctiler/setup.py) +mercantile==1.2.1 + # via supermercado +morecantile==3.4.0 + # via + # cogeo-mosaic + # rio-tiler +numexpr==2.9.0 + # via rio-tiler +numpy==1.26.4 + # via + # color-operations + # matplotlib + # numexpr + # rasterio + # rio-tiler + # shapely + # snuggs + # supermercado + # titiler-core +orjson==3.9.15 + # via pctiler (pctiler/setup.py) +pillow==10.2.0 + # via + # matplotlib + # pctiler (pctiler/setup.py) +planetary-computer==0.4.9 + # via pctiler (pctiler/setup.py) +psycopg[binary,pool]==3.1.18 + # via pctiler (pctiler/setup.py) +psycopg-binary==3.1.18 + # via psycopg +psycopg-pool==3.2.1 + # via psycopg +pydantic[dotenv]==1.10.14 + # via + # cogeo-mosaic + # fastapi + # geojson-pydantic + # morecantile + # pctiler (pctiler/setup.py) + # planetary-computer + # rio-tiler + # stac-pydantic + # titiler-core +pyparsing==3.1.2 + # via + # matplotlib + # snuggs +pyproj==3.6.1 + # via morecantile +pystac==1.7.1 + # via + # pctiler (pctiler/setup.py) + # planetary-computer + # pystac-client + # rio-tiler +pystac-client==0.6.1 + # via planetary-computer +python-dateutil==2.9.0.post0 + # via + # botocore + # matplotlib + # pystac + # pystac-client +python-dotenv==1.0.1 + # via + # pydantic + # uvicorn +pytz==2024.1 + # via planetary-computer +pyyaml==6.0.1 + # via uvicorn +rasterio==1.3.9 + # via + # cogeo-mosaic + # pctiler (pctiler/setup.py) + # rio-tiler + # supermercado + # titiler-core +requests==2.31.0 + # via + # planetary-computer + # pystac-client +rio-tiler==4.1.13 + # via + # cogeo-mosaic + # titiler-core +s3transfer==0.10.1 + # via boto3 +shapely==2.0.3 + # via cogeo-mosaic +simplejson==3.19.2 + # via titiler-core +six==1.16.0 + # via python-dateutil +sniffio==1.3.1 + # via + # anyio + # httpx +snuggs==1.4.7 + # via rasterio +stac-pydantic==2.0.3 + # via titiler-pgstac +starlette==0.24.0 + # via + # fastapi + # titiler-pgstac +supermercado==0.2.0 + # via cogeo-mosaic +titiler-core==0.10.2 + # via + # pctiler (pctiler/setup.py) + # titiler-mosaic + # titiler-pgstac +titiler-mosaic==0.10.2 + # via + # pctiler (pctiler/setup.py) + # titiler-pgstac +titiler-pgstac==0.2.4 + # via pctiler (pctiler/setup.py) +typing-extensions==4.10.0 + # via + # asgiref + # psycopg + # psycopg-pool + # pydantic + # starlette +urllib3==1.26.18 + # via + # botocore + # requests +uvicorn[standard]==0.17.6 + # via pctiler (pctiler/setup.py) +uvloop==0.19.0 + # via uvicorn +watchgod==0.8.2 + # via uvicorn +websockets==12.0 + # via uvicorn + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/pctiler/setup.py b/pctiler/setup.py index 7fe371a8..0b2f3e1c 100644 --- a/pctiler/setup.py +++ b/pctiler/setup.py @@ -4,7 +4,28 @@ from setuptools import find_packages, setup # Runtime requirements, see environment.yaml -inst_reqs: List[str] = [] +inst_reqs: List[str] = [ + "geojson-pydantic==0.4.2", + "jinja2==3.0.3", + "pystac==1.7.1", + "planetary-computer==0.4.9", + "rasterio==1.3.9", + "titiler.core==0.10.2", + "titiler.mosaic==0.10.2", + "pillow==10.2.0", + "boto3==1.34.71", + "botocore==1.34.71", + "pydantic==1.10.14", + # titiler-pgstac + "psycopg[binary,pool]", + "titiler.pgstac==0.2.4", + + # colormap dependencies + "matplotlib==3.4.3", + + "orjson==3.9.15", + "importlib_resources>=1.1.0;python_version<'3.9'", +] extra_reqs = { "dev": ["types-requests"], diff --git a/pctiler/tests/conftest.py b/pctiler/tests/conftest.py index be96f7a2..95d50231 100644 --- a/pctiler/tests/conftest.py +++ b/pctiler/tests/conftest.py @@ -2,10 +2,10 @@ import pytest from httpx import AsyncClient -from pytest import Config, Item +from pytest import Config, Item, Parser -def pytest_addoption(parser) -> None: +def pytest_addoption(parser: Parser) -> None: parser.addoption( "--no-integration", action="store_true", diff --git a/pctiler/tests/test_asset_read.py b/pctiler/tests/test_asset_read.py new file mode 100644 index 00000000..67baaa03 --- /dev/null +++ b/pctiler/tests/test_asset_read.py @@ -0,0 +1,20 @@ +import planetary_computer as pc +import rasterio + + +def test_rasterio_lerc_decompression() -> None: + # Read a LERC file from Planetary Computer + url = ( + "https://usgslidareuwest.blob.core.windows.net/" + "usgs-3dep-cogs/usgs-cogs/" + "USGS_LPC_VA_Fairfax_County_2018/hag/" + "USGS_LPC_VA_Fairfax_County_2018-hag-2m-3-1.tif" + ) + signed_url = pc.sign(url) + # Although we call rasterio, this is really testing GDAL's LERC support. + # If the version of GDAL being shipped doesn't support LERC, then an exception like + # "rasterio.errors.RasterioIOError: Cannot open TIFF file due to missing codec."" + # Would be thrown. + with rasterio.open(signed_url) as src: + compression = src.profile.get("compress") + assert compression == "lerc_zstd" diff --git a/requirements-dev.txt b/requirements-dev.txt index 2c8fd003..d165d367 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,7 @@ cachetools<=4.2. pytest==7.* pytest-asyncio==0.18.* httpx>=0.22.0 - +pip-tools # Mypy types types-python-dateutil types-cachetools diff --git a/scripts/generate-requirements b/scripts/generate-requirements new file mode 100755 index 00000000..6110291f --- /dev/null +++ b/scripts/generate-requirements @@ -0,0 +1,38 @@ +#!/bin/bash + +set -e + +if [[ "${CI}" ]]; then + set -x +fi + +echo "== Generating requirements.txt ..." +pip_compile_options="--emit-index-url --annotate --verbose" + +docker-compose \ + -f docker-compose.yml \ + -f docker-compose.dev.yml \ + run --rm \ + stac-dev \ + pip-compile ./pcstac/setup.py --extra server -o pcstac/requirements-server.txt $pip_compile_options + +docker-compose \ + -f docker-compose.yml \ + -f docker-compose.dev.yml \ + run --rm \ + tiler-dev \ + pip-compile ./pctiler/setup.py --extra server -o pctiler/requirements-server.txt $pip_compile_options + +docker-compose \ + -f docker-compose.yml \ + -f docker-compose.dev.yml \ + run --rm \ + tiler-dev \ + pip-compile ./pctiler/setup.py --extra dev -o pctiler/requirements-dev.txt $pip_compile_options + +docker-compose \ + -f docker-compose.yml \ + -f docker-compose.dev.yml \ + run --rm \ + tiler-dev \ + pip-compile ./pccommon/setup.py --extra server -o pccommon/requirements.txt $pip_compile_options \ No newline at end of file diff --git a/scripts/validate b/scripts/validate index 57e0cd94..3315af6e 100755 --- a/scripts/validate +++ b/scripts/validate @@ -17,7 +17,7 @@ geometry='{"type":"Polygon","coordinates":[[[-85.3125,30.9375],[-85.3125,31],[-8 if [ "${BASH_SOURCE[0]}" = "${0}" ]; then scripts/setup - which stac-api-validator >/dev/null || pip install stac-api-validator==0.4.1 + which stac-api-validator >/dev/null || pip install stac-api-validator==0.6.1 stac-api-validator --root-url http://localhost:8080/stac/ \ --conformance collections \ --conformance core \