Security patching (#191)

* Generate requirments.txt

* remove hashes

* Delete non-extra requirements file

* Now all projects have requirements.txt

* Remove BaseHTTPMiddlewares ,  Ensure origin host is used in STAC links

* Successfully build containers

* lint

* Docker images build

* Delete unused requirements file

* save

* Now requirements do get generated

* Update Azurite

* format setup.py

* add type annotation

* Read lerc_zstd

* speed up image rebuilds and better UT for lerc

* Update more packages

* upgrade stac-api-validation

* Remove files from older commits

---------

Co-authored-by: maitjoshi <[email protected]>

README.md

@@ -75,6 +75,11 @@ After migrations and development database loading are in place, you can rebuild
 > ./scripts/update
 ```
 
+`pip` dependencies in `setup.py` are collected and installed through requirements files.
+If you modify dependencies, run `./scripts/generate-requirements` to regenerate
+`requirements-*.txt` used by Dockerfiles otherwise your dependency change will not
+be realized.
+
 #### Running the services
 
 There is a local proxy service that facilitates a local "managed identity" functionality, run as your local identity. Make sure to run

docker-compose.yml

@@ -101,7 +101,7 @@ services:
 
   azurite:
     container_name: pcapis-azurite
-    image: mcr.microsoft.com/azure-storage/azurite:3.17.1
+    image: mcr.microsoft.com/azure-storage/azurite:3.29.0
     hostname: azurite
     command: "azurite --silent --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost
       0.0.0.0 -l /workspace"

pccommon/requirements.txt

@@ -0,0 +1,165 @@
+#
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
+#
+#    pip-compile --extra=server --output-file=pccommon/requirements.txt ./pccommon/setup.py
+#
+anyio==4.3.0
+    # via starlette
+async-timeout==4.0.3
+    # via redis
+azure-core==1.30.1
+    # via
+    #   azure-data-tables
+    #   azure-identity
+    #   azure-storage-blob
+    #   msrest
+azure-data-tables==12.4.0
+    # via pccommon (pccommon/setup.py)
+azure-identity==1.7.1
+    # via pccommon (pccommon/setup.py)
+azure-storage-blob==12.19.1
+    # via pccommon (pccommon/setup.py)
+beautifulsoup4==4.12.3
+    # via html-sanitizer
+cachetools==5.3.3
+    # via
+    #   google-auth
+    #   pccommon (pccommon/setup.py)
+certifi==2024.2.2
+    # via
+    #   msrest
+    #   requests
+cffi==1.16.0
+    # via cryptography
+charset-normalizer==3.3.2
+    # via requests
+cryptography==42.0.5
+    # via
+    #   azure-identity
+    #   azure-storage-blob
+    #   msal
+    #   pyjwt
+deprecated==1.2.14
+    # via redis
+exceptiongroup==1.2.0
+    # via anyio
+fastapi==0.90.1
+    # via pccommon (pccommon/setup.py)
+google-api-core==2.18.0
+    # via opencensus
+google-auth==2.29.0
+    # via google-api-core
+googleapis-common-protos==1.63.0
+    # via google-api-core
+html-sanitizer==2.4.0
+    # via pccommon (pccommon/setup.py)
+idna==3.6
+    # via
+    #   anyio
+    #   requests
+isodate==0.6.1
+    # via
+    #   azure-storage-blob
+    #   msrest
+lxml[html-clean]==5.2.1
+    # via
+    #   html-sanitizer
+    #   lxml-html-clean
+lxml-html-clean==0.1.0
+    # via pccommon (pccommon/setup.py)
+msal==1.28.0
+    # via
+    #   azure-identity
+    #   msal-extensions
+msal-extensions==0.3.1
+    # via azure-identity
+msrest==0.7.1
+    # via azure-data-tables
+oauthlib==3.2.2
+    # via requests-oauthlib
+opencensus==0.11.4
+    # via
+    #   opencensus-ext-azure
+    #   opencensus-ext-logging
+opencensus-context==0.1.3
+    # via opencensus
+opencensus-ext-azure==1.0.8
+    # via pccommon (pccommon/setup.py)
+opencensus-ext-logging==0.1.0
+    # via pccommon (pccommon/setup.py)
+orjson==3.9.15
+    # via pccommon (pccommon/setup.py)
+packaging==24.0
+    # via redis
+portalocker==2.8.2
+    # via msal-extensions
+proto-plus==1.23.0
+    # via google-api-core
+protobuf==4.25.3
+    # via
+    #   google-api-core
+    #   googleapis-common-protos
+    #   proto-plus
+psutil==5.9.8
+    # via opencensus-ext-azure
+pyasn1==0.5.1
+    # via
+    #   pyasn1-modules
+    #   rsa
+pyasn1-modules==0.3.0
+    # via google-auth
+pycparser==2.21
+    # via cffi
+pydantic==1.10.14
+    # via
+    #   fastapi
+    #   pccommon (pccommon/setup.py)
+pyhumps==3.5.3
+    # via pccommon (pccommon/setup.py)
+pyjwt[crypto]==2.8.0
+    # via msal
+redis==4.2.0rc1
+    # via pccommon (pccommon/setup.py)
+requests==2.31.0
+    # via
+    #   azure-core
+    #   google-api-core
+    #   msal
+    #   msrest
+    #   opencensus-ext-azure
+    #   requests-oauthlib
+requests-oauthlib==2.0.0
+    # via msrest
+rsa==4.9
+    # via google-auth
+six==1.16.0
+    # via
+    #   azure-core
+    #   azure-identity
+    #   isodate
+    #   opencensus
+sniffio==1.3.1
+    # via anyio
+soupsieve==2.5
+    # via beautifulsoup4
+starlette==0.22.0
+    # via
+    #   fastapi
+    #   pccommon (pccommon/setup.py)
+types-cachetools==4.2.9
+    # via pccommon (pccommon/setup.py)
+typing-extensions==4.10.0
+    # via
+    #   anyio
+    #   azure-core
+    #   azure-storage-blob
+    #   pydantic
+    #   redis
+    #   starlette
+urllib3==2.2.1
+    # via
+    #   pccommon (pccommon/setup.py)
+    #   requests
+wrapt==1.16.0
+    # via deprecated

pccommon/setup.py

@@ -8,10 +8,10 @@
     "starlette>=0.22.0,<0.23.0",
     "opencensus-ext-azure==1.0.8",
     "opencensus-ext-logging==0.1.0",
-    "orjson==3.5.2",
+    "orjson>=3.9.15",
     "azure-identity==1.7.1",
     "azure-data-tables==12.4.0",
-    "azure-storage-blob==12.12.0",
+    "azure-storage-blob>=12.19.1",
     "pydantic>=1.9, <2.0.0",
     "cachetools~=5.3",
     "types-cachetools==4.2.9",
@@ -20,6 +20,7 @@
     "html-sanitizer==2.4",
     # Soon available as lxml[html_clean]
     "lxml_html_clean==0.1.0",
+    "urllib3>=1.26.18",
 ]
 
 extra_reqs = {

pcfuncs/Dockerfile

@@ -11,10 +11,12 @@ COPY pccommon /home/site/pccommon
 RUN mkdir -p /home/site/wwwroot
 WORKDIR /home/site/wwwroot
 
-COPY pcfuncs/requirements.txt /home/site/wwwroot/requirements.txt
-RUN pip install -r /home/site/wwwroot/requirements.txt
+RUN --mount=type=cache,target=/root/.cache \
+    --mount=type=bind,source=pcfuncs/requirements.txt,target=/home/site/wwwroot/requirements.txt \
+    pip install -r /home/site/wwwroot/requirements.txt
 
-COPY requirements-dev.txt /home/site/
-RUN pip install -r /home/site/requirements-dev.txt
+RUN --mount=type=cache,target=/root/.cache \
+    --mount=type=bind,source=requirements-dev.txt,target=/home/site/requirements-dev.txt \
+    pip install -r /home/site/requirements-dev.txt
 
 ENV PYTHONPATH=/home/site/wwwroot

pcfuncs/requirements-deploy.txt

@@ -6,7 +6,7 @@
 
 azure-functions
 
-requests==2.28.1
+requests==2.31.0
 aiohttp==3.8.1
 dateutils==0.6.12
 mercantile==1.2.1

pcfuncs/requirements.txt

@@ -6,7 +6,7 @@
 
 azure-functions
 
-requests==2.28.1
+requests==2.31.0
 aiohttp==3.8.1
 dateutils==0.6.12
 mercantile==1.2.1

pcstac/Dockerfile

@@ -9,7 +9,15 @@ WORKDIR /opt/src
 
 COPY pcstac /opt/src/pcstac
 COPY pccommon /opt/src/pccommon
-RUN pip install -r ./pcstac/requirements.txt -e ./pccommon -e ./pcstac[server]
+RUN --mount=type=cache,target=/root/.cache \
+    pip install -U "setuptools>=65.5.1"
+# The order of these pip installs is important :(
+RUN --mount=type=cache,target=/root/.cache \
+    pip install -r ./pccommon/requirements.txt
+RUN --mount=type=cache,target=/root/.cache \
+    pip install -r ./pcstac/requirements-server.txt
+RUN --mount=type=cache,target=/root/.cache \
+    pip install --no-deps -e ./pccommon -e ./pcstac[server]
 
 ENV APP_HOST=0.0.0.0
 ENV APP_PORT=81

pcstac/Dockerfile.dev

@@ -1,6 +1,9 @@
 FROM pc-apis-stac
 
-COPY requirements-dev.txt requirements-dev.txt
-RUN pip install -r requirements-dev.txt
 
-RUN pip install -e ./pccommon[dev] -e ./pcstac
+RUN --mount=type=cache,target=/root/.cache \
+    --mount=type=bind,source=requirements-dev.txt,target=requirements-dev.txt \
+    pip install -r requirements-dev.txt
+
+RUN --mount=type=cache,target=/root/.cache \
+    pip install -e ./pccommon[dev] -e ./pcstac