Skip to content

Commit

Permalink
speed up image rebuilds and better UT for lerc
Browse files Browse the repository at this point in the history
  • Loading branch information
ghidalgo3 committed Mar 28, 2024
1 parent dc260b5 commit 4909cd8
Show file tree
Hide file tree
Showing 5 changed files with 41 additions and 17 deletions.
12 changes: 8 additions & 4 deletions pcstac/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,15 @@ WORKDIR /opt/src

COPY pcstac /opt/src/pcstac
COPY pccommon /opt/src/pccommon
RUN pip install -U "setuptools>=65.5.1"
RUN --mount=type=cache,target=/root/.cache \
pip install -U "setuptools>=65.5.1"
# The order of these pip installs is important :(
RUN pip install -r ./pccommon/requirements.txt
RUN pip install -r ./pcstac/requirements-server.txt
RUN pip install --no-deps -e ./pccommon -e ./pcstac[server]
RUN --mount=type=cache,target=/root/.cache \
pip install -r ./pccommon/requirements.txt
RUN --mount=type=cache,target=/root/.cache \
pip install -r ./pcstac/requirements-server.txt
RUN --mount=type=cache,target=/root/.cache \
pip install --no-deps -e ./pccommon -e ./pcstac[server]

ENV APP_HOST=0.0.0.0
ENV APP_PORT=81
Expand Down
9 changes: 6 additions & 3 deletions pcstac/Dockerfile.dev
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
FROM pc-apis-stac

COPY requirements-dev.txt requirements-dev.txt
RUN pip install -r requirements-dev.txt

RUN pip install -e ./pccommon[dev] -e ./pcstac
RUN --mount=type=cache,target=/root/.cache \
--mount=type=bind,source=requirements-dev.txt,target=requirements-dev.txt \
pip install -r requirements-dev.txt

RUN --mount=type=cache,target=/root/.cache \
pip install -e ./pccommon[dev] -e ./pcstac
12 changes: 8 additions & 4 deletions pctiler/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,15 @@ COPY pccommon /opt/src/pccommon
COPY pctiler /opt/src/pctiler

# Install the local modules in the new environment
RUN /bin/sh -c "python -m pip install -U 'setuptools>=65.5.1'"
RUN --mount=type=cache,target=/root/.cache \
/bin/sh -c "python -m pip install -U 'setuptools>=65.5.1'"
# The order of these pip installs is important :(
RUN /bin/sh -c "python -m pip install -r ./pccommon/requirements.txt"
RUN /bin/sh -c "python -m pip install -r ./pctiler/requirements-server.txt"
RUN /bin/sh -c "python -m pip install --no-deps -e ./pccommon -e ./pctiler[server]"
RUN --mount=type=cache,target=/root/.cache \
/bin/sh -c "python -m pip install -r ./pccommon/requirements.txt"
RUN --mount=type=cache,target=/root/.cache \
/bin/sh -c "python -m pip install -r ./pctiler/requirements-server.txt"
RUN --mount=type=cache,target=/root/.cache \
/bin/sh -c "python -m pip install --no-deps -e ./pccommon -e ./pctiler[server]"

# GDAL config
ENV GDAL_CACHEMAX 200
Expand Down
8 changes: 6 additions & 2 deletions pctiler/Dockerfile.dev
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,9 @@ RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash

COPY requirements-dev.txt requirements-dev.txt

RUN python3 -m pip install -r requirements-dev.txt
RUN python3 -m pip install -r pctiler/requirements-dev.txt
RUN --mount=type=cache,target=/root/.cache \
--mount=type=bind,source=requirements-dev.txt,target=requirements-dev.txt \
python3 -m pip install -r requirements-dev.txt

RUN --mount=type=cache,target=/root/.cache \
python3 -m pip install -r pctiler/requirements-dev.txt
17 changes: 13 additions & 4 deletions pctiler/tests/test_asset_read.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,20 @@
import planetary_computer as pc
import rasterio
from titiler.core import utils as titiler_utils


def test_rasterio_lerc_decompression() -> None:
# Read a LERC file from Planetary Computer
url = "https://usgslidareuwest.blob.core.windows.net/usgs-3dep-cogs/usgs-cogs/USGS_LPC_VA_Fairfax_County_2018/hag/USGS_LPC_VA_Fairfax_County_2018-hag-2m-3-1.tif"
url = (
"https://usgslidareuwest.blob.core.windows.net/"
"usgs-3dep-cogs/usgs-cogs/"
"USGS_LPC_VA_Fairfax_County_2018/hag/"
"USGS_LPC_VA_Fairfax_County_2018-hag-2m-3-1.tif"
)
signed_url = pc.sign(url)
# Although we call rasterio, this is really testing GDAL's LERC support.
# If the version of GDAL being shipped doesn't support LERC, then an exception like
# "rasterio.errors.RasterioIOError: Cannot open TIFF file due to missing codec.""
# Would be thrown.
with rasterio.open(signed_url) as src:
compression = src.profile.get('compress')
assert compression == 'lerc_zstd'
compression = src.profile.get("compress")
assert compression == "lerc_zstd"

0 comments on commit 4909cd8

Please sign in to comment.