diff --git a/deployment/terraform/resources/storage_account.tf b/deployment/terraform/resources/storage_account.tf index 1ad494b0..a3033d1d 100644 --- a/deployment/terraform/resources/storage_account.tf +++ b/deployment/terraform/resources/storage_account.tf @@ -10,6 +10,11 @@ resource "azurerm_storage_account" "pc" { network_rules { default_action = "Deny" virtual_network_subnet_ids = [azurerm_subnet.node_subnet.id, azurerm_subnet.function_subnet.id, data.azurerm_subnet.sas_node_subnet.id] + + private_link_access { + endpoint_resource_id = "/subscriptions/a84a690d-585b-4c7c-80d9-851a48af5a50/providers/Microsoft.Security/datascanners/storageDataScanner" + endpoint_tenant_id = "72f988bf-86f1-41af-91ab-2d7cd011db47" + } } # Disabling shared access keys breaks terraform's ability to do subsequent @@ -60,4 +65,9 @@ resource "azurerm_storage_account_network_rules" "pcfunc-vnet-access" { default_action = "Deny" virtual_network_subnet_ids = [azurerm_subnet.function_subnet.id] + + private_link_access { + endpoint_resource_id = "/subscriptions/a84a690d-585b-4c7c-80d9-851a48af5a50/providers/Microsoft.Security/datascanners/storageDataScanner" + endpoint_tenant_id = "72f988bf-86f1-41af-91ab-2d7cd011db47" + } } diff --git a/pccommon/pccommon/config/collections.py b/pccommon/pccommon/config/collections.py index bcbc145b..a627f336 100644 --- a/pccommon/pccommon/config/collections.py +++ b/pccommon/pccommon/config/collections.py @@ -24,6 +24,7 @@ class CamelModel(BaseModel): # https://docs.pydantic.dev/latest/api/config/#pydantic.alias_generators.to_camel "alias_generator": camelize, "populate_by_name": True, + "use_enum_values": True, } @@ -267,7 +268,9 @@ class RenderOptions(CamelModel): name: str description: Optional[str] = None - type: Optional[RenderOptionType] = Field(default=RenderOptionType.raster_tile) + type: Optional[RenderOptionType] = Field( + default=RenderOptionType.raster_tile, validate_default=True + ) options: Optional[str] vector_options: Optional[VectorTileOptions] = None min_zoom: int diff --git a/pctiler/pctiler/endpoints/dependencies.py b/pctiler/pctiler/endpoints/dependencies.py new file mode 100644 index 00000000..bfd4559b --- /dev/null +++ b/pctiler/pctiler/endpoints/dependencies.py @@ -0,0 +1,21 @@ +import logging +from typing import Callable + +import fastapi +import starlette + +logger = logging.getLogger(__name__) + + +def get_endpoint_function( + router: fastapi.APIRouter, path: str, method: str +) -> Callable: + for route in router.routes: + match, _ = route.matches({"type": "http", "path": path, "method": method}) + if match == starlette.routing.Match.FULL: + # The abstract BaseRoute doesn't have a `.endpoint` attribute, + # but all of its subclasses do. + return route.endpoint # type: ignore [attr-defined] + + logger.warning(f"Could not find endpoint. method={method} path={path}") + raise fastapi.HTTPException(detail="Internal system error", status_code=500) diff --git a/pctiler/pctiler/endpoints/item.py b/pctiler/pctiler/endpoints/item.py index e82a09fc..187c75b3 100644 --- a/pctiler/pctiler/endpoints/item.py +++ b/pctiler/pctiler/endpoints/item.py @@ -1,11 +1,10 @@ import logging -from typing import Annotated, Callable, Optional +from typing import Annotated, Optional from urllib.parse import quote_plus, urljoin import fastapi import pystac -import starlette -from fastapi import Body, Depends, HTTPException, Query, Request, Response +from fastapi import Body, Depends, Query, Request, Response from fastapi.templating import Jinja2Templates from geojson_pydantic.features import Feature from html_sanitizer.sanitizer import Sanitizer @@ -18,6 +17,7 @@ from pccommon.config import get_render_config from pctiler.colormaps import PCColorMapParams from pctiler.config import get_settings +from pctiler.endpoints.dependencies import get_endpoint_function from pctiler.reader import ItemSTACReader, ReaderParams try: @@ -158,17 +158,3 @@ def geojson_crop( # type: ignore env=env, ) return result - - -def get_endpoint_function( - router: fastapi.APIRouter, path: str, method: str -) -> Callable: - for route in router.routes: - match, _ = route.matches({"type": "http", "path": path, "method": method}) - if match == starlette.routing.Match.FULL: - # The abstract BaseRoute doesn't have a `.endpoint` attribute, - # but all of its subclasses do. - return route.endpoint # type: ignore [attr-defined] - - logger.warning(f"Could not find endpoint. method={method} path={path}") - raise HTTPException(detail="Internal system error", status_code=500) diff --git a/pctiler/pctiler/endpoints/pg_mosaic.py b/pctiler/pctiler/endpoints/pg_mosaic.py index 4e602adc..3dd77839 100644 --- a/pctiler/pctiler/endpoints/pg_mosaic.py +++ b/pctiler/pctiler/endpoints/pg_mosaic.py @@ -1,17 +1,22 @@ from dataclasses import dataclass, field -from typing import List, Optional +from typing import Annotated, List, Literal, Optional -from fastapi import FastAPI, Query, Request +from fastapi import APIRouter, Depends, FastAPI, Query, Request, Response from fastapi.responses import ORJSONResponse from psycopg_pool import ConnectionPool +from pydantic import Field from titiler.core import dependencies -from titiler.pgstac.dependencies import SearchIdParams +from titiler.core.dependencies import ColorFormulaParams +from titiler.core.factory import img_endpoint_params +from titiler.core.resources.enums import ImageType +from titiler.pgstac.dependencies import SearchIdParams, TmsTileParams from titiler.pgstac.factory import MosaicTilerFactory from pccommon.config import get_collection_config from pccommon.config.collections import MosaicInfo from pctiler.colormaps import PCColorMapParams from pctiler.config import get_settings +from pctiler.endpoints.dependencies import get_endpoint_function from pctiler.reader import PGSTACBackend, ReaderParams @@ -75,3 +80,90 @@ def mosaic_info( by_alias=True, exclude_unset=True ), ) + + +legacy_mosaic_router = APIRouter() + + +@legacy_mosaic_router.get("/tiles/{search_id}/{z}/{x}/{y}", **img_endpoint_params) +@legacy_mosaic_router.get( + "/tiles/{search_id}/{z}/{x}/{y}.{format}", **img_endpoint_params +) +@legacy_mosaic_router.get( + "/tiles/{search_id}/{z}/{x}/{y}@{scale}x", **img_endpoint_params +) +@legacy_mosaic_router.get( + "/tiles/{search_id}/{z}/{x}/{y}@{scale}x.{format}", **img_endpoint_params +) +@legacy_mosaic_router.get( + "/tiles/{search_id}/{tileMatrixSetId}/{z}/{x}/{y}", **img_endpoint_params +) +@legacy_mosaic_router.get( + "/tiles/{search_id}/{tileMatrixSetId}/{z}/{x}/{y}.{format}", + **img_endpoint_params, +) +@legacy_mosaic_router.get( + "/tiles/{search_id}/{tileMatrixSetId}/{z}/{x}/{y}@{scale}x", + **img_endpoint_params, +) +@legacy_mosaic_router.get( + "/tiles/{search_id}/{tileMatrixSetId}/{z}/{x}/{y}@{scale}x.{format}", + **img_endpoint_params, +) +def tile_routes( # type: ignore + request: Request, + search_id=Depends(pgstac_mosaic_factory.path_dependency), + tile=Depends(TmsTileParams), + tileMatrixSetId: Annotated[ # type: ignore + Literal[tuple(pgstac_mosaic_factory.supported_tms.list())], + f"Identifier selecting one of the TileMatrixSetId supported (default: '{pgstac_mosaic_factory.default_tms}')", # noqa: E501,F722 + ] = pgstac_mosaic_factory.default_tms, + scale: Annotated[ # type: ignore + Optional[Annotated[int, Field(gt=0, le=4)]], + "Tile size scale. 1=256x256, 2=512x512...", # noqa: E501,F722 + ] = None, + format: Annotated[ + Optional[ImageType], + "Default will be automatically defined if the output image needs a mask (png) or not (jpeg).", # noqa: E501,F722 + ] = None, + layer_params=Depends(pgstac_mosaic_factory.layer_dependency), + dataset_params=Depends(pgstac_mosaic_factory.dataset_dependency), + pixel_selection=Depends(pgstac_mosaic_factory.pixel_selection_dependency), + tile_params=Depends(pgstac_mosaic_factory.tile_dependency), + post_process=Depends(pgstac_mosaic_factory.process_dependency), + rescale=Depends(pgstac_mosaic_factory.rescale_dependency), + color_formula=Depends(ColorFormulaParams), + colormap=Depends(pgstac_mosaic_factory.colormap_dependency), + render_params=Depends(pgstac_mosaic_factory.render_dependency), + pgstac_params=Depends(pgstac_mosaic_factory.pgstac_dependency), + backend_params=Depends(pgstac_mosaic_factory.backend_dependency), + reader_params=Depends(pgstac_mosaic_factory.reader_dependency), + env=Depends(pgstac_mosaic_factory.environment_dependency), +) -> Response: + """Create map tile.""" + endpoint = get_endpoint_function( + pgstac_mosaic_factory.router, + path="/tiles/{z}/{x}/{y}", + method=request.method, + ) + result = endpoint( + search_id=search_id, + tile=tile, + tileMatrixSetId=tileMatrixSetId, + scale=scale, + format=format, + tile_params=tile_params, + layer_params=layer_params, + dataset_params=dataset_params, + pixel_selection=pixel_selection, + post_process=post_process, + rescale=rescale, + color_formula=color_formula, + colormap=colormap, + render_params=render_params, + pgstac_params=pgstac_params, + backend_params=backend_params, + reader_params=reader_params, + env=env, + ) + return result diff --git a/pctiler/pctiler/main.py b/pctiler/pctiler/main.py index 682e3428..f6557303 100755 --- a/pctiler/pctiler/main.py +++ b/pctiler/pctiler/main.py @@ -71,6 +71,11 @@ async def lifespan(app: FastAPI) -> AsyncGenerator: prefix=settings.mosaic_endpoint_prefix + "/{search_id}", tags=["PgSTAC Mosaic endpoints"], ) +app.include_router( + pg_mosaic.legacy_mosaic_router, + prefix=settings.mosaic_endpoint_prefix, + tags=["PgSTAC Mosaic endpoints"], +) pg_mosaic.add_collection_mosaic_info_route( app, prefix=settings.mosaic_endpoint_prefix, diff --git a/pctiler/pctiler/middleware.py b/pctiler/pctiler/middleware.py index 7c5ccc4f..42255aa6 100644 --- a/pctiler/pctiler/middleware.py +++ b/pctiler/pctiler/middleware.py @@ -1,5 +1,4 @@ import json -from collections import OrderedDict from starlette.datastructures import MutableHeaders from starlette.types import ASGIApp, Message, Receive, Scope, Send @@ -23,11 +22,16 @@ async def send_with_searchid(message: Message) -> None: elif message_type == "http.response.body": # Rewrite id to searchid for backwards compatibility, keep key order body = json.loads(message["body"]) - ordered_body = OrderedDict() - ordered_body["searchid"] = body.get("id") - ordered_body.update(body) + body["searchid"] = body.get("id") - resp_body = json.dumps(ordered_body, ensure_ascii=False).encode("utf-8") + updated_links = [] + for link in body.get("links", []): + link["href"] = link["href"].replace("/{tileMatrixSetId}", "") + updated_links.append(link) + + body["links"] = updated_links + + resp_body = json.dumps(body, ensure_ascii=False).encode("utf-8") message["body"] = resp_body # Update the content-length header on the start message diff --git a/pctiler/tests/endpoints/test_pg_mosaic.py b/pctiler/tests/endpoints/test_pg_mosaic.py index e9b1e9c1..508bb5aa 100644 --- a/pctiler/tests/endpoints/test_pg_mosaic.py +++ b/pctiler/tests/endpoints/test_pg_mosaic.py @@ -1,20 +1,16 @@ +from typing import Any, Dict, Tuple + import pytest from httpx import AsyncClient from pccommon.config.collections import MosaicInfo +REGISTER_TYPE = Tuple[str, Dict[str, Any]] -@pytest.mark.asyncio -async def test_get(client: AsyncClient) -> None: - response = await client.get("/mosaic/info?collection=naip") - assert response.status_code == 200 - info_dict = response.json() - mosaic_info = MosaicInfo(**info_dict) - assert mosaic_info.default_location.zoom == 13 +@pytest.fixture +async def register_search(client: AsyncClient) -> REGISTER_TYPE: -@pytest.mark.asyncio -async def test_register(client: AsyncClient) -> None: cql = { "filter-lang": "cql2-json", "filter": { @@ -24,9 +20,107 @@ async def test_register(client: AsyncClient) -> None: } expected_content_hash = "8b989f86a149628eabfde894fb965982" response = await client.post("/mosaic/register", json=cql) - assert response.status_code == 200 resp = response.json() + return (expected_content_hash, resp) + + +@pytest.mark.asyncio +async def test_mosaic_info(client: AsyncClient) -> None: + response = await client.get("/mosaic/info?collection=naip") + assert response.status_code == 200 + info_dict = response.json() + mosaic_info = MosaicInfo(**info_dict) + assert mosaic_info.default_location.zoom == 13 + + +@pytest.mark.asyncio +async def test_register(client: AsyncClient, register_search: REGISTER_TYPE) -> None: + + expected_content_hash, register_resp = register_search + # Test that `searchid` which has been removed in titiler remains in pctiler, # and that the search hash remains consistent - assert resp["searchid"] == expected_content_hash + assert register_resp["searchid"] == expected_content_hash + # Test that the links have had the {tileMatrixSetId} template string removed + assert len(register_resp["links"]) == 2 + assert register_resp["links"][0]["href"].endswith( + f"/mosaic/{expected_content_hash}/tilejson.json" + ) + assert register_resp["links"][1]["href"].endswith( + f"/mosaic/{expected_content_hash}/WMTSCapabilities.xml" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "route", + [ + # Legacy path routes + "mosaic/{searchId}/tiles/{z}/{x}/{y}", + "mosaic/{searchId}/tiles/{z}/{x}/{y}.{format}", + "mosaic/{searchId}/tiles/{z}/{x}/{y}@{scale}x", + "mosaic/{searchId}/tiles/{z}/{x}/{y}@{scale}x.{format}", + "mosaic/{searchId}/tiles/{tileMatrixSetId}/{z}/{x}/{y}", + "mosaic/{searchId}/tiles/{tileMatrixSetId}/{z}/{x}/{y}.{format}", + "mosaic/{searchId}/tiles/{tileMatrixSetId}/{z}/{x}/{y}@{scale}x", + "mosaic/{searchId}/tiles/{tileMatrixSetId}/{z}/{x}/{y}@{scale}x.{format}", + # Current path routes + "mosaic/tiles/{searchId}/{z}/{x}/{y}", + "mosaic/tiles/{searchId}/{z}/{x}/{y}.{format}", + "mosaic/tiles/{searchId}/{z}/{x}/{y}@{scale}x", + "mosaic/tiles/{searchId}/{z}/{x}/{y}@{scale}x.{format}", + "mosaic/tiles/{searchId}/{tileMatrixSetId}/{z}/{x}/{y}", + "mosaic/tiles/{searchId}/{tileMatrixSetId}/{z}/{x}/{y}.{format}", + "mosaic/tiles/{searchId}/{tileMatrixSetId}/{z}/{x}/{y}@{scale}x", + "mosaic/tiles/{searchId}/{tileMatrixSetId}/{z}/{x}/{y}@{scale}x.{format}", + ], +) +async def test_mosaic_tile_routes( + client: AsyncClient, register_search: REGISTER_TYPE, route: str +) -> None: + """ + For backwards compatibility, support both mosaic/tiles/{searchId} and + mosaic/{searchId}/tiles routes + """ + expected_content_hash, _ = register_search + + formatted_route = route.format( + searchId=expected_content_hash, + tileMatrixSetId="WebMercatorQuad", + z=16, + x=17218, + y=26838, + scale=2, + format="png", + ) + url = ( + f"/{formatted_route}?asset_bidx=image%7C1%2C2%2C3&assets=image&collection=naip" + ) + response = await client.get(url) + assert response.status_code == 200 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "route", + [ + "mosaic/{searchId}/tilejson.json", + "mosaic/{searchId}/{tileMatrixSetId}/tilejson.json", + "mosaic/{searchId}/WMTSCapabilities.xml", + "mosaic/{searchId}/{tileMatrixSetId}/WMTSCapabilities.xml", + ], +) +async def test_tile_metadata_routes( + client: AsyncClient, register_search: REGISTER_TYPE, route: str +) -> None: + search_id, _ = register_search + + formatted_route = route.format( + searchId=search_id, tileMatrixSetId="WebMercatorQuad" + ) + url = ( + f"/{formatted_route}?asset_bidx=image%7C1%2C2%2C3&assets=image&collection=naip" + ) + response = await client.get(url) + assert response.status_code == 200