Update pydantic, titiler, stac-fastapi (#233)

* update pccommon requirements

* update pcstac requirements

* update pcfuncs

* update pcfuncs/pcstac/pccommon

* allow YYYY-MM-DD date format in search

* update pctiler

* update stac-fastapi.*

* Dependencies

* Get mosaic tiles working

* Formatting

* Use items request model

* Backwards compat with searchid

* certifi upgrade

* Read envs correctly

* Restore health check filtering

* Update titiler uvicorn

* lint

* Fix casing on colormaps

* Fix cmap and add /crop

* Fixups

---------

Co-authored-by: vincentsarago <[email protected]>

.gitignore

@@ -28,6 +28,7 @@ wheels/
 .installed.cfg
 *.egg
 MANIFEST
+pyvenv.cfg
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

docker-compose.dev.yml

@@ -1,5 +1,6 @@
 services:
   stac-dev:
+    platform: linux/amd64
     image: pc-apis-stac-dev
     build:
       context: .
@@ -63,6 +64,7 @@ services:
     depends_on:
       - stac
   tiler-dev:
+    platform: linux/amd64
     image: pc-apis-tiler-dev
     # For Mac OS M1 user, you'll need to add `platform: linux/amd64`.
     # see https://github.com/developmentseed/titiler/discussions/387#discussioncomment-1643110

docker-compose.yml

@@ -1,6 +1,7 @@
 services:
 
   stac:
+    platform: linux/amd64
     image: pc-apis-stac
     build:
       context: .
@@ -22,7 +23,7 @@ services:
     image: pc-apis-tiler
     # For Mac OS M1 user, you'll need to add `platform: linux/amd64`.
     # see https://github.com/developmentseed/titiler/discussions/387#discussioncomment-1643110
-    # platform: linux/amd64
+    platform: linux/amd64
     build:
       context: .
       dockerfile: pctiler/Dockerfile
@@ -39,10 +40,11 @@ services:
       - ./pccommon:/opt/src/pccommon
     depends_on:
       - database
-    command: [ "uvicorn", "pctiler.main:app", "--host", "0.0.0.0", "--port", "8082", "--reload", "--proxy-headers" ]
+    command: [ "uvicorn", "pctiler.main:app", "--host", "0.0.0.0", "--port", "8082", "--reload", "--proxy-headers", "--root-path", "/data" ]
 
   funcs:
     image: pc-apis-funcs
+    platform: linux/amd64
     build:
       context: .
       dockerfile: pcfuncs/Dockerfile

pccommon/pccommon/cli.py

@@ -61,12 +61,12 @@ def dump(account: str, table: str, type: str, **kwargs: Any) -> int:
         if id:
             col_config = col_config_table.get_config(id)
             assert col_config
-            result[id] = col_config.dict()
+            result[id] = col_config.model_dump()
         else:
             for _, collection_id, col_config in col_config_table.get_all():
                 assert collection_id
                 assert col_config
-                result[collection_id] = col_config.dict()
+                result[collection_id] = col_config.model_dump()
 
     elif type == "container":
         con_config_table = ContainerConfigTable.from_environment(
@@ -77,11 +77,11 @@ def dump(account: str, table: str, type: str, **kwargs: Any) -> int:
             assert con_account
             con_config = con_config_table.get_config(con_account, id)
             assert con_config
-            result[f"{con_account}/{id}"] = con_config.dict()
+            result[f"{con_account}/{id}"] = con_config.model_dump()
         else:
             for storage_account, container, con_config in con_config_table.get_all():
                 assert con_config
-                result[f"{storage_account}/{container}"] = con_config.dict()
+                result[f"{storage_account}/{container}"] = con_config.model_dump()
     else:
         print(f"Unknown type: {type}")
         return 1

pccommon/pccommon/config/collections.py

@@ -1,12 +1,11 @@
 from enum import Enum
 from typing import Any, Dict, List, Optional, Tuple
 
-import orjson
 from humps import camelize
 from pydantic import BaseModel, Field
 
 from pccommon.tables import ModelTableService
-from pccommon.utils import get_param_str, orjson_dumps
+from pccommon.utils import get_param_str
 
 
 class RenderOptionType(str, Enum):
@@ -19,11 +18,13 @@ def __str__(self) -> str:
 
 
 class CamelModel(BaseModel):
-    class Config:
-        alias_generator = camelize
-        allow_population_by_field_name = True
-        json_loads = orjson.loads
-        json_dumps = orjson_dumps
+
+    model_config = {
+        # TODO, see if we can use pydantic native function
+        # https://docs.pydantic.dev/latest/api/config/#pydantic.alias_generators.to_camel
+        "alias_generator": camelize,
+        "populate_by_name": True,
+    }
 
 
 class VectorTileset(CamelModel):
@@ -137,10 +138,6 @@ def should_add_collection_links(self) -> bool:
     def should_add_item_links(self) -> bool:
         return self.create_links and (not self.hidden)
 
-    class Config:
-        json_loads = orjson.loads
-        json_dumps = orjson_dumps
-
 
 class Mosaics(CamelModel):
     """
@@ -187,11 +184,11 @@ class LegendConfig(CamelModel):
         showing legend labels as scaled values.
     """
 
-    type: Optional[str]
-    labels: Optional[List[str]]
-    trim_start: Optional[int]
-    trim_end: Optional[int]
-    scale_factor: Optional[float]
+    type: Optional[str] = None
+    labels: Optional[List[str]] = None
+    trim_start: Optional[int] = None
+    trim_end: Optional[int] = None
+    scale_factor: Optional[float] = None
 
 
 class VectorTileOptions(CamelModel):
@@ -216,10 +213,10 @@ class VectorTileOptions(CamelModel):
 
     tilejson_key: str
     source_layer: str
-    fill_color: Optional[str]
-    stroke_color: Optional[str]
-    stroke_width: Optional[int]
-    filter: Optional[List[Any]]
+    fill_color: Optional[str] = None
+    stroke_color: Optional[str] = None
+    stroke_width: Optional[int] = None
+    filter: Optional[List[Any]] = None
 
 
 class RenderOptionCondition(CamelModel):
@@ -329,10 +326,6 @@ class CollectionConfig(BaseModel):
     render_config: DefaultRenderConfig
     mosaic_info: MosaicInfo
 
-    class Config:
-        json_loads = orjson.loads
-        json_dumps = orjson_dumps
-
 
 class CollectionConfigTable(ModelTableService[CollectionConfig]):
     _model = CollectionConfig

pccommon/pccommon/config/containers.py

@@ -1,18 +1,20 @@
 from typing import Optional
 
-import orjson
 from pydantic import BaseModel
 
 from pccommon.tables import ModelTableService
-from pccommon.utils import orjson_dumps
 
 
 class ContainerConfig(BaseModel):
     has_cdn: bool = False
 
-    class Config:
-        json_loads = orjson.loads
-        json_dumps = orjson_dumps
+    # json_loads/json_dumps config have been removed
+    # the authors seems to indicate that parsing/serialization
+    # in Rust (pydantic-core) is fast (but maybe not as fast as orjson)
+    # https://github.com/pydantic/pydantic/discussions/6388
+    # class Config:
+    #     json_loads = orjson.loads
+    #     json_dumps = orjson_dumps
 
 
 class ContainerConfigTable(ModelTableService[ContainerConfig]):

pccommon/pccommon/config/core.py

@@ -4,7 +4,8 @@
 from cachetools import Cache, LRUCache, cachedmethod
 from cachetools.func import lru_cache
 from cachetools.keys import hashkey
-from pydantic import BaseModel, BaseSettings, Field, PrivateAttr, validator
+from pydantic import BaseModel, Field, PrivateAttr, field_validator
+from pydantic_settings import BaseSettings
 
 from pccommon.config.collections import CollectionConfigTable
 from pccommon.config.containers import ContainerConfigTable
@@ -23,7 +24,7 @@ class TableConfig(BaseModel):
     table_name: str
     account_url: Optional[str] = None
 
-    @validator("account_url")
+    @field_validator("account_url")
     def validate_url(cls, value: str) -> str:
         if value and not value.startswith("http://azurite:"):
             raise ValueError(
@@ -39,7 +40,7 @@ class PCAPIsConfig(BaseSettings):
 
     app_insights_instrumentation_key: Optional[str] = Field(  # type: ignore
         default=None,
-        env=APP_INSIGHTS_INSTRUMENTATION_KEY,
+        validation_alias=APP_INSIGHTS_INSTRUMENTATION_KEY,
     )
     collection_config: TableConfig
     container_config: TableConfig
@@ -55,6 +56,15 @@ class PCAPIsConfig(BaseSettings):
 
     debug: bool = False
 
+    model_config = {
+        "env_prefix": ENV_VAR_PCAPIS_PREFIX,
+        "env_nested_delimiter": "__",
+        # Mypy is complaining about this with
+        # error: Incompatible types (expression has type "str",
+        # TypedDict item "extra" has type "Extra")
+        "extra": "ignore",  # type: ignore
+    }
+
     @cachedmethod(cache=lambda self: self._cache, key=lambda _: hashkey("collection"))
     def get_collection_config_table(self) -> CollectionConfigTable:
         return CollectionConfigTable.from_environment(
@@ -86,8 +96,3 @@ def get_ip_exception_list_table(self) -> IPExceptionListTable:
     @lru_cache(maxsize=1)
     def from_environment(cls) -> "PCAPIsConfig":
         return PCAPIsConfig()  # type: ignore
-
-    class Config:
-        env_prefix = ENV_VAR_PCAPIS_PREFIX
-        extra = "ignore"
-        env_nested_delimiter = "__"

pccommon/pccommon/logging.py

@@ -60,26 +60,30 @@ def filter(self, record: logging.LogRecord) -> bool:
 
 # Prevent successful health check pings from being logged
 class HealthCheckFilter(logging.Filter):
+    def __init__(self, app_root_path: str):
+        super().__init__()
+        self.app_root_path = app_root_path
+
     def filter(self, record: logging.LogRecord) -> bool:
         if record.args is not None and len(record.args) != 5:
             return True
 
         args = cast(Tuple[str, str, str, str, int], record.args)
         endpoint = args[2]
         status = args[4]
-        if endpoint == "/_mgmt/ping" and status == 200:
+        if f"{self.app_root_path}/_mgmt/ping" == endpoint and status == 200:
             return False
 
         return True
 
 
 # Initialize logging, including a console handler, and sending all logs containing
 # custom_dimensions to Application Insights
-def init_logging(service_name: str) -> None:
+def init_logging(service_name: str, app_root_path: str) -> None:
     config = get_apis_config()
 
     # Exclude health check endpoint pings from the uvicorn logs
-    logging.getLogger("uvicorn.access").addFilter(HealthCheckFilter())
+    logging.getLogger("uvicorn.access").addFilter(HealthCheckFilter(app_root_path))
 
     # Setup logging for current package and pccommon
     for package in [PACKAGES[service_name], "pccommon"]:

pccommon/pccommon/tables.py

@@ -45,8 +45,10 @@ class TableError(Exception):
     pass
 
 
+# TODO: mypy is complaining locally about
+# "BaseModel" has no attribute "model_dump_json"
 def encode_model(m: BaseModel) -> str:
-    return m.json()
+    return m.model_dump_json()  # type: ignore
 
 
 def decode_dict(s: str) -> Dict[str, Any]:

pccommon/pccommon/tracing.py

@@ -3,7 +3,6 @@
 import re
 from typing import List, Optional, Tuple, Union, cast
 
-import fastapi
 from fastapi import Request
 from opencensus.ext.azure.trace_exporter import AzureExporter
 from opencensus.trace import execution_context
@@ -211,7 +210,7 @@ def _iter_cql(cql: dict, property_name: str) -> Optional[Union[str, List[str]]]:
     return None
 
 
-def add_stac_attributes_from_search(search_json: str, request: fastapi.Request) -> None:
+def add_stac_attributes_from_search(search_json: str, request: Request) -> None:
     """
     Try to add the Collection ID and Item ID from a search to the current span.
     """