From 5e5bbb073eec7d93919fd5a18b77347322e2d0d7 Mon Sep 17 00:00:00 2001 From: DenizYil Date: Wed, 9 Aug 2023 13:49:12 +0200 Subject: [PATCH 01/16] add the ability to query multiple metadata --- terracotta/handlers/metadata.py | 32 +++++++++++-- terracotta/server/metadata.py | 79 +++++++++++++++++++++++++++++++-- tests/handlers/test_metadata.py | 19 ++++++++ 3 files changed, 123 insertions(+), 7 deletions(-) diff --git a/terracotta/handlers/metadata.py b/terracotta/handlers/metadata.py index 325f4b5e..24cfc6fa 100644 --- a/terracotta/handlers/metadata.py +++ b/terracotta/handlers/metadata.py @@ -3,18 +3,44 @@ Handle /metadata API endpoint. """ -from typing import Mapping, Sequence, Dict, Any, Union +from typing import Mapping, Sequence, Dict, Any, Union, List, Optional from collections import OrderedDict from terracotta import get_settings, get_driver from terracotta.profile import trace +def filter_metadata(metadata: Dict[str, Any], columns: Optional[List[str]]) -> Dict[str, Any]: + """Filter metadata by columns, if given""" + assert columns is None or len(columns) > 0, "columns must either be a non-empty list or None" + + if columns: + metadata = {c: metadata[c] for c in columns} + + return metadata + + @trace("metadata_handler") -def metadata(keys: Union[Sequence[str], Mapping[str, str]]) -> Dict[str, Any]: +def metadata(columns: Optional[List[str]], keys: Union[Sequence[str], Mapping[str, str]]) -> Dict[str, Any]: """Returns all metadata for a single dataset""" settings = get_settings() driver = get_driver(settings.DRIVER_PATH, provider=settings.DRIVER_PROVIDER) - metadata = driver.get_metadata(keys) + metadata = filter_metadata(driver.get_metadata(keys), columns) metadata["keys"] = OrderedDict(zip(driver.key_names, keys)) return metadata + + +@trace("multiple_metadata_handler") +def multiple_metadata(columns: Optional[List[str]], datasets: List[List[str]]) -> List[Dict[str, Any]]: + """Returns all metadata for multiple datasets""" + settings = get_settings() + driver = get_driver(settings.DRIVER_PATH, provider=settings.DRIVER_PROVIDER) + key_names = driver.key_names + + out = [] + for dataset in datasets: + metadata = filter_metadata(driver.get_metadata(dataset), columns) + metadata["keys"] = OrderedDict(zip(key_names, dataset)) + out.append(metadata) + + return out diff --git a/terracotta/server/metadata.py b/terracotta/server/metadata.py index 8a547162..3981b131 100644 --- a/terracotta/server/metadata.py +++ b/terracotta/server/metadata.py @@ -3,8 +3,8 @@ Flask route to handle /metadata calls. """ -from marshmallow import Schema, fields, validate -from flask import jsonify, Response +from marshmallow import Schema, fields, validate, ValidationError +from flask import jsonify, Response, request from terracotta.server.flask_api import METADATA_API @@ -50,6 +50,37 @@ class Meta: ) +class CommaSeparatedListField(fields.Field): + def _deserialize(self, value, attr, data, **kwargs): + try: + assert value[0] == "[" and value[-1] == "]" + + if value == "[]": + return [] + + return value[1:-1].split(', ') + except ValueError: + raise ValidationError("Invalid input for a list of values.") + + +class MetadataColumnsSchema(Schema): + columns = CommaSeparatedListField( + description="Columns of dataset to be returned" + ) + + +class MultipleMetadataDatasetsSchema(Schema): + keys = fields.List( + fields.List( + fields.String(), + description="Keys identifying dataset", + required=True, + ), + required=True, + description="Array containing all available key combinations", + ) + + @METADATA_API.route("/metadata/", methods=["GET"]) def get_metadata(keys: str) -> Response: """Get metadata for given dataset @@ -63,6 +94,8 @@ def get_metadata(keys: str) -> Response: description: Keys of dataset to retrieve metadata for (e.g. 'value1/value2') type: path required: true + - in: query + schema: MetadataColumnsSchema responses: 200: description: All metadata for given dataset @@ -72,7 +105,45 @@ def get_metadata(keys: str) -> Response: """ from terracotta.handlers.metadata import metadata + columns_schema = MetadataColumnsSchema() + columns = columns_schema.load(request.args).get("columns") + parsed_keys = [key for key in keys.split("/") if key] - payload = metadata(parsed_keys) - schema = MetadataSchema() + + payload = metadata(columns, parsed_keys) + schema = MetadataSchema(partial=columns is not None) return jsonify(schema.load(payload)) + + +@METADATA_API.route("/metadata", methods=["POST"]) +def get_multiple_metadata() -> Response: + """Get metadata for multiple datasets + --- + post: + summary: /metadata + description: + Retrieve metadata for multiple datasets, identified by the + body payload. Desired columns can be filtered using the ?columns + query. + parameters: + - in: query + schema: MetadataColumnsSchema + - in: body + schema: MultipleMetadataDatasetsSchema + responses: + 200: + description: All metadata for given dataset + schema: MetadataSchema + 404: + description: No dataset found for given key combination + """ + from terracotta.handlers.metadata import multiple_metadata + + datasets_schema = MultipleMetadataDatasetsSchema() + datasets = datasets_schema.load(request.json).get("keys") + + columns_schema = MetadataColumnsSchema() + columns = columns_schema.load(request.args).get("columns") + + schema = MetadataSchema(many=True, partial=columns is not None) + return jsonify(schema.load(multiple_metadata(columns, datasets))) diff --git a/tests/handlers/test_metadata.py b/tests/handlers/test_metadata.py index 3d06ba07..7e885dc1 100644 --- a/tests/handlers/test_metadata.py +++ b/tests/handlers/test_metadata.py @@ -5,3 +5,22 @@ def test_metadata_handler(use_testdb): md = metadata.metadata(ds) assert md assert md["metadata"] == ["extra_data"] + + +def test_multiple_metadata_handler(use_testdb): + from terracotta.handlers import metadata, datasets + + ds = datasets.datasets() + ds1 = list(ds[0].values()) + ds2 = list(ds[1].values()) + + md = metadata.multiple_metadata(None, [ds1, ds2]) + + assert md + assert md[0]["metadata"] == ["extra_data"] + assert len(md) == 2 + + md = metadata.multiple_metadata(["metadata", "bounds"], [ds1, ds2]) + assert md + assert len(md[0].keys()) == 3 + assert all(k in md[0].keys() for k in ("metadata", "bounds", "keys")) From 7e400d8cdb19f1c6f65caf87a6088d1a0e35ea21 Mon Sep 17 00:00:00 2001 From: DenizYil Date: Wed, 9 Aug 2023 21:40:50 +0200 Subject: [PATCH 02/16] fix tests --- tests/handlers/test_metadata.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/handlers/test_metadata.py b/tests/handlers/test_metadata.py index 7e885dc1..9a6395ec 100644 --- a/tests/handlers/test_metadata.py +++ b/tests/handlers/test_metadata.py @@ -2,10 +2,15 @@ def test_metadata_handler(use_testdb): from terracotta.handlers import metadata, datasets ds = datasets.datasets()[0] - md = metadata.metadata(ds) + md = metadata.metadata(None, ds) assert md assert md["metadata"] == ["extra_data"] + md = metadata.metadata(["metadata", "bounds"], ds) + assert md + assert len(md.keys()) == 3 + assert all(k in md.keys() for k in ("metadata", "bounds", "keys")) + def test_multiple_metadata_handler(use_testdb): from terracotta.handlers import metadata, datasets From 354a91512f6b8349c228107d55436cc0a8223aa2 Mon Sep 17 00:00:00 2001 From: DenizYil Date: Wed, 9 Aug 2023 22:21:54 +0200 Subject: [PATCH 03/16] fix formatting --- terracotta/handlers/metadata.py | 16 ++++++++++++---- terracotta/server/metadata.py | 6 ++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/terracotta/handlers/metadata.py b/terracotta/handlers/metadata.py index 24cfc6fa..830217e1 100644 --- a/terracotta/handlers/metadata.py +++ b/terracotta/handlers/metadata.py @@ -10,9 +10,13 @@ from terracotta.profile import trace -def filter_metadata(metadata: Dict[str, Any], columns: Optional[List[str]]) -> Dict[str, Any]: +def filter_metadata( + metadata: Dict[str, Any], columns: Optional[List[str]] +) -> Dict[str, Any]: """Filter metadata by columns, if given""" - assert columns is None or len(columns) > 0, "columns must either be a non-empty list or None" + assert ( + columns is None or len(columns) > 0 + ), "columns must either be a non-empty list or None" if columns: metadata = {c: metadata[c] for c in columns} @@ -21,7 +25,9 @@ def filter_metadata(metadata: Dict[str, Any], columns: Optional[List[str]]) -> D @trace("metadata_handler") -def metadata(columns: Optional[List[str]], keys: Union[Sequence[str], Mapping[str, str]]) -> Dict[str, Any]: +def metadata( + columns: Optional[List[str]], keys: Union[Sequence[str], Mapping[str, str]] +) -> Dict[str, Any]: """Returns all metadata for a single dataset""" settings = get_settings() driver = get_driver(settings.DRIVER_PATH, provider=settings.DRIVER_PROVIDER) @@ -31,7 +37,9 @@ def metadata(columns: Optional[List[str]], keys: Union[Sequence[str], Mapping[st @trace("multiple_metadata_handler") -def multiple_metadata(columns: Optional[List[str]], datasets: List[List[str]]) -> List[Dict[str, Any]]: +def multiple_metadata( + columns: Optional[List[str]], datasets: List[List[str]] +) -> List[Dict[str, Any]]: """Returns all metadata for multiple datasets""" settings = get_settings() driver = get_driver(settings.DRIVER_PATH, provider=settings.DRIVER_PROVIDER) diff --git a/terracotta/server/metadata.py b/terracotta/server/metadata.py index 3981b131..25d1be45 100644 --- a/terracotta/server/metadata.py +++ b/terracotta/server/metadata.py @@ -58,15 +58,13 @@ def _deserialize(self, value, attr, data, **kwargs): if value == "[]": return [] - return value[1:-1].split(', ') + return value[1:-1].split(", ") except ValueError: raise ValidationError("Invalid input for a list of values.") class MetadataColumnsSchema(Schema): - columns = CommaSeparatedListField( - description="Columns of dataset to be returned" - ) + columns = CommaSeparatedListField(description="Columns of dataset to be returned") class MultipleMetadataDatasetsSchema(Schema): From 2cda97e184843b078b5b3a6a62d73fd793a9991a Mon Sep 17 00:00:00 2001 From: DenizYil Date: Thu, 10 Aug 2023 01:31:19 +0200 Subject: [PATCH 04/16] surround block with driver.connect() to keep connection opened --- terracotta/handlers/metadata.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/terracotta/handlers/metadata.py b/terracotta/handlers/metadata.py index 830217e1..6afba6a5 100644 --- a/terracotta/handlers/metadata.py +++ b/terracotta/handlers/metadata.py @@ -46,9 +46,10 @@ def multiple_metadata( key_names = driver.key_names out = [] - for dataset in datasets: - metadata = filter_metadata(driver.get_metadata(dataset), columns) - metadata["keys"] = OrderedDict(zip(key_names, dataset)) - out.append(metadata) + with driver.connect(): + for dataset in datasets: + metadata = filter_metadata(driver.get_metadata(dataset), columns) + metadata["keys"] = OrderedDict(zip(key_names, dataset)) + out.append(metadata) return out From 7af851986a1a18eac0f38c24a81c66e8d33a7ca8 Mon Sep 17 00:00:00 2001 From: DenizYil Date: Thu, 10 Aug 2023 23:38:11 +0200 Subject: [PATCH 05/16] change list parsing to use json and pre_load --- terracotta/server/metadata.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/terracotta/server/metadata.py b/terracotta/server/metadata.py index 25d1be45..bd415ce5 100644 --- a/terracotta/server/metadata.py +++ b/terracotta/server/metadata.py @@ -3,7 +3,10 @@ Flask route to handle /metadata calls. """ -from marshmallow import Schema, fields, validate, ValidationError +from typing import Any, Mapping, Dict +import json + +from marshmallow import Schema, fields, validate, pre_load, ValidationError from flask import jsonify, Response, request from terracotta.server.flask_api import METADATA_API @@ -50,21 +53,28 @@ class Meta: ) -class CommaSeparatedListField(fields.Field): - def _deserialize(self, value, attr, data, **kwargs): - try: - assert value[0] == "[" and value[-1] == "]" +class MetadataColumnsSchema(Schema): + columns = fields.List( + fields.String(), + description="List of columns to return", + required=False, + ) - if value == "[]": - return [] + @pre_load + def validate_columns(self, data: Mapping[str, Any], **kwargs: Any) -> Dict[str, Any]: + columns = data.get("columns") - return value[1:-1].split(", ") - except ValueError: - raise ValidationError("Invalid input for a list of values.") + if columns: + data = dict(data.items()) + try: + data["columns"] = json.loads(columns) + except json.decoder.JSONDecodeError as exc: + raise ValidationError( + "columns must be a JSON list" + ) from exc -class MetadataColumnsSchema(Schema): - columns = CommaSeparatedListField(description="Columns of dataset to be returned") + return data class MultipleMetadataDatasetsSchema(Schema): From 286bf2db398034b6c3b08124c55d5aadd2640478 Mon Sep 17 00:00:00 2001 From: DenizYil Date: Thu, 10 Aug 2023 23:40:05 +0200 Subject: [PATCH 06/16] fix formatting --- terracotta/server/metadata.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/terracotta/server/metadata.py b/terracotta/server/metadata.py index bd415ce5..74ac3b3b 100644 --- a/terracotta/server/metadata.py +++ b/terracotta/server/metadata.py @@ -61,7 +61,9 @@ class MetadataColumnsSchema(Schema): ) @pre_load - def validate_columns(self, data: Mapping[str, Any], **kwargs: Any) -> Dict[str, Any]: + def validate_columns( + self, data: Mapping[str, Any], **kwargs: Any + ) -> Dict[str, Any]: columns = data.get("columns") if columns: @@ -70,9 +72,7 @@ def validate_columns(self, data: Mapping[str, Any], **kwargs: Any) -> Dict[str, try: data["columns"] = json.loads(columns) except json.decoder.JSONDecodeError as exc: - raise ValidationError( - "columns must be a JSON list" - ) from exc + raise ValidationError("columns must be a JSON list") from exc return data From 75435db34977105dbd9ff96d6c2dfdb9d3d8c746 Mon Sep 17 00:00:00 2001 From: DenizYil Date: Thu, 10 Aug 2023 23:57:43 +0200 Subject: [PATCH 07/16] fix type checking --- terracotta/server/metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terracotta/server/metadata.py b/terracotta/server/metadata.py index 74ac3b3b..c4e3a5f5 100644 --- a/terracotta/server/metadata.py +++ b/terracotta/server/metadata.py @@ -63,7 +63,7 @@ class MetadataColumnsSchema(Schema): @pre_load def validate_columns( self, data: Mapping[str, Any], **kwargs: Any - ) -> Dict[str, Any]: + ) -> Dict[str, Any] | Mapping[str, Any]: columns = data.get("columns") if columns: @@ -148,7 +148,7 @@ def get_multiple_metadata() -> Response: from terracotta.handlers.metadata import multiple_metadata datasets_schema = MultipleMetadataDatasetsSchema() - datasets = datasets_schema.load(request.json).get("keys") + datasets = datasets_schema.load(request.json or {}).get("keys") columns_schema = MetadataColumnsSchema() columns = columns_schema.load(request.args).get("columns") From bbeb12a57e95b1a289ebdcedc95008cf83c84e96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dion=20H=C3=A4fner?= Date: Fri, 11 Aug 2023 01:59:36 +0200 Subject: [PATCH 08/16] fix typing errors --- terracotta/scripts/click_types.py | 2 +- terracotta/server/metadata.py | 28 +++++++++++++++++----------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/terracotta/scripts/click_types.py b/terracotta/scripts/click_types.py index 341c19ad..cb282b4c 100644 --- a/terracotta/scripts/click_types.py +++ b/terracotta/scripts/click_types.py @@ -26,7 +26,7 @@ class PathlibPath(click.Path): """Converts a string to a pathlib.Path object""" def convert(self, *args: Any) -> pathlib.Path: - return pathlib.Path(super().convert(*args)) + return pathlib.Path(str(super().convert(*args))) RasterPatternType = Tuple[List[str], Dict[Tuple[str, ...], str]] diff --git a/terracotta/server/metadata.py b/terracotta/server/metadata.py index c4e3a5f5..cce1b7f9 100644 --- a/terracotta/server/metadata.py +++ b/terracotta/server/metadata.py @@ -10,6 +10,7 @@ from flask import jsonify, Response, request from terracotta.server.flask_api import METADATA_API +from terracotta.exceptions import InvalidArgumentsError class MetadataSchema(Schema): @@ -63,17 +64,17 @@ class MetadataColumnsSchema(Schema): @pre_load def validate_columns( self, data: Mapping[str, Any], **kwargs: Any - ) -> Dict[str, Any] | Mapping[str, Any]: - columns = data.get("columns") - - if columns: - data = dict(data.items()) - + ) -> Dict[str, Any]: + data = dict(data.items()) + var = "columns" + val = data.get(var) + if val: try: - data["columns"] = json.loads(columns) + data[var] = json.loads(val) except json.decoder.JSONDecodeError as exc: - raise ValidationError("columns must be a JSON list") from exc - + raise ValidationError( + f"Could not decode value for {var} as JSON" + ) from exc return data @@ -147,11 +148,16 @@ def get_multiple_metadata() -> Response: """ from terracotta.handlers.metadata import multiple_metadata + request_body = request.json + if not isinstance(request_body, dict): + raise InvalidArgumentsError("Request body must be a JSON object") + datasets_schema = MultipleMetadataDatasetsSchema() - datasets = datasets_schema.load(request.json or {}).get("keys") + datasets = datasets_schema.load(request_body).get("keys") columns_schema = MetadataColumnsSchema() columns = columns_schema.load(request.args).get("columns") + payload = multiple_metadata(columns, datasets) schema = MetadataSchema(many=True, partial=columns is not None) - return jsonify(schema.load(multiple_metadata(columns, datasets))) + return jsonify(schema.load(payload)) From 90a9038910866fc799acef405c6ce09146ccc682 Mon Sep 17 00:00:00 2001 From: DenizYil Date: Thu, 17 Aug 2023 20:55:08 +0200 Subject: [PATCH 09/16] add MAX_POST_METADATA_KEYS setting and testing for endpoint --- terracotta/config.py | 3 +++ terracotta/handlers/metadata.py | 2 +- tests/server/test_flask_api.py | 25 +++++++++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/terracotta/config.py b/terracotta/config.py index 4dbcfc3e..ff9cbc00 100644 --- a/terracotta/config.py +++ b/terracotta/config.py @@ -92,6 +92,9 @@ class TerracottaSettings(NamedTuple): #: Use a process pool for band retrieval in parallel USE_MULTIPROCESSING: bool = True + #: Maximum number of metadata keys per POST /metadata request + MAX_POST_METADATA_KEYS: int = 100 + AVAILABLE_SETTINGS: Tuple[str, ...] = TerracottaSettings._fields diff --git a/terracotta/handlers/metadata.py b/terracotta/handlers/metadata.py index 6afba6a5..381e936b 100644 --- a/terracotta/handlers/metadata.py +++ b/terracotta/handlers/metadata.py @@ -47,7 +47,7 @@ def multiple_metadata( out = [] with driver.connect(): - for dataset in datasets: + for dataset in datasets[:settings.MAX_POST_METADATA_KEYS]: metadata = filter_metadata(driver.get_metadata(dataset), columns) metadata["keys"] = OrderedDict(zip(key_names, dataset)) out.append(metadata) diff --git a/tests/server/test_flask_api.py b/tests/server/test_flask_api.py index 6a216ad2..8e46a139 100644 --- a/tests/server/test_flask_api.py +++ b/tests/server/test_flask_api.py @@ -90,6 +90,31 @@ def test_get_metadata_nonexisting(client, use_testdb): assert rv.status_code == 404 +def test_post_metadata(client, use_testdb): + rv = client.post( + "/metadata", + json={ + "keys": [["val11", "x", "val12"], ["val21", "x", "val22"]] + }, + ) + + assert rv.status_code == 200 + assert len(json.loads(rv.data)) == 2 + + +def test_post_metadata_specific_columns(client, use_testdb): + rv = client.post( + '/metadata?columns=["bounds", "range"]', + json={ + "keys": [["val11", "x", "val12"], ["val21", "x", "val22"]] + }, + ) + + assert rv.status_code == 200 + assert len(json.loads(rv.data)) == 2 + assert set(json.loads(rv.data)[0].keys()) == {"bounds", "range", "keys"} + + def test_get_datasets(client, use_testdb): rv = client.get("/datasets") assert rv.status_code == 200 From 767928b261c0ba58287596aa006b6e35924e647e Mon Sep 17 00:00:00 2001 From: DenizYil Date: Thu, 17 Aug 2023 20:56:38 +0200 Subject: [PATCH 10/16] fix styling --- terracotta/handlers/metadata.py | 2 +- tests/server/test_flask_api.py | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/terracotta/handlers/metadata.py b/terracotta/handlers/metadata.py index 381e936b..2e01f3c0 100644 --- a/terracotta/handlers/metadata.py +++ b/terracotta/handlers/metadata.py @@ -47,7 +47,7 @@ def multiple_metadata( out = [] with driver.connect(): - for dataset in datasets[:settings.MAX_POST_METADATA_KEYS]: + for dataset in datasets[: settings.MAX_POST_METADATA_KEYS]: metadata = filter_metadata(driver.get_metadata(dataset), columns) metadata["keys"] = OrderedDict(zip(key_names, dataset)) out.append(metadata) diff --git a/tests/server/test_flask_api.py b/tests/server/test_flask_api.py index 8e46a139..16c2ca03 100644 --- a/tests/server/test_flask_api.py +++ b/tests/server/test_flask_api.py @@ -93,9 +93,7 @@ def test_get_metadata_nonexisting(client, use_testdb): def test_post_metadata(client, use_testdb): rv = client.post( "/metadata", - json={ - "keys": [["val11", "x", "val12"], ["val21", "x", "val22"]] - }, + json={"keys": [["val11", "x", "val12"], ["val21", "x", "val22"]]}, ) assert rv.status_code == 200 @@ -105,9 +103,7 @@ def test_post_metadata(client, use_testdb): def test_post_metadata_specific_columns(client, use_testdb): rv = client.post( '/metadata?columns=["bounds", "range"]', - json={ - "keys": [["val11", "x", "val12"], ["val21", "x", "val22"]] - }, + json={"keys": [["val11", "x", "val12"], ["val21", "x", "val22"]]}, ) assert rv.status_code == 200 From 6e597108b214da575315e228ddf4d54d5b47f192 Mon Sep 17 00:00:00 2001 From: DenizYil Date: Thu, 17 Aug 2023 21:06:06 +0200 Subject: [PATCH 11/16] add it to schema as well --- terracotta/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/terracotta/config.py b/terracotta/config.py index ff9cbc00..791f6f33 100644 --- a/terracotta/config.py +++ b/terracotta/config.py @@ -161,6 +161,8 @@ class SettingSchema(Schema): USE_MULTIPROCESSING = fields.Boolean() + MAX_POST_METADATA_KEYS = fields.Integer(validate=validate.Range(min=1)) + @pre_load def decode_lists(self, data: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]: for var in ( From 2d184e5392bb5698b8e1d915e7d6dc7e24b1240e Mon Sep 17 00:00:00 2001 From: DenizYil Date: Thu, 17 Aug 2023 21:58:55 +0200 Subject: [PATCH 12/16] test some exceptions --- tests/server/test_flask_api.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/server/test_flask_api.py b/tests/server/test_flask_api.py index 16c2ca03..0d22d8dc 100644 --- a/tests/server/test_flask_api.py +++ b/tests/server/test_flask_api.py @@ -111,6 +111,22 @@ def test_post_metadata_specific_columns(client, use_testdb): assert set(json.loads(rv.data)[0].keys()) == {"bounds", "range", "keys"} +def test_post_metadata_errors(debug_client, use_non_writable_testdb): + import marshmallow + + with pytest.raises(marshmallow.ValidationError): + debug_client.post( + '/metadata?columns=["range]', + json={"keys": [["val11", "x", "val12"], ["val21", "x", "val22"]], }, + ) + + with pytest.raises(KeyError): + debug_client.post( + '/metadata?columns=["invalid"]', + json={"keys": [["val11", "x", "val12"], ["val21", "x", "val22"]]}, + ) + + def test_get_datasets(client, use_testdb): rv = client.get("/datasets") assert rv.status_code == 200 From 98eb8e2012464739ad26f607456c8afc8e73082b Mon Sep 17 00:00:00 2001 From: DenizYil Date: Thu, 17 Aug 2023 21:59:33 +0200 Subject: [PATCH 13/16] raise 400 if limit is exceeded instead --- terracotta/handlers/metadata.py | 9 ++++++++- terracotta/server/metadata.py | 3 +++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/terracotta/handlers/metadata.py b/terracotta/handlers/metadata.py index 2e01f3c0..15b22e3b 100644 --- a/terracotta/handlers/metadata.py +++ b/terracotta/handlers/metadata.py @@ -7,6 +7,7 @@ from collections import OrderedDict from terracotta import get_settings, get_driver +from terracotta.exceptions import InvalidArgumentsError from terracotta.profile import trace @@ -45,9 +46,15 @@ def multiple_metadata( driver = get_driver(settings.DRIVER_PATH, provider=settings.DRIVER_PROVIDER) key_names = driver.key_names + if len(datasets) > settings.MAX_POST_METADATA_KEYS: + raise InvalidArgumentsError( + f"Maximum number of keys exceeded ({settings.MAX_POST_METADATA_KEYS}). " + f"This limit can be configured in the server settings." + ) + out = [] with driver.connect(): - for dataset in datasets[: settings.MAX_POST_METADATA_KEYS]: + for dataset in datasets: metadata = filter_metadata(driver.get_metadata(dataset), columns) metadata["keys"] = OrderedDict(zip(key_names, dataset)) out.append(metadata) diff --git a/terracotta/server/metadata.py b/terracotta/server/metadata.py index cce1b7f9..62d128ff 100644 --- a/terracotta/server/metadata.py +++ b/terracotta/server/metadata.py @@ -143,6 +143,9 @@ def get_multiple_metadata() -> Response: 200: description: All metadata for given dataset schema: MetadataSchema + 400: + description: + If the MAX_POST_METADATA_KEYS (100 by default) limit is exceeded 404: description: No dataset found for given key combination """ From 78e5377128a6ab1d92d6ff18264db1ca1c5092f6 Mon Sep 17 00:00:00 2001 From: DenizYil Date: Thu, 17 Aug 2023 22:02:19 +0200 Subject: [PATCH 14/16] fix styling --- tests/server/test_flask_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/server/test_flask_api.py b/tests/server/test_flask_api.py index 0d22d8dc..1417736d 100644 --- a/tests/server/test_flask_api.py +++ b/tests/server/test_flask_api.py @@ -117,7 +117,7 @@ def test_post_metadata_errors(debug_client, use_non_writable_testdb): with pytest.raises(marshmallow.ValidationError): debug_client.post( '/metadata?columns=["range]', - json={"keys": [["val11", "x", "val12"], ["val21", "x", "val22"]], }, + json={"keys": [["val11", "x", "val12"], ["val21", "x", "val22"]]}, ) with pytest.raises(KeyError): From 0e51fc721b943c0f2457c86d5cf9acf326da4c5a Mon Sep 17 00:00:00 2001 From: DenizYil Date: Sun, 20 Aug 2023 02:28:23 +0200 Subject: [PATCH 15/16] test more exceptions too --- tests/server/test_flask_api.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/server/test_flask_api.py b/tests/server/test_flask_api.py index 1417736d..f7e5aff8 100644 --- a/tests/server/test_flask_api.py +++ b/tests/server/test_flask_api.py @@ -112,6 +112,7 @@ def test_post_metadata_specific_columns(client, use_testdb): def test_post_metadata_errors(debug_client, use_non_writable_testdb): + from terracotta import exceptions import marshmallow with pytest.raises(marshmallow.ValidationError): @@ -120,6 +121,18 @@ def test_post_metadata_errors(debug_client, use_non_writable_testdb): json={"keys": [["val11", "x", "val12"], ["val21", "x", "val22"]]}, ) + with pytest.raises(exceptions.InvalidArgumentsError): + debug_client.post( + '/metadata?columns=["range"]', + json={"keys": [["val11", "x", "val12"] for _ in range(101)]}, + ) + + with pytest.raises(exceptions.InvalidArgumentsError): + debug_client.post( + '/metadata?columns=["range"]', + json="Invalid JSON", + ) + with pytest.raises(KeyError): debug_client.post( '/metadata?columns=["invalid"]', From ecff640081f06f09a90c372d3735fc257a83ba75 Mon Sep 17 00:00:00 2001 From: DenizYil Date: Sun, 20 Aug 2023 02:28:38 +0200 Subject: [PATCH 16/16] improve 400 description --- terracotta/server/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terracotta/server/metadata.py b/terracotta/server/metadata.py index 62d128ff..573926fd 100644 --- a/terracotta/server/metadata.py +++ b/terracotta/server/metadata.py @@ -145,7 +145,7 @@ def get_multiple_metadata() -> Response: schema: MetadataSchema 400: description: - If the MAX_POST_METADATA_KEYS (100 by default) limit is exceeded + If the maximum number of requested datasets is exceeded 404: description: No dataset found for given key combination """