feat: report to user if no markings were detected

Add custom task status with additional metadata/information such as
progress of the task or part-failure (e.g. no markings for one of many
uploaded files).

Add exception for the case that no markings were detected on all
uploaded files.

client-src/shared/poll/poll.js

@@ -32,7 +32,21 @@ async function poll(url, prefix) {
     async function onProgress(response) {
         // console.log("progress", response);
         const result = await response.json();
-        setTaskStatus(`${prefix}-status`, `Processing ${result.status}`);
+        if (result.status === "PROGRESS" && "info" in result) {
+            const messageProgress = `Processing in ${result.status}: ${result.info.current} of ${result.info.total} uploaded files have been processed.`;
+            if (result.info.failures && result.info.failures.length) {
+                const messageFailure = `For following files no markings were detected: ${result.info.failures.join(", ")}.`;
+                const messageContact = "Please feel free to report this failure ([email protected])";
+                setTaskStatus(
+                    `${prefix}-status`,
+                    [messageProgress, messageFailure, messageContact].join(" "),
+                );
+            } else {
+                setTaskStatus(`${prefix}-status`, messageProgress);
+            }
+        } else {
+            setTaskStatus(`${prefix}-status`, `Processing ${result.status}`);
+        }
     }
 
     async function onValid(response) {

sketch_map_tool/routes.py

@@ -201,6 +201,7 @@ def status(uuid: str, type_: REQUEST_TYPES, lang="en") -> Response:
 
     href = None
     error = None
+    info = None
     if task.ready():
         if task.successful():  # SUCCESS
             http_status = 200
@@ -216,6 +217,10 @@ def status(uuid: str, type_: REQUEST_TYPES, lang="en") -> Response:
             except Exception as err:
                 http_status = 500  # Internal Server Error
                 error = "{}: {}".format(type(err).__name__, str(err))
+    elif task.status == "PROGRESS":
+        # In progress, but has not been completed
+        http_status = 202  # Accepted
+        info = task.info
     else:  # PENDING, RETRY, STARTED
         # Accepted for processing, but has not been completed
         http_status = 202  # Accepted
@@ -225,6 +230,7 @@ def status(uuid: str, type_: REQUEST_TYPES, lang="en") -> Response:
         "type": type_,
         "href": href,
         "error": error,
+        "info": info,
     }
     body = {k: v for k, v in body_raw.items() if v is not None}
     return Response(json.dumps(body), status=http_status, mimetype="application/json")

sketch_map_tool/tasks.py

@@ -3,6 +3,7 @@
 from uuid import UUID
 from zipfile import ZipFile
 
+import celery.states
 from celery.result import AsyncResult
 from celery.signals import setup_logging, worker_process_init, worker_process_shutdown
 from geojson import FeatureCollection
@@ -16,7 +17,8 @@
 from sketch_map_tool import get_config_value, map_generation
 from sketch_map_tool.database import client_celery as db_client_celery
 from sketch_map_tool.definitions import get_attribution
-from sketch_map_tool.helpers import to_array
+from sketch_map_tool.exceptions import MarkingDetectionError
+from sketch_map_tool.helpers import N_, to_array
 from sketch_map_tool.models import Bbox, Layer, PaperFormat, Size
 from sketch_map_tool.upload_processing import (
     clip,
@@ -100,8 +102,9 @@ def generate_quality_report(bbox: Bbox) -> BytesIO | AsyncResult:
 
 # 2. DIGITIZE RESULTS
 #
-@celery.task()
+@celery.task(bind=True)
 def georeference_sketch_maps(
+    self,
     file_ids: list[int],
     file_names: list[str],
     uuids: list[str],
@@ -135,7 +138,11 @@ def zip_(file: BytesIO, file_name: str):
             zip_file.writestr(f"{name}.geotiff", file.read())
 
     buffer = BytesIO()
-    for file_id, uuid, file_name in zip(file_ids, uuids, file_names):
+    for i, (file_id, uuid, file_name) in enumerate(zip(file_ids, uuids, file_names)):
+        self.update_state(
+            state="PROGRESS",
+            meta={"current": i, "total": len(file_ids), "failures": []},
+        )
         zip_(process(file_id, uuid), file_name)
     with ZipFile(buffer, "a") as zip_file:
         zip_file.writestr("attributions.txt", get_attribution_file().read())
@@ -144,8 +151,9 @@ def zip_(file: BytesIO, file_name: str):
     return buffer
 
 
-@celery.task()
+@celery.task(bind=True)
 def digitize_sketches(
+    self,
     file_ids: list[int],
     file_names: list[str],
     uuids: list[str],
@@ -179,7 +187,12 @@ def digitize_sketches(
         yolo_cls_esri: YOLO = YOLO(path)
 
     l = []  # noqa: E741
-    for file_id, file_name, uuid in zip(file_ids, file_names, uuids):
+    failures = []
+    for i, (file_id, file_name, uuid) in enumerate(zip(file_ids, file_names, uuids)):
+        self.update_state(
+            state="PROGRESS",
+            meta={"current": i, "total": len(file_ids), "failures": failures},
+        )
         # r = interim result
         r: BytesIO = db_client_celery.select_file(file_id)  # type: ignore
         r: NDArray = to_array(r)  # type: ignore
@@ -200,12 +213,18 @@ def digitize_sketches(
             yolo_cls,
             sam_predictor,
         )  # type: ignore
+        if len(r) == 0:
+            logging.warning("No markings were detected for file " + file_name)
+            failures.append(file_name)
+            continue
         # m = marking
         for m in r:
             m: BytesIO = georeference(m, bboxes[uuid], bgr=False)  # type: ignore
             m: FeatureCollection = polygonize(m, layer_name=file_name)  # type: ignore
             m: FeatureCollection = post_process(m, file_name)
             l.append(m)
+    if len(l) == 0:
+        raise MarkingDetectionError(N_("No markings have been detected."))
     return merge(l)
 
 

tests/integration/test_routes.py

@@ -1,4 +1,5 @@
 from io import BytesIO
+from time import time
 from uuid import UUID, uuid4
 
 import pytest
@@ -141,6 +142,31 @@ def test_api_status_uuid_digitize(uuid_digitize, type_, flask_client):
     assert resp.json["href"] == f"/api/download/{uuid_digitize}/{type_}"
 
 
+# TODO: Make test case work in a run of the whole test suite
+@pytest.mark.skip("Only works in a single test run")
+@vcr_app.use_cassette
+def test_api_status_uuid_digitize_progress(sketch_map_marked, flask_client):
+    """Test if custom task status information is return by /status"""
+    unique_file_name = str(uuid4())
+    data = {"file": [(BytesIO(sketch_map_marked), unique_file_name)], "consent": "True"}
+    response = flask_client.post("/digitize/results", data=data, follow_redirects=True)
+    assert response.status_code == 200
+
+    # Extract UUID from response
+    url_parts = response.request.path.rsplit("/")
+    uuid = url_parts[-1]
+
+    # try for 5 sec
+    end = time() + 5
+    while time() < end:
+        resp = flask_client.get(f"/api/status/{uuid}/raster-results")
+        if resp.json["status"] == "PROGRESS":
+            break
+    assert resp.status_code == 202
+    assert resp.json["status"] == "PROGRESS"
+    assert resp.json["info"] == {"current": 0, "total": 1}
+
+
 def test_api_download_uuid_sketch_map(uuid_create, flask_client):
     resp = flask_client.get(f"/api/download/{uuid_create}/sketch-map")
     assert resp.status_code == 200