✨✅ catalog: service-layer for registry and increased test coverage (p…

…art 4) (#6050)

Makefile

@@ -316,7 +316,7 @@ printf "$$rows" "Postgres DB" "http://$(get_my_ip).nip.io:18080/?pgsql=postgres&
 printf "$$rows" "Portainer" "http://$(get_my_ip).nip.io:9000" admin adminadmin;\
 printf "$$rows" "Redis" "http://$(get_my_ip).nip.io:18081";\
 printf "$$rows" "Dask Dashboard" "http://$(get_my_ip).nip.io:8787";\
-printf "$$rows" "Docker Registry" "$${REGISTRY_URL}" $${REGISTRY_USER} $${REGISTRY_PW};\
+printf "$$rows" "Docker Registry" "http://$${REGISTRY_URL}/v2/_catalog" $${REGISTRY_USER} $${REGISTRY_PW};\
 printf "$$rows" "Invitations" "http://$(get_my_ip).nip.io:8008/dev/doc" $${INVITATIONS_USERNAME} $${INVITATIONS_PASSWORD};\
 printf "$$rows" "Payments" "http://$(get_my_ip).nip.io:8011/dev/doc" $${PAYMENTS_USERNAME} $${PAYMENTS_PASSWORD};\
 printf "$$rows" "Rabbit Dashboard" "http://$(get_my_ip).nip.io:15672" admin adminadmin;\

services/catalog/openapi.json

@@ -581,8 +581,7 @@
           "name": {
             "type": "string",
             "title": "Name",
-            "description": "Name of the author",
-            "example": "Jim Knopf"
+            "description": "Name of the author"
           },
           "email": {
             "type": "string",
@@ -592,8 +591,7 @@
           },
           "affiliation": {
             "type": "string",
-            "title": "Affiliation",
-            "description": "Affiliation of the author"
+            "title": "Affiliation"
           }
         },
         "type": "object",
@@ -633,7 +631,12 @@
           "image",
           "url"
         ],
-        "title": "Badge"
+        "title": "Badge",
+        "example": {
+          "name": "osparc.io",
+          "image": "https://img.shields.io/website-up-down-green-red/https/itisfoundation.github.io.svg?label=documentation",
+          "url": "https://itisfoundation.github.io/"
+        }
       },
       "BaseMeta": {
         "properties": {
@@ -2186,6 +2189,11 @@
             "title": "Progress Regexp",
             "description": "regexp pattern for detecting computational service's progress"
           },
+          "image_digest": {
+            "type": "string",
+            "title": "Image Digest",
+            "description": "Image manifest digest. Note that this is NOT injected as an image label"
+          },
           "owner": {
             "type": "string",
             "format": "email",
@@ -2205,43 +2213,7 @@
           "outputs"
         ],
         "title": "ServiceGet",
-        "description": "Service metadata at publication time\n\n- read-only (can only be changed overwriting the image labels in the registry)\n- base metaddata\n- injected in the image labels\n\nNOTE: This model is serialized in .osparc/metadata.yml and in the labels of the docker image",
-        "example": {
-          "name": "File Picker",
-          "description": "description",
-          "classifiers": [],
-          "quality": {},
-          "accessRights": {
-            "1": {
-              "execute_access": true,
-              "write_access": false
-            },
-            "4": {
-              "execute_access": true,
-              "write_access": true
-            }
-          },
-          "key": "simcore/services/frontend/file-picker",
-          "version": "1.0.0",
-          "type": "dynamic",
-          "authors": [
-            {
-              "name": "Red Pandas",
-              "email": "[email protected]"
-            }
-          ],
-          "contact": "[email protected]",
-          "inputs": {},
-          "outputs": {
-            "outFile": {
-              "displayOrder": 0,
-              "label": "File",
-              "description": "Chosen File",
-              "type": "data:*/*"
-            }
-          },
-          "owner": "[email protected]"
-        }
+        "description": "Service metadata at publication time\n\n- read-only (can only be changed overwriting the image labels in the registry)\n- base metaddata\n- injected in the image labels\n\nNOTE: This model is serialized in .osparc/metadata.yml and in the labels of the docker image"
       },
       "ServiceGroupAccessRights": {
         "properties": {

services/catalog/src/simcore_service_catalog/api/dependencies/services.py

@@ -1,23 +1,22 @@
 import logging
-import urllib.parse
 from dataclasses import dataclass
-from typing import Annotated, Any, cast
+from typing import Annotated
 
 from fastapi import Depends, FastAPI, Header, HTTPException, status
-from models_library.api_schemas_catalog.services import ServiceGet
 from models_library.api_schemas_catalog.services_specifications import (
     ServiceSpecifications,
 )
-from models_library.services import ServiceKey, ServiceVersion
+from models_library.services_metadata_published import ServiceMetaDataPublished
 from models_library.services_resources import ResourcesDict
+from models_library.services_types import ServiceKey, ServiceVersion
 from pydantic import ValidationError
 from servicelib.fastapi.dependencies import get_app
 
 from ...core.settings import ApplicationSettings
 from ...db.repositories.groups import GroupsRepository
 from ...db.repositories.services import ServicesRepository
+from ...services import manifest
 from ...services.director import DirectorApi
-from ...services.function_services import get_function_service, is_function_service
 from .database import get_repository
 from .director import get_director_api
 
@@ -84,32 +83,20 @@ async def check_service_read_access(
     )
 
 
-async def get_service_from_registry(
+async def get_service_from_manifest(
     service_key: ServiceKey,
     service_version: ServiceVersion,
     director_client: Annotated[DirectorApi, Depends(get_director_api)],
-) -> ServiceGet:
+) -> ServiceMetaDataPublished:
     """
     Retrieves service metadata from the docker registry via the director
     """
     try:
-        if is_function_service(service_key):
-            frontend_service: dict[str, Any] = get_function_service(
-                key=service_key, version=service_version
-            )
-            _service_data = frontend_service
-        else:
-            # NOTE: raises HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE) on ANY failure
-            services_in_registry = cast(
-                list[Any],
-                await director_client.get(
-                    f"/services/{urllib.parse.quote_plus(service_key)}/{service_version}"
-                ),
-            )
-            _service_data = services_in_registry[0]
-
-        service: ServiceGet = ServiceGet.parse_obj(_service_data)
-        return service
+        return await manifest.get_service(
+            service_key=service_key,
+            service_version=service_version,
+            director_client=director_client,
+        )
 
     except ValidationError as exc:
         _logger.warning(

services/catalog/src/simcore_service_catalog/api/rest/_services.py

@@ -9,24 +9,25 @@
 from fastapi import APIRouter, Depends, Header, HTTPException, status
 from models_library.api_schemas_catalog.services import ServiceGet, ServiceUpdate
 from models_library.services import ServiceKey, ServiceType, ServiceVersion
+from models_library.services_metadata_published import ServiceMetaDataPublished
 from pydantic import ValidationError
 from pydantic.types import PositiveInt
 from servicelib.fastapi.requests_decorators import cancel_on_disconnect
 from starlette.requests import Request
 
+from ..._constants import (
+    DIRECTOR_CACHING_TTL,
+    LIST_SERVICES_CACHING_TTL,
+    RESPONSE_MODEL_POLICY,
+)
 from ...db.repositories.groups import GroupsRepository
 from ...db.repositories.services import ServicesRepository
 from ...models.services_db import ServiceAccessRightsAtDB, ServiceMetaDataAtDB
 from ...services.director import DirectorApi
 from ...services.function_services import is_function_service
 from ..dependencies.database import get_repository
 from ..dependencies.director import get_director_api
-from ..dependencies.services import get_service_from_registry
-from ._constants import (
-    DIRECTOR_CACHING_TTL,
-    LIST_SERVICES_CACHING_TTL,
-    RESPONSE_MODEL_POLICY,
-)
+from ..dependencies.services import get_service_from_manifest
 
 _logger = logging.getLogger(__name__)
 
@@ -82,6 +83,7 @@ def _build_cache_key(fct, *_, **kwargs):
 )
 async def list_services(
     request: Request,  # pylint:disable=unused-argument
+    *,
     user_id: PositiveInt,
     director_client: Annotated[DirectorApi, Depends(get_director_api)],
     groups_repository: Annotated[
@@ -91,7 +93,7 @@ async def list_services(
         ServicesRepository, Depends(get_repository(ServicesRepository))
     ],
     x_simcore_products_name: Annotated[str, Header(...)],
-    details: bool | None = True,  # noqa: FBT002
+    details: bool = True,
 ):
     # Access layer
     user_groups = await groups_repository.list_user_groups(user_id)
@@ -116,7 +118,6 @@ async def list_services(
     # Non-detailed views from the services_repo database
     if not details:
         # only return a stripped down version
-        # FIXME: add name, ddescription, type, etc...
         # NOTE: here validation is not necessary since key,version were already validated
         # in terms of time, this takes the most
         return [
@@ -189,7 +190,9 @@ async def cached_registry_services() -> dict[str, Any]:
 )
 async def get_service(
     user_id: int,
-    service: Annotated[ServiceGet, Depends(get_service_from_registry)],
+    service_in_manifest: Annotated[
+        ServiceMetaDataPublished, Depends(get_service_from_manifest)
+    ],
     groups_repository: Annotated[
         GroupsRepository, Depends(get_repository(GroupsRepository))
     ],
@@ -198,6 +201,8 @@ async def get_service(
     ],
     x_simcore_products_name: str = Header(None),
 ):
+    service_data: dict[str, Any] = {}
+
     # get the user groups
     user_groups = await groups_repository.list_user_groups(user_id)
     if not user_groups:
@@ -208,8 +213,8 @@ async def get_service(
         )
     # check the user has access to this service and to which extent
     service_in_db = await services_repo.get_service(
-        service.key,
-        service.version,
+        service_in_manifest.key,
+        service_in_manifest.version,
         gids=[group.gid for group in user_groups],
         write_access=True,
         product_name=x_simcore_products_name,
@@ -219,14 +224,18 @@ async def get_service(
         service_access_rights: list[
             ServiceAccessRightsAtDB
         ] = await services_repo.get_service_access_rights(
-            service.key, service.version, product_name=x_simcore_products_name
+            service_in_manifest.key,
+            service_in_manifest.version,
+            product_name=x_simcore_products_name,
         )
-        service.access_rights = {rights.gid: rights for rights in service_access_rights}
+        service_data["access_rights"] = {
+            rights.gid: rights for rights in service_access_rights
+        }
     else:
         # check if we have executable rights
         service_in_db = await services_repo.get_service(
-            service.key,
-            service.version,
+            service_in_manifest.key,
+            service_in_manifest.version,
             gids=[group.gid for group in user_groups],
             execute_access=True,
             product_name=x_simcore_products_name,
@@ -237,17 +246,19 @@ async def get_service(
                 status_code=status.HTTP_403_FORBIDDEN,
                 detail="You have insufficient rights to access the service",
             )
-    # access is allowed, override some of the values with what is in the db
-    service = service.copy(
-        update=service_in_db.dict(exclude_unset=True, exclude={"owner"})
-    )
+
     # the owner shall be converted to an email address
     if service_in_db.owner:
-        service.owner = await groups_repository.get_user_email_from_gid(
+        service_data["owner"] = await groups_repository.get_user_email_from_gid(
             service_in_db.owner
         )
 
-    return service
+    # access is allowed, override some of the values with what is in the db
+    service_in_manifest = service_in_manifest.copy(
+        update=service_in_db.dict(exclude_unset=True, exclude={"owner"})
+    )
+    service_data.update(service_in_manifest.dict(exclude_unset=True, by_alias=True))
+    return service_data
 
 
 @router.patch(
@@ -357,7 +368,7 @@ async def update_service(
     # now return the service
     return await get_service(
         user_id=user_id,
-        service=await get_service_from_registry(
+        service_in_manifest=await get_service_from_manifest(
             service_key, service_version, director_client
         ),
         groups_repository=groups_repository,

services/catalog/src/simcore_service_catalog/api/rest/_services_access_rights.py

@@ -7,11 +7,11 @@
 )
 from models_library.services import ServiceKey, ServiceVersion
 
+from ..._constants import RESPONSE_MODEL_POLICY
 from ...db.repositories.services import ServicesRepository
 from ...models.services_db import ServiceAccessRightsAtDB
 from ..dependencies.database import get_repository
 from ..dependencies.services import AccessInfo, check_service_read_access
-from ._constants import RESPONSE_MODEL_POLICY
 
 _logger = logging.getLogger(__name__)
 

services/catalog/src/simcore_service_catalog/api/rest/_services_ports.py

@@ -2,15 +2,15 @@
 from typing import Annotated
 
 from fastapi import APIRouter, Depends
-from models_library.api_schemas_catalog.services import ServiceGet
 from models_library.api_schemas_catalog.services_ports import ServicePortGet
+from models_library.services_metadata_published import ServiceMetaDataPublished
 
+from ..._constants import RESPONSE_MODEL_POLICY
 from ..dependencies.services import (
     AccessInfo,
     check_service_read_access,
-    get_service_from_registry,
+    get_service_from_manifest,
 )
-from ._constants import RESPONSE_MODEL_POLICY
 
 _logger = logging.getLogger(__name__)
 
@@ -25,7 +25,7 @@
 )
 async def list_service_ports(
     _user: Annotated[AccessInfo, Depends(check_service_read_access)],
-    service: Annotated[ServiceGet, Depends(get_service_from_registry)],
+    service: Annotated[ServiceMetaDataPublished, Depends(get_service_from_manifest)],
 ):
     ports: list[ServicePortGet] = []
 

services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py

@@ -22,6 +22,7 @@
 from models_library.utils.docker_compose import replace_env_vars_in_compose_spec
 from pydantic import parse_obj_as, parse_raw_as
 
+from ..._constants import RESPONSE_MODEL_POLICY, SIMCORE_SERVICE_SETTINGS_LABELS
 from ...db.repositories.services import ServicesRepository
 from ...services.director import DirectorApi
 from ...services.function_services import is_function_service
@@ -33,7 +34,6 @@
 from ..dependencies.director import get_director_api
 from ..dependencies.services import get_default_service_resources
 from ..dependencies.user_groups import list_user_groups
-from ._constants import RESPONSE_MODEL_POLICY, SIMCORE_SERVICE_SETTINGS_LABELS
 
 router = APIRouter()
 _logger = logging.getLogger(__name__)

services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py

@@ -9,12 +9,12 @@
 from models_library.services import ServiceKey, ServiceVersion
 from models_library.users import UserID
 
+from ..._constants import RESPONSE_MODEL_POLICY
 from ...db.repositories.groups import GroupsRepository
 from ...db.repositories.services import ServicesRepository
 from ...services.function_services import is_function_service
 from ..dependencies.database import get_repository
 from ..dependencies.services import get_default_service_specifications
-from ._constants import RESPONSE_MODEL_POLICY
 
 router = APIRouter()
 _logger = logging.getLogger(__name__)