Tkakar/cat 673 update epic image pyramid path (#96)

* Removed epic_entity parameter

* Base image-conf builder updated

* Epic builder channels fixed`

* Fixed tests for epic builder

* Minor fixes

README.md

@@ -24,7 +24,6 @@ $ src/vis-preview.py --help
 usage: vis-preview.py [-h] (--url URL | --json JSON) [--assaytypes_url URL]
                       [--assets_url URL] [--token TOKEN] [--marker MARKER]
                       [--to_json] [--epic_uuid UUID] [--parent_uuid UUID]
-                      [--epic_url EPIC_URL] [--epic_json EPIC_JSON]
 
 Given HuBMAP Dataset JSON, generate a Vitessce viewconf, and load vitessce.io.
 
@@ -43,9 +42,6 @@ optional arguments:
   --epic_uuid UUID      uuid of the EPIC dataset.
   --parent_uuid UUID    Parent uuid - Only needed for an image-pyramid support
                         dataset.
-  --epic_url EPIC_URL   URL which returns Dataset JSON for the EPIC dataset
-  --epic_json EPIC_JSON
-                        File containing Dataset JSON for the EPIC dataset
   ```
 
 

src/portal_visualization/builder_factory.py

@@ -34,7 +34,6 @@ def process_hints(hints):
     is_json = "json_based" in hints
     is_spatial = "spatial" in hints
     is_support = "is_support" in hints
-    is_segmentation_base = "segmentation_base" in hints
 
     return (
         is_image,
@@ -46,7 +45,6 @@ def process_hints(hints):
         is_json,
         is_spatial,
         is_support,
-        is_segmentation_base,
     )
 
 
@@ -71,14 +69,13 @@ def get_view_config_builder(entity, get_assaytype, parent=None, epic_uuid=None):
         is_anndata,
         is_json,
         is_spatial,
-        is_support,
-        is_segmentation_base
+        is_support
     ) = process_hints(hints)
 
     # vis-lifted image pyramids
     if parent is not None:
         # TODO: For now epic (base image's) support datasets doesn't have any hints
-        if epic_uuid is not None or is_segmentation_base:
+        if epic_uuid is not None:
             return SegImagePyramidViewConfBuilder
 
         elif is_support and is_image:
@@ -141,6 +138,6 @@ def get_view_config_builder(entity, get_assaytype, parent=None, epic_uuid=None):
     return NullViewConfBuilder
 
 
-def has_visualization(entity, get_assaytype, parent=None):
-    builder = get_view_config_builder(entity, get_assaytype, parent, epic_uuid=None)
+def has_visualization(entity, get_assaytype, parent=None, epic_uuid=None):
+    builder = get_view_config_builder(entity, get_assaytype, parent, epic_uuid)
     return builder != NullViewConfBuilder

src/portal_visualization/builders/base_builders.py

@@ -1,9 +1,6 @@
 import urllib
 from collections import namedtuple
 from abc import ABC, abstractmethod
-# import json
-# import requests
-# from pathlib import Path
 
 ConfCells = namedtuple('ConfCells', ['conf', 'cells'])
 
@@ -123,23 +120,6 @@ def _get_file_paths(self):
         """
         return [file["rel_path"] for file in self._entity["files"]]
 
-    # def _get_epic_entity(self):
-    # TODO: might need this if we decide to read the epic_entity on run time
-    #     request_init = self._get_request_init()
-    #     file_path = Path(__file__).resolve().parent.parent.parent / 'defaults.json'
-    #     print(file_path)
-    #     defaults = json.load(file_path.open())
-    #     # headers = {"headers": {"Authorization": f"Bearer {self._groups_token}"}}
-    #     url = f'{defaults['dataset_url']}/{self._epic_uuid}.json'
-    #     print(url)
-    #     response = requests.get(url, request_init)
-    #     if response.status_code == 403:
-    #         raise Exception('Protected data: Download JSON via browser; Redo with --json')
-    #     response.raise_for_status()
-    #     json_str = response.text
-    #     entity = json.loads(json_str)
-    #     return entity
-
 
 class _DocTestBuilder(ViewConfBuilder):  # pragma: no cover
     # The doctests on the methods in this file need a concrete class to instantiate:

src/portal_visualization/builders/epic_builders.py

@@ -7,8 +7,8 @@
 from requests import get
 import re
 import random
-from ..paths import OFFSETS_DIR, IMAGE_PYRAMID_DIR, SEGMENTATION_SUBDIR, SEGMENTATION_ZARR_STORES, SEGMENTATION_DIR
-
+from ..paths import OFFSETS_DIR, IMAGE_PYRAMID_DIR, SEGMENTATION_SUBDIR, SEGMENTATION_ZARR_STORES, \
+    SEGMENTATION_DIR, SEGMENTATION_SUPPORT_IMAGE_SUBDIR
 
 transformations_filename = 'transformations.json'
 zarr_path = f'{SEGMENTATION_SUBDIR}/{SEGMENTATION_ZARR_STORES}'
@@ -18,7 +18,7 @@
 
 
 class EPICConfBuilder(ViewConfBuilder):
-    def __init__(self, epic_uuid, base_conf: ConfCells, entity, epic_entity,
+    def __init__(self, epic_uuid, base_conf: ConfCells, entity,
                  groups_token, assets_endpoint, **kwargs) -> None:
         super().__init__(entity, groups_token, assets_endpoint, **kwargs)
 
@@ -37,7 +37,6 @@ def __init__(self, epic_uuid, base_conf: ConfCells, entity, epic_entity,
             self._base_conf: VitessceConfig = VitessceConfig.from_dict(base_conf.conf)
 
         self._epic_uuid = epic_uuid
-        self._epic_entity = epic_entity
 
         pass
 
@@ -84,24 +83,21 @@ class SegmentationMaskBuilder(EPICConfBuilder):
     def _apply(self, conf):
         zarr_url = self.zarr_store_url()
         datasets = conf.get_datasets()
-        # TODO: if extracting epic_entity on the fly is preferred rather than sending as param
-        # epic_entity = self._get_epic_entity()
-        # print(epic_entity)
-        file_paths_found = [file["rel_path"] for file in self._epic_entity["files"]]
-
+        file_paths_found = [file["rel_path"] for file in self._entity["files"]]
         found_images = [
             path for path in get_matches(
                 file_paths_found, IMAGE_PYRAMID_DIR + r".*\.ome\.tiff?$",
             )
         ]
-        found_images = sorted(found_images)
-        if len(found_images) == 0:  # pragma: no cover
+        # Remove the base-image pyramids from the found_images
+        filtered_images = [img_path for img_path in found_images if SEGMENTATION_SUPPORT_IMAGE_SUBDIR not in img_path]
+        if len(filtered_images) == 0:  # pragma: no cover
             message = f"Image pyramid assay with uuid {self._uuid} has no matching files"
             raise FileNotFoundError(message)
 
-        elif len(found_images) >= 1:
+        elif len(filtered_images) >= 1:
             img_url, offsets_url = self.segmentations_ome_offset_url(
-                found_images[0]
+                filtered_images[0]
             )
 
         segmentation_scale = self.read_segmentation_scale()
@@ -173,6 +169,11 @@ def read_segmentation_scale(self):  # pragma: no cover
 
 
 def create_segmentation_objects(base_url, mask_names):  # pragma: no cover
+    spatialTargetCMapping = {
+        'arteries-arterioles': 4,
+        'glomeruli': 2,
+        'tubules': 3,
+    }
     segmentation_objects = []
     segmentations_CL = []
     for index, mask_name in enumerate(mask_names):
@@ -186,9 +187,13 @@ def create_segmentation_objects(base_url, mask_names):  # pragma: no cover
                 "obsType": mask_name
             }
         )
+        # TODO: manually adjusted for the test dataset, need to be fixed on Vitessce side
+        if all(mask in mask_names for mask in spatialTargetCMapping.keys()):
+            channelIndex = spatialTargetCMapping[mask_name]
+        else:
+            channelIndex = index
         seg_CL = {
-            # TODO: manually to match image channels - need to be fixed on the JS side
-            "spatialTargetC": index + 2,
+            "spatialTargetC": channelIndex,
             "obsType": mask_name,
             "spatialChannelOpacity": 1,
             "spatialChannelColor": color_channel,

src/portal_visualization/builders/imaging_builders.py

@@ -11,7 +11,8 @@
 
 from ..utils import get_matches, group_by_file_name, get_conf_cells
 from ..paths import (IMAGE_PYRAMID_DIR, OFFSETS_DIR, SEQFISH_HYB_CYCLE_REGEX,
-                     SEQFISH_FILE_REGEX, SEGMENTATION_SUPPORT_IMAGE_SUBDIR)
+                     SEQFISH_FILE_REGEX, SEGMENTATION_SUPPORT_IMAGE_SUBDIR,
+                     SEGMENTATION_SUBDIR)
 from .base_builders import ViewConfBuilder
 
 
@@ -57,7 +58,7 @@ def _get_img_and_offset_url_seg(self, img_path, img_dir):
 
         """
         img_url = self._build_assets_url(img_path)
-        offset_path = f'{OFFSETS_DIR}/{SEGMENTATION_SUPPORT_IMAGE_SUBDIR}'
+        offset_path = f'{SEGMENTATION_SUBDIR}/{OFFSETS_DIR}/{SEGMENTATION_SUPPORT_IMAGE_SUBDIR}'
         return (
             img_url,
             str(
@@ -145,7 +146,7 @@ def __init__(self, entity, groups_token, assets_endpoint, **kwargs):
         i.e for high resolution viz-lifted imaging datasets like
         https://portal.hubmapconsortium.org/browse/dataset/
         """
-        self.image_pyramid_regex = f'{IMAGE_PYRAMID_DIR}/{SEGMENTATION_SUPPORT_IMAGE_SUBDIR}'
+        self.image_pyramid_regex = f'{SEGMENTATION_SUBDIR}/{IMAGE_PYRAMID_DIR}/{SEGMENTATION_SUPPORT_IMAGE_SUBDIR}'
         self.use_full_resolution = []
         self.use_physical_size_scaling = False
         super().__init__(entity, groups_token, assets_endpoint, **kwargs)

src/vis-preview.py

@@ -51,10 +51,6 @@ def main():  # pragma: no cover
         '--parent_uuid', metavar='UUID',
         help='Parent uuid - Only needed for an image-pyramid support dataset.',
         default=None)
-    parser.add_argument(
-        '--epic_url', help='URL which returns Dataset JSON for the EPIC dataset')
-    parser.add_argument(
-        '--epic_json', type=Path, help='File containing Dataset JSON for the EPIC dataset')
 
     args = parser.parse_args()
     marker = args.marker
@@ -64,11 +60,6 @@ def main():  # pragma: no cover
     headers = get_headers(args.token)
     entity = get_entity(args.url, args.json, headers)
 
-    if epic_uuid is not None:
-        if args.epic_url is None and args.epic_json is None:
-            raise ValueError('Provide the epic_url or epic_json parameter')
-        epic_entity = get_entity(args.epic_url, args.epic_json, headers)
-
     def get_assaytype(uuid):
         try:
             response = requests.get(f'{defaults["assaytypes_url"]}{uuid}', headers=headers)
@@ -90,7 +81,7 @@ def get_assaytype(uuid):
 
     if (epic_uuid is not None and conf_cells is not None):  # pragma: no cover
         EpicBuilder = get_epic_builder(epic_uuid)
-        epic_builder = EpicBuilder(epic_uuid, conf_cells, entity, epic_entity, args.token, args.assets_url)
+        epic_builder = EpicBuilder(epic_uuid, conf_cells, entity, args.token, args.assets_url)
         print(f'Using: {epic_builder.__class__.__name__}', file=stderr)
         conf_cells = epic_builder.get_conf_cells()