From 6884c8780f11e61ebf2d2f0be298cae8a2850578 Mon Sep 17 00:00:00 2001 From: brimoor Date: Thu, 24 Oct 2024 12:16:35 -0400 Subject: [PATCH 01/67] fix model zoo docs build --- docs/.gitignore | 2 +- docs/scripts/make_model_zoo_docs.py | 2 +- fiftyone/zoo/models/manifest-torch.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/.gitignore b/docs/.gitignore index 9ae01323f6..992724ddb3 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1 +1 @@ -/source/user_guide/model_zoo/models.rst +/source/model_zoo/models.rst diff --git a/docs/scripts/make_model_zoo_docs.py b/docs/scripts/make_model_zoo_docs.py index 21f1e77424..57d7bf32b9 100644 --- a/docs/scripts/make_model_zoo_docs.py +++ b/docs/scripts/make_model_zoo_docs.py @@ -423,7 +423,7 @@ def main(): # Write docs page docs_dir = "/".join(os.path.realpath(__file__).split("/")[:-2]) - outpath = os.path.join(docs_dir, "source/user_guide/model_zoo/models.rst") + outpath = os.path.join(docs_dir, "source/model_zoo/models.rst") print("Writing '%s'" % outpath) etau.write_file("\n".join(content), outpath) diff --git a/fiftyone/zoo/models/manifest-torch.json b/fiftyone/zoo/models/manifest-torch.json index 468d4d8004..be435bd193 100644 --- a/fiftyone/zoo/models/manifest-torch.json +++ b/fiftyone/zoo/models/manifest-torch.json @@ -452,7 +452,7 @@ "base_name": "med-sam-2-video-torch", "base_filename": "med-sam-2_pretrain.pth", "version": null, - "description": "Fine-tuned SAM2-hiera-tiny model from paper: Medical SAM 2 - Segment Medical Images as Video via Segment Anything Model 2 `_", + "description": "Fine-tuned SAM2-hiera-tiny model from `Medical SAM 2 - Segment Medical Images as Video via Segment Anything Model 2 `_", "source": "https://github.com/MedicineToken/Medical-SAM2", "size_bytes": 155906050, "manager": { From ab27a4e2e26ea94b3fc58e62db57eb147cd3fd22 Mon Sep 17 00:00:00 2001 From: Orvis Date: Tue, 22 Oct 2024 12:08:06 -0700 Subject: [PATCH 02/67] Fix documentation error --- docs/source/plugins/developing_plugins.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/plugins/developing_plugins.rst b/docs/source/plugins/developing_plugins.rst index 68b9a1c12a..6d5f655c4b 100644 --- a/docs/source/plugins/developing_plugins.rst +++ b/docs/source/plugins/developing_plugins.rst @@ -2242,7 +2242,7 @@ The example code below shows how to access and update panel state. def decrement(self, ctx): count = ctx.panel.get_state("v_stack.h_stack.count", 0) - ctx.panel.set_state("v_stack.h_stack.count", count + 1) + ctx.panel.set_state("v_stack.h_stack.count", count - 1) def render(self, ctx): panel = types.Object() From 3c68905a1e5afb8d49f36ff3cf63bffd5672aea6 Mon Sep 17 00:00:00 2001 From: brimoor Date: Thu, 17 Oct 2024 09:22:51 -0400 Subject: [PATCH 03/67] documenting ctx.user --- docs/source/plugins/developing_plugins.rst | 3 +++ fiftyone/operators/executor.py | 13 ++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/source/plugins/developing_plugins.rst b/docs/source/plugins/developing_plugins.rst index 6d5f655c4b..dca4b06bb8 100644 --- a/docs/source/plugins/developing_plugins.rst +++ b/docs/source/plugins/developing_plugins.rst @@ -981,6 +981,9 @@ contains the following properties: - `ctx.extended_selection` - the extended selection of the view, if any - `ctx.group_slice` - the active group slice in the App, if any - `ctx.user_id` - the ID of the user that invoked the operator, if known +- `ctx.user` - an object of information about the user that invoked the + operator, if known, including the user's `id`, `name`, `email`, `role`, and + `dataset_permission` - `ctx.panel_id` - the ID of the panel that invoked the operator, if any - `ctx.panel` - a :class:`PanelRef ` instance that you can use to read and write the :ref:`state ` diff --git a/fiftyone/operators/executor.py b/fiftyone/operators/executor.py index 8b5da0d5ea..2eb2eb3621 100644 --- a/fiftyone/operators/executor.py +++ b/fiftyone/operators/executor.py @@ -385,7 +385,7 @@ async def resolve_type(registry, operator_uri, request_params): return ExecutionResult(error=traceback.format_exc()) -async def resolve_type_with_context(request_params, target: str = None): +async def resolve_type_with_context(request_params, target=None): """Resolves the "inputs" or "outputs" schema of an operator with the given context. @@ -486,7 +486,7 @@ def __init__( self._dataset = None self._view = None self._ops = Operations(self) - self.user = None + self._user = None self._set_progress = set_progress self._delegated_operation_id = delegated_operation_id @@ -646,7 +646,14 @@ def current_sample(self): @property def user_id(self): """The ID of the user executing the operation, if known.""" - return self.user.id if self.user else None + return self._user.id if self._user else None + + @property + def user(self): + """An object of information about the user executing the operation, if + known. + """ + return self._user @property def panel_id(self): From 8a371068f6e5458d68d91517ab87d3f8c2395b94 Mon Sep 17 00:00:00 2001 From: brimoor Date: Thu, 17 Oct 2024 09:36:18 -0400 Subject: [PATCH 04/67] adding user_request_token --- docs/source/plugins/developing_plugins.rst | 11 +++++++---- fiftyone/operators/executor.py | 7 +++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/source/plugins/developing_plugins.rst b/docs/source/plugins/developing_plugins.rst index dca4b06bb8..9e157c595f 100644 --- a/docs/source/plugins/developing_plugins.rst +++ b/docs/source/plugins/developing_plugins.rst @@ -980,10 +980,13 @@ contains the following properties: if any - `ctx.extended_selection` - the extended selection of the view, if any - `ctx.group_slice` - the active group slice in the App, if any -- `ctx.user_id` - the ID of the user that invoked the operator, if known -- `ctx.user` - an object of information about the user that invoked the - operator, if known, including the user's `id`, `name`, `email`, `role`, and - `dataset_permission` +- `ctx.user_id` **(Teams only)** - the ID of the user that invoked the + operator, if known +- `ctx.user` **(Teams only)** - an object of information about the user that + invoked the operator, if known, including the user's `id`, `name`, `email`, + `role`, and `dataset_permission` +- `ctx.user_request_token` **(Teams only)** - the request token + authenticating the user executing the operation, if known - `ctx.panel_id` - the ID of the panel that invoked the operator, if any - `ctx.panel` - a :class:`PanelRef ` instance that you can use to read and write the :ref:`state ` diff --git a/fiftyone/operators/executor.py b/fiftyone/operators/executor.py index 2eb2eb3621..e8bca4cf04 100644 --- a/fiftyone/operators/executor.py +++ b/fiftyone/operators/executor.py @@ -655,6 +655,13 @@ def user(self): """ return self._user + @property + def user_request_token(self): + """The request token authenticating the user executing the operation, + if known. + """ + return self._user._request_token if self._user else None + @property def panel_id(self): """The ID of the panel that invoked the operator, if any.""" From 0048f7b387dea6feabef91bfacb69a43008b361f Mon Sep 17 00:00:00 2001 From: brimoor Date: Thu, 17 Oct 2024 09:42:20 -0400 Subject: [PATCH 05/67] removing Teams-only tag --- docs/source/plugins/developing_plugins.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/source/plugins/developing_plugins.rst b/docs/source/plugins/developing_plugins.rst index 9e157c595f..a4a6c4e464 100644 --- a/docs/source/plugins/developing_plugins.rst +++ b/docs/source/plugins/developing_plugins.rst @@ -980,13 +980,12 @@ contains the following properties: if any - `ctx.extended_selection` - the extended selection of the view, if any - `ctx.group_slice` - the active group slice in the App, if any -- `ctx.user_id` **(Teams only)** - the ID of the user that invoked the - operator, if known -- `ctx.user` **(Teams only)** - an object of information about the user that - invoked the operator, if known, including the user's `id`, `name`, `email`, - `role`, and `dataset_permission` -- `ctx.user_request_token` **(Teams only)** - the request token - authenticating the user executing the operation, if known +- `ctx.user_id` - the ID of the user that invoked the operator, if known +- `ctx.user` - an object of information about the user that invoked the + operator, if known, including the user's `id`, `name`, `email`, `role`, and + `dataset_permission` +- `ctx.user_request_token` - the request token authenticating the user + executing the operation, if known - `ctx.panel_id` - the ID of the panel that invoked the operator, if any - `ctx.panel` - a :class:`PanelRef ` instance that you can use to read and write the :ref:`state ` From 842efa80ef43b024958ad9f5a6c36887100f6211 Mon Sep 17 00:00:00 2001 From: brimoor Date: Thu, 17 Oct 2024 10:39:33 -0400 Subject: [PATCH 06/67] must use instance variable --- fiftyone/operators/executor.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/fiftyone/operators/executor.py b/fiftyone/operators/executor.py index e8bca4cf04..14fd9b504f 100644 --- a/fiftyone/operators/executor.py +++ b/fiftyone/operators/executor.py @@ -482,11 +482,11 @@ def __init__( self.request_params = request_params or {} self.params = self.request_params.get("params", {}) self.executor = executor + self.user = None self._dataset = None self._view = None self._ops = Operations(self) - self._user = None self._set_progress = set_progress self._delegated_operation_id = delegated_operation_id @@ -646,21 +646,14 @@ def current_sample(self): @property def user_id(self): """The ID of the user executing the operation, if known.""" - return self._user.id if self._user else None - - @property - def user(self): - """An object of information about the user executing the operation, if - known. - """ - return self._user + return self.user.id if self.user else None @property def user_request_token(self): """The request token authenticating the user executing the operation, if known. """ - return self._user._request_token if self._user else None + return self.user._request_token if self.user else None @property def panel_id(self): From 6c0b581f05869260178e86d6e6bbf8fc379a1a34 Mon Sep 17 00:00:00 2001 From: Jacob Marks Date: Thu, 17 Oct 2024 10:22:21 -0400 Subject: [PATCH 07/67] set model to eval model --- fiftyone/utils/open_clip.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fiftyone/utils/open_clip.py b/fiftyone/utils/open_clip.py index 4824248443..f462284289 100644 --- a/fiftyone/utils/open_clip.py +++ b/fiftyone/utils/open_clip.py @@ -95,6 +95,7 @@ def _load_model(self, config): device=self.device, ) self._tokenizer = open_clip.get_tokenizer(config.clip_model) + self._model.eval() return self._model def _get_text_features(self): From 44c04b836fae0fedd4c34391a60b6d3e9e3e9802 Mon Sep 17 00:00:00 2001 From: Jacob Marks Date: Thu, 17 Oct 2024 10:22:49 -0400 Subject: [PATCH 08/67] document open_clip eval() mode --- docs/source/integrations/openclip.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/source/integrations/openclip.rst b/docs/source/integrations/openclip.rst index a3f2653564..47ac3e66fb 100644 --- a/docs/source/integrations/openclip.rst +++ b/docs/source/integrations/openclip.rst @@ -88,6 +88,11 @@ When running inference with OpenCLIP, you can specify a text prompt to help guide the model towards a solution as well as only specify a certain number of classes to output during zero shot classification. +.. note:: + + While OpenCLIP models are typically set to train mode by default, the FiftyOne + integration sets the model to eval mode before running inference. + For example we can run inference as such: .. code-block:: python From e8be6b600b7ca88dbb4f0add92214cf80a8fa21d Mon Sep 17 00:00:00 2001 From: Jacob Marks Date: Thu, 17 Oct 2024 10:28:28 -0400 Subject: [PATCH 09/67] update amp autocast --- fiftyone/utils/open_clip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fiftyone/utils/open_clip.py b/fiftyone/utils/open_clip.py index f462284289..6c0f4cd253 100644 --- a/fiftyone/utils/open_clip.py +++ b/fiftyone/utils/open_clip.py @@ -145,7 +145,7 @@ def _predict_all(self, imgs): if self._using_gpu: imgs = imgs.cuda() - with torch.no_grad(), torch.cuda.amp.autocast(): + with torch.no_grad(), torch.amp.autocast("cuda"): image_features = self._model.encode_image(imgs) text_features = self._get_text_features() From 69d4a7ed43fdc67ee3818aa155b9ef27edee597f Mon Sep 17 00:00:00 2001 From: Jacob Marks Date: Thu, 17 Oct 2024 10:39:31 -0400 Subject: [PATCH 10/67] update HF dataset repo ids --- docs/source/integrations/huggingface.rst | 58 ++++++++++++------------ 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/docs/source/integrations/huggingface.rst b/docs/source/integrations/huggingface.rst index ac553e62bc..e7fd8851d0 100644 --- a/docs/source/integrations/huggingface.rst +++ b/docs/source/integrations/huggingface.rst @@ -1309,7 +1309,7 @@ If the repo was uploaded to the Hugging Face Hub via FiftyOne's :func:`push_to_hub() ` function, then the `fiftyone.yml` config file will be generated and uploaded to the repo. However, some common datasets like -`mnist `_ were uploaded to the Hub +`mnist `_ were uploaded to the Hub using the `datasets` library and do not contain a `fiftyone.yml` or `fiftyone.yaml` file. If you know how the dataset is structured, you can load the dataset by passing the path to a local yaml config file that describes the @@ -1332,7 +1332,7 @@ the path to the local yaml config file: from fiftyone.utils.huggingface import load_from_hub dataset = load_from_hub( - "mnist", + "ylecun/mnist", config_file="/path/to/mnist.yml", ) @@ -1360,7 +1360,7 @@ and `classification_fields` arguments directly: from fiftyone.utils.huggingface import load_from_hub dataset = load_from_hub( - "mnist", + "ylecun/mnist", format="ParquetFilesDataset", classification_fields="label", ) @@ -1400,7 +1400,7 @@ Let's look at these categories in more detail: dataset that are *compatible* with this config, and are *available* to be loaded. In Hugging Face, the "dataset" in a repo can contain multiple "subsets", which may or may not have the same schema. Take the - `Street View House Numbers `_ dataset for + `Street View House Numbers `_ dataset for example. This dataset has two subsets: `"cropped_digits"` and `"full_numbers"`. The `cropped_digits` subset contains classification labels, while the `full_numbers` subset contains detection labels. A single config would not be @@ -1419,7 +1419,7 @@ Let's look at these categories in more detail: identifies the names of all splits and by default, will assume that all of these splits are to be loaded. If you only want to load a specific split or splits, you can specify them with the `splits` field. For example, to load the - training split of the `CIFAR10 `_ + training split of the `CIFAR10 `_ dataset, you can pass `splits="train"`. If you want to load multiple splits, you can pass them as a list, e.g., `splits=["train", "test"]`. Note that this is not a required field, and by default all splits are loaded. @@ -1554,8 +1554,8 @@ easy it is in practice to load datasets from the Hugging Face Hub. **Classification Datasets** Let's start by loading the -`MNIST `_ dataset into FiftyOne. All you -need to do is pass the `repo_id` of the dataset — in this case `"mnist"` — to +`MNIST `_ dataset into FiftyOne. All you +need to do is pass the `repo_id` of the dataset — in this case `"ylecun/mnist"` — to :func:`load_from_hub() `, specify the format as `"parquet"`, and specify the `classification_fields` as `"label"`: @@ -1565,7 +1565,7 @@ format as `"parquet"`, and specify the `classification_fields` as `"label"`: from fiftyone.utils.huggingface import load_from_hub dataset = load_from_hub( - "mnist", + "ylecun/mnist", format="parquet", classification_fields="label", max_samples=1000, @@ -1574,25 +1574,25 @@ format as `"parquet"`, and specify the `classification_fields` as `"label"`: session = fo.launch_app(dataset) The same exact syntax works for the `CIFAR-10 `_ -and `FashionMNIST `_ datasets, +and `FashionMNIST `_ datasets, which are also available on the Hub. In fact, you can load any of the following classification datasets from the Hub using the same syntax, just by changing the `repo_id`: -- `CIFAR-10 `_ (use `"cifar10"`) -- `ImageNet `_ (use `"imagenet-1k"`) -- `FashionMNIST `_ (use `"fashion_mnist"`) +- `CIFAR-10 `_ (use `"uoft-cs/cifar10"`) +- `ImageNet `_ (use `"ILSVRC/imagenet-1k"`) +- `FashionMNIST `_ (use `"zalando-datasets/fashion_mnist"`) - `Tiny ImageNet `_ (use `"zh-plus/tiny-imagenet"`) -- `Food-101 `_ (use `"food101"`) -- `Dog Food `_ (use `"sasha/dog-food"`) -- `ImageNet-Sketch `_ (use `"imagenet_sketch"`) +- `Food-101 `_ (use `"ethz/food101"`) +- `Dog Food `_ (use `"sasha/dog-food"`) +- `ImageNet-Sketch `_ (use `"songweig/imagenet_sketch"`) - `Oxford Flowers `_ (use `"nelorth/oxford-flowers"`) -- `Cats vs. Dogs `_ (use `"cats_vs_dogs"`) +- `Cats vs. Dogs `_ (use `"microsoft/cats_vs_dogs"`) - `ObjectNet-1.0 `_ (use `"timm/objectnet"`) A very similar syntax can be used to load classification datasets that contain *multiple* classification fields, such as -`CIFAR-100 `_ and the +`CIFAR-100 `_ and the `WikiArt `_ dataset. For example, to load the CIFAR-100 dataset, you can specify the `classification_fields` as `["coarse_label", "fine_label"]`: @@ -1603,7 +1603,7 @@ to load the CIFAR-100 dataset, you can specify the `classification_fields` as from fiftyone.utils.huggingface import load_from_hub dataset = load_from_hub( - "cifar100", + "uoft-cs/cifar100", format="parquet", classification_fields=["coarse_label", "fine_label"], max_samples=1000, @@ -1638,7 +1638,7 @@ dataset. For example, to load the `cropped_digits` subset of the from fiftyone.utils.huggingface import load_from_hub dataset = load_from_hub( - "svhn", + "ufldl-stanford/svhn", format="parquet", classification_fields="label", subsets="cropped_digits", @@ -1671,8 +1671,8 @@ standard column name for detection features in Hugging Face datasets: The same syntax works for many other popular detection datasets on the Hub, including: -- `CPPE - 5 `_ (use `"cppe-5"`) -- `WIDER FACE `_ (use `"wider_face"`) +- `CPPE - 5 `_ (use `"rishitdagli/cppe-5"`) +- `WIDER FACE `_ (use `"CUHK-CSE/wider_face"`) - `License Plate Object Detection `_ (use `"keremberke/license-plate-object-detection"`) - `Aerial Sheep Object Detection `_ @@ -1680,7 +1680,7 @@ including: Some detection datasets have their detections stored under a column with a different name. For example, the `full_numbers` subset of the -`Street View House Numbers `_ dataset +`Street View House Numbers `_ dataset stores its detections under the column `digits`. To load this subset, you can specify the `detection_fields` as `"digits"`: @@ -1690,7 +1690,7 @@ specify the `detection_fields` as `"digits"`: from fiftyone.utils.huggingface import load_from_hub dataset = load_from_hub( - "svhn", + "ufldl-stanford/svhn", format="parquet", detection_fields="digits", subsets="full_numbers", @@ -1711,7 +1711,7 @@ specify the `detection_fields` as `"digits"`: Loading segmentation datasets from the Hub is also a breeze. For example, to load the "instance_segmentation" subset from -`SceneParse150 `_, all you +`SceneParse150 `_, all you need to do is specify the `mask_fields` as `"annotation"`: .. code-block:: python @@ -1720,7 +1720,7 @@ need to do is specify the `mask_fields` as `"annotation"`: from fiftyone.utils.huggingface import load_from_hub dataset = load_from_hub( - "scene_parse150", + "zhoubolei/scene_parse150", format="parquet", subsets="instance_segmentation", mask_fields="annotation", @@ -1838,7 +1838,7 @@ need to specify the `filepath` as `"url"`: session = fo.launch_app(dataset) -For `RedCaps `_, we instead use +For `RedCaps `_, we instead use `"image_url"` as the `filepath`: .. code-block:: python @@ -1847,7 +1847,7 @@ For `RedCaps `_, we instead use from fiftyone.utils.huggingface import load_from_hub dataset = load_from_hub( - "red_caps", + "kdexd/red_caps", format="parquet", filepath="image_url", max_samples=1000, @@ -1944,7 +1944,7 @@ Now, you can load the dataset using the local yaml config file: When loading datasets from the Hub, you can customize the download process by specifying the `batch_size`, `num_workers`, and `overwrite` arguments. For example, to download the `full_numbers` subset of the `Street View House Numbers -`_ dataset with a batch size of 50 and 4 +`_ dataset with a batch size of 50 and 4 workers, you can do the following: .. code-block:: python @@ -1953,7 +1953,7 @@ workers, you can do the following: from fiftyone.utils.huggingface import load_from_hub dataset = load_from_hub( - "svhn", + "ufldl-stanford/svhn", format="parquet", detection_fields="digits", subsets="full_numbers", From e009587331b3a6b6c0e5ea9ac49cafed5de1fb86 Mon Sep 17 00:00:00 2001 From: Jacob Marks Date: Thu, 17 Oct 2024 10:53:59 -0400 Subject: [PATCH 11/67] add depth anything models to integration docs --- docs/source/integrations/huggingface.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/source/integrations/huggingface.rst b/docs/source/integrations/huggingface.rst index e7fd8851d0..b7aaccda7e 100644 --- a/docs/source/integrations/huggingface.rst +++ b/docs/source/integrations/huggingface.rst @@ -402,6 +402,15 @@ method: from transformers import GLPNForDepthEstimation model = GLPNForDepthEstimation.from_pretrained("vinvino02/glpn-kitti") + # Depth Anything + from transformers import AutoModelForDepthEstimation + model = AutoModelForDepthEstimation.from_pretrained("LiheYoung/depth-anything-small-hf") + + # Depth Anything-V2 + from transformers import AutoModelForDepthEstimation + model = AutoModelForDepthEstimation.from_pretrained("depth-anything/Depth-Anything-V2-Small-hf") + + .. code-block:: python :linenos: From 5e4f01b5a59b648a6267fa0046379b8c86dd2e46 Mon Sep 17 00:00:00 2001 From: brimoor Date: Wed, 16 Oct 2024 08:44:03 -0400 Subject: [PATCH 12/67] clarifying model vs source metadata for remote zoo models --- docs/source/model_zoo/remote.rst | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/source/model_zoo/remote.rst b/docs/source/model_zoo/remote.rst index bfb6242fb4..fe513589d0 100644 --- a/docs/source/model_zoo/remote.rst +++ b/docs/source/model_zoo/remote.rst @@ -231,10 +231,6 @@ model(s) that it contains: +----------------------------------+-----------+-------------------------------------------------------------------------------------------+ | Field | Required? | Description | +==================================+===========+===========================================================================================+ - | `name` | | A name for the remote model source | - +----------------------------------+-----------+-------------------------------------------------------------------------------------------+ - | `url` | | The URL of the remote model source | - +----------------------------------+-----------+-------------------------------------------------------------------------------------------+ | `base_name` | **yes** | The base name of the model (no version info) | +----------------------------------+-----------+-------------------------------------------------------------------------------------------+ | `base_filename` | | The base filename or directory of the model (no version info), if applicable. | @@ -279,6 +275,19 @@ model(s) that it contains: | | | must be provided | +----------------------------------+-----------+-------------------------------------------------------------------------------------------+ +It can also provide optional metadata about the remote source itself: + +.. table:: + :widths: 20,10,70 + + +----------------------------------+-----------+-------------------------------------------------------------------------------------------+ + | Field | Required? | Description | + +==================================+===========+===========================================================================================+ + | `name` | | A name for the remote model source | + +----------------------------------+-----------+-------------------------------------------------------------------------------------------+ + | `url` | | The URL of the remote model source | + +----------------------------------+-----------+-------------------------------------------------------------------------------------------+ + Here's an exaxmple model manifest file that declares a single model: .. code-block:: json From dd3c8de0094e8b5b9ade7c7b16f5a0d57124fe63 Mon Sep 17 00:00:00 2001 From: Jacob Marks Date: Sat, 5 Oct 2024 11:50:09 -0400 Subject: [PATCH 13/67] rm unused import --- fiftyone/utils/ultralytics.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fiftyone/utils/ultralytics.py b/fiftyone/utils/ultralytics.py index c5473a669d..6eafa7eab2 100644 --- a/fiftyone/utils/ultralytics.py +++ b/fiftyone/utils/ultralytics.py @@ -17,7 +17,6 @@ from fiftyone.core.models import Model import fiftyone.utils.torch as fout import fiftyone.core.utils as fou -import fiftyone.zoo as foz import fiftyone.zoo.models as fozm ultralytics = fou.lazy_import("ultralytics") From 7b15892b2995dcdb6a633d389b50640801be4129 Mon Sep 17 00:00:00 2001 From: Jacob Marks Date: Sat, 5 Oct 2024 11:51:26 -0400 Subject: [PATCH 14/67] fix yolonas tag --- fiftyone/zoo/models/manifest-torch.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fiftyone/zoo/models/manifest-torch.json b/fiftyone/zoo/models/manifest-torch.json index be435bd193..3b24b4f256 100644 --- a/fiftyone/zoo/models/manifest-torch.json +++ b/fiftyone/zoo/models/manifest-torch.json @@ -3796,7 +3796,7 @@ "support": true } }, - "tags": ["classification", "torch", "yolo"], + "tags": ["detection", "torch", "yolo"], "date_added": "2024-01-06 08:51:14" }, { From 1ca5e9324e2a531aaf4e0b42dca69b6f9cc16992 Mon Sep 17 00:00:00 2001 From: Jacob Marks Date: Sat, 5 Oct 2024 12:22:51 -0400 Subject: [PATCH 15/67] add yolov11 det models to zoo --- fiftyone/zoo/models/manifest-torch.json | 160 ++++++++++++++++++++++++ 1 file changed, 160 insertions(+) diff --git a/fiftyone/zoo/models/manifest-torch.json b/fiftyone/zoo/models/manifest-torch.json index 3b24b4f256..00d8a83f15 100644 --- a/fiftyone/zoo/models/manifest-torch.json +++ b/fiftyone/zoo/models/manifest-torch.json @@ -3260,6 +3260,166 @@ "tags": ["detection", "coco", "torch", "yolo"], "date_added": "2024-07-01 19:22:51" }, + { + "base_name": "yolov11n-coco-torch", + "base_filename": "yolov11n-coco.pt", + "description": "YOLOv11-N model trained on COCO", + "source": "https://docs.ultralytics.com/models/yolov11/", + "size_bytes": 5613764, + "manager": { + "type": "fiftyone.core.models.ModelManager", + "config": { + "url": "https://github.com/ultralytics/assets/releases/download/v8.3.0/yolo11n.pt" + } + }, + "default_deployment_config_dict": { + "type": "fiftyone.utils.ultralytics.FiftyOneYOLODetectionModel", + "config": {} + }, + "requirements": { + "packages": [ + "torch>=1.7.0", + "torchvision>=0.8.1", + "ultralytics>=8.3.0" + ], + "cpu": { + "support": true + }, + "gpu": { + "support": true + } + }, + "tags": ["detection", "coco", "torch", "yolo"], + "date_added": "2024-10-05 19:22:51" + }, + { + "base_name": "yolov11s-coco-torch", + "base_filename": "yolov11s-coco.pt", + "description": "YOLOv11-S model trained on COCO", + "source": "https://docs.ultralytics.com/models/yolov11/", + "size_bytes": 19313732, + "manager": { + "type": "fiftyone.core.models.ModelManager", + "config": { + "url": "https://github.com/ultralytics/assets/releases/download/v8.3.0/yolo11s.pt" + } + }, + "default_deployment_config_dict": { + "type": "fiftyone.utils.ultralytics.FiftyOneYOLODetectionModel", + "config": {} + }, + "requirements": { + "packages": [ + "torch>=1.7.0", + "torchvision>=0.8.1", + "ultralytics>=8.3.0" + ], + "cpu": { + "support": true + }, + "gpu": { + "support": true + } + }, + "tags": ["detection", "coco", "torch", "yolo"], + "date_added": "2024-10-05 19:22:51" + }, + { + "base_name": "yolov11m-coco-torch", + "base_filename": "yolov11m-coco.pt", + "description": "YOLOv11-M model trained on COCO", + "source": "https://docs.ultralytics.com/models/yolov11/", + "size_bytes": 40684120, + "manager": { + "type": "fiftyone.core.models.ModelManager", + "config": { + "url": "https://github.com/ultralytics/assets/releases/download/v8.3.0/yolo11m.pt" + } + }, + "default_deployment_config_dict": { + "type": "fiftyone.utils.ultralytics.FiftyOneYOLODetectionModel", + "config": {} + }, + "requirements": { + "packages": [ + "torch>=1.7.0", + "torchvision>=0.8.1", + "ultralytics>=8.3.0" + ], + "cpu": { + "support": true + }, + "gpu": { + "support": true + } + }, + "tags": ["detection", "coco", "torch", "yolo"], + "date_added": "2024-10-05 19:22:51" + }, + { + "base_name": "yolov11l-coco-torch", + "base_filename": "yolov11l-coco.pt", + "description": "YOLOv11-L model trained on COCO", + "source": "https://docs.ultralytics.com/models/yolov11/", + "size_bytes": 51387343, + "manager": { + "type": "fiftyone.core.models.ModelManager", + "config": { + "url": "https://github.com/ultralytics/assets/releases/download/v8.3.0/yolo11l.pt" + } + }, + "default_deployment_config_dict": { + "type": "fiftyone.utils.ultralytics.FiftyOneYOLODetectionModel", + "config": {} + }, + "requirements": { + "packages": [ + "torch>=1.7.0", + "torchvision>=0.8.1", + "ultralytics>=8.3.0" + ], + "cpu": { + "support": true + }, + "gpu": { + "support": true + } + }, + "tags": ["detection", "coco", "torch", "yolo"], + "date_added": "2024-10-05 19:22:51" + }, + { + "base_name": "yolov11x-coco-torch", + "base_filename": "yolov11x-coco.pt", + "description": "YOLOv11-X model trained on COCO", + "source": "https://docs.ultralytics.com/models/yolov11/", + "size_bytes": 114636239, + "manager": { + "type": "fiftyone.core.models.ModelManager", + "config": { + "url": "https://github.com/ultralytics/assets/releases/download/v8.3.0/yolo11x.pt" + } + }, + "default_deployment_config_dict": { + "type": "fiftyone.utils.ultralytics.FiftyOneYOLODetectionModel", + "config": {} + }, + "requirements": { + "packages": [ + "torch>=1.7.0", + "torchvision>=0.8.1", + "ultralytics>=8.3.0" + ], + "cpu": { + "support": true + }, + "gpu": { + "support": true + } + }, + "tags": ["detection", "coco", "torch", "yolo"], + "date_added": "2024-10-05 19:22:51" + }, { "base_name": "rtdetr-l-coco-torch", "base_filename": "rtdetr-l-coco.pt", From 6c5387f962472715282088ed35e4605c003fb1ae Mon Sep 17 00:00:00 2001 From: Jacob Marks Date: Sat, 5 Oct 2024 12:26:52 -0400 Subject: [PATCH 16/67] update syntax to match ultralytics --- fiftyone/zoo/models/manifest-torch.json | 30 ++++++++++++------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/fiftyone/zoo/models/manifest-torch.json b/fiftyone/zoo/models/manifest-torch.json index 00d8a83f15..d23fa4fc5d 100644 --- a/fiftyone/zoo/models/manifest-torch.json +++ b/fiftyone/zoo/models/manifest-torch.json @@ -3261,9 +3261,9 @@ "date_added": "2024-07-01 19:22:51" }, { - "base_name": "yolov11n-coco-torch", - "base_filename": "yolov11n-coco.pt", - "description": "YOLOv11-N model trained on COCO", + "base_name": "yolo11n-coco-torch", + "base_filename": "yolo11n-coco.pt", + "description": "YOLO11-N model trained on COCO", "source": "https://docs.ultralytics.com/models/yolov11/", "size_bytes": 5613764, "manager": { @@ -3293,9 +3293,9 @@ "date_added": "2024-10-05 19:22:51" }, { - "base_name": "yolov11s-coco-torch", - "base_filename": "yolov11s-coco.pt", - "description": "YOLOv11-S model trained on COCO", + "base_name": "yolo11s-coco-torch", + "base_filename": "yolo11s-coco.pt", + "description": "YOLO11-S model trained on COCO", "source": "https://docs.ultralytics.com/models/yolov11/", "size_bytes": 19313732, "manager": { @@ -3325,9 +3325,9 @@ "date_added": "2024-10-05 19:22:51" }, { - "base_name": "yolov11m-coco-torch", - "base_filename": "yolov11m-coco.pt", - "description": "YOLOv11-M model trained on COCO", + "base_name": "yolo11m-coco-torch", + "base_filename": "yolo11m-coco.pt", + "description": "YOLO11-M model trained on COCO", "source": "https://docs.ultralytics.com/models/yolov11/", "size_bytes": 40684120, "manager": { @@ -3357,9 +3357,9 @@ "date_added": "2024-10-05 19:22:51" }, { - "base_name": "yolov11l-coco-torch", - "base_filename": "yolov11l-coco.pt", - "description": "YOLOv11-L model trained on COCO", + "base_name": "yolo11l-coco-torch", + "base_filename": "yolo11l-coco.pt", + "description": "YOLO11-L model trained on COCO", "source": "https://docs.ultralytics.com/models/yolov11/", "size_bytes": 51387343, "manager": { @@ -3389,9 +3389,9 @@ "date_added": "2024-10-05 19:22:51" }, { - "base_name": "yolov11x-coco-torch", - "base_filename": "yolov11x-coco.pt", - "description": "YOLOv11-X model trained on COCO", + "base_name": "yolo11x-coco-torch", + "base_filename": "yolo11x-coco.pt", + "description": "YOLO11-X model trained on COCO", "source": "https://docs.ultralytics.com/models/yolov11/", "size_bytes": 114636239, "manager": { From eef88acd5f9d546ec1e0c11b34f8273a66394db1 Mon Sep 17 00:00:00 2001 From: Jacob Marks Date: Sat, 5 Oct 2024 12:27:07 -0400 Subject: [PATCH 17/67] add yolo11 det models to docs --- docs/source/integrations/ultralytics.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/source/integrations/ultralytics.rst b/docs/source/integrations/ultralytics.rst index 3ce0946233..79bef30e7d 100644 --- a/docs/source/integrations/ultralytics.rst +++ b/docs/source/integrations/ultralytics.rst @@ -105,6 +105,13 @@ You can directly pass Ultralytics `YOLO` or `RTDETR` detection models to # model = YOLO("yolov10l.pt) # model = YOLO("yolov10x.pt) + # YOLOv11 + # model = YOLO("yolo11n.pt) + # model = YOLO("yolo11s.pt) + # model = YOLO("yolo11m.pt) + # model = YOLO("yolo11l.pt) + # model = YOLO("yolo11x.pt) + # RTDETR # model = YOLO("rtdetr-l.pt") # model = YOLO("rtdetr-x.pt") @@ -140,6 +147,7 @@ You can also load any of these models directly from the # model_name = "yolov8m-coco-torch" # model_name = "yolov9e-coco-torch" # model_name = "yolov10s-coco-torch" + # model_name = "yolo11x-coco-torch" # model_name = "rtdetr-l-coco-torch" model = foz.load_zoo_model( From 6757a66a902a59c5241d9ac265cf2e38b6b9360a Mon Sep 17 00:00:00 2001 From: Jacob Marks Date: Sat, 5 Oct 2024 12:36:44 -0400 Subject: [PATCH 18/67] add yolo11seg models to manifest --- fiftyone/zoo/models/manifest-torch.json | 160 ++++++++++++++++++++++++ 1 file changed, 160 insertions(+) diff --git a/fiftyone/zoo/models/manifest-torch.json b/fiftyone/zoo/models/manifest-torch.json index d23fa4fc5d..8d00620325 100644 --- a/fiftyone/zoo/models/manifest-torch.json +++ b/fiftyone/zoo/models/manifest-torch.json @@ -3420,6 +3420,166 @@ "tags": ["detection", "coco", "torch", "yolo"], "date_added": "2024-10-05 19:22:51" }, + { + "base_name": "yolo11n-seg-coco-torch", + "base_filename": "yolo11n-seg-coco.pt", + "description": "YOLO11-N Segmentation model trained on COCO", + "source": "https://docs.ultralytics.com/models/yolo11/#__tabbed_1_2", + "size_bytes": 6182636, + "manager": { + "type": "fiftyone.core.models.ModelManager", + "config": { + "url": "https://github.com/ultralytics/assets/releases/download/v8.3.0/yolo11n-seg.pt" + } + }, + "default_deployment_config_dict": { + "type": "fiftyone.utils.ultralytics.FiftyOneYOLOSegmentationModel", + "config": {} + }, + "requirements": { + "packages": [ + "torch>=1.7.0", + "torchvision>=0.8.1", + "ultralytics>=8.3.0" + ], + "cpu": { + "support": true + }, + "gpu": { + "support": true + } + }, + "tags": ["segmentation", "coco", "torch", "yolo"], + "date_added": "2024-10-05 19:22:51" + }, + { + "base_name": "yolo11s-seg-coco-torch", + "base_filename": "yolo11s-seg-coco.pt", + "description": "YOLO11-S Segmentation model trained on COCO", + "source": "https://docs.ultralytics.com/models/yolo11/#__tabbed_1_2", + "size_bytes": 20669228, + "manager": { + "type": "fiftyone.core.models.ModelManager", + "config": { + "url": "https://github.com/ultralytics/assets/releases/download/v8.3.0/yolo11s-seg.pt" + } + }, + "default_deployment_config_dict": { + "type": "fiftyone.utils.ultralytics.FiftyOneYOLOSegmentationModel", + "config": {} + }, + "requirements": { + "packages": [ + "torch>=1.7.0", + "torchvision>=0.8.1", + "ultralytics>=8.3.0" + ], + "cpu": { + "support": true + }, + "gpu": { + "support": true + } + }, + "tags": ["segmentation", "coco", "torch", "yolo"], + "date_added": "2024-10-05 19:22:51" + }, + { + "base_name": "yolo11m-seg-coco-torch", + "base_filename": "yolo11m-seg-coco.pt", + "description": "YOLO11-M Segmentation model trained on COCO", + "source": "https://docs.ultralytics.com/models/yolo11/#__tabbed_1_2", + "size_bytes": 45400152, + "manager": { + "type": "fiftyone.core.models.ModelManager", + "config": { + "url": "https://github.com/ultralytics/assets/releases/download/v8.3.0/yolo11m-seg.pt" + } + }, + "default_deployment_config_dict": { + "type": "fiftyone.utils.ultralytics.FiftyOneYOLOSegmentationModel", + "config": {} + }, + "requirements": { + "packages": [ + "torch>=1.7.0", + "torchvision>=0.8.1", + "ultralytics>=8.3.0" + ], + "cpu": { + "support": true + }, + "gpu": { + "support": true + } + }, + "tags": ["segmentation", "coco", "torch", "yolo"], + "date_added": "2024-10-05 19:22:51" + }, + { + "base_name": "yolo11l-seg-coco-torch", + "base_filename": "yolo11l-seg-coco.pt", + "description": "YOLO11-L Segmentation model trained on COCO", + "source": "https://docs.ultralytics.com/models/yolo11/#__tabbed_1_2", + "size_bytes": 56096965, + "manager": { + "type": "fiftyone.core.models.ModelManager", + "config": { + "url": "https://github.com/ultralytics/assets/releases/download/v8.3.0/yolo11l-seg.pt" + } + }, + "default_deployment_config_dict": { + "type": "fiftyone.utils.ultralytics.FiftyOneYOLOSegmentationModel", + "config": {} + }, + "requirements": { + "packages": [ + "torch>=1.7.0", + "torchvision>=0.8.1", + "ultralytics>=8.3.0" + ], + "cpu": { + "support": true + }, + "gpu": { + "support": true + } + }, + "tags": ["segmentation", "coco", "torch", "yolo"], + "date_added": "2024-10-05 19:22:51" + }, + { + "base_name": "yolo11x-seg-coco-torch", + "base_filename": "yolo11x-seg-coco.pt", + "description": "YOLO11-X Segmentation model trained on COCO", + "source": "https://docs.ultralytics.com/models/yolo11/#__tabbed_1_2", + "size_bytes": 125090821, + "manager": { + "type": "fiftyone.core.models.ModelManager", + "config": { + "url": "https://github.com/ultralytics/assets/releases/download/v8.3.0/yolo11x-seg.pt" + } + }, + "default_deployment_config_dict": { + "type": "fiftyone.utils.ultralytics.FiftyOneYOLOSegmentationModel", + "config": {} + }, + "requirements": { + "packages": [ + "torch>=1.7.0", + "torchvision>=0.8.1", + "ultralytics>=8.3.0" + ], + "cpu": { + "support": true + }, + "gpu": { + "support": true + } + }, + "tags": ["segmentation", "coco", "torch", "yolo"], + "date_added": "2024-10-05 19:22:51" + }, { "base_name": "rtdetr-l-coco-torch", "base_filename": "rtdetr-l-coco.pt", From 832af1a386e2d91de25a05f109eb6be162069e97 Mon Sep 17 00:00:00 2001 From: Jacob Marks Date: Sat, 5 Oct 2024 12:36:44 -0400 Subject: [PATCH 19/67] add yolo11seg models to manifest From d073a1e90f3a1aa78e01486a8a395df010b679e2 Mon Sep 17 00:00:00 2001 From: Jacob Marks Date: Sat, 5 Oct 2024 12:39:14 -0400 Subject: [PATCH 20/67] add yolo11seg models to docs --- docs/source/integrations/ultralytics.rst | 27 ++++++++++++++++++------ 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/docs/source/integrations/ultralytics.rst b/docs/source/integrations/ultralytics.rst index 79bef30e7d..dd01ddd16e 100644 --- a/docs/source/integrations/ultralytics.rst +++ b/docs/source/integrations/ultralytics.rst @@ -190,6 +190,11 @@ You can directly pass Ultralytics YOLO segmentation models to # model = YOLO("yolov8l-seg.pt") # model = YOLO("yolov8x-seg.pt") + # model = YOLO("yolo11s-seg.pt") + # model = YOLO("yolo11m-seg.pt") + # model = YOLO("yolo11l-seg.pt") + # model = YOLO("yolo11x-seg.pt") + dataset.apply_model(model, label_field="instances") session = fo.launch_app(dataset) @@ -215,19 +220,27 @@ manually convert Ultralytics predictions into the desired :align: center -You can also load YOLOv8 and YOLOv9 segmentation models from the +You can also load YOLOv8, YOLOv9, and YOLO11 segmentation models from the :ref:`FiftyOne Model Zoo `: .. code-block:: python :linenos: - model_name = "yolov9c-seg-coco-torch" - # model_name = "yolov9e-seg-coco-torch" - # model_name = "yolov8x-seg-coco-torch" - # model_name = "yolov8l-seg-coco-torch" - # model_name = "yolov8m-seg-coco-torch" + model_name = "yolov8n-seg-coco-torch" # model_name = "yolov8s-seg-coco-torch" - # model_name = "yolov8n-seg-coco-torch" + # model_name = "yolov8m-seg-coco-torch" + # model_name = "yolov8l-seg-coco-torch" + # model_name = "yolov8x-seg-coco-torch" + + # model_name = "yolov9c-seg-coco-torch" + # model_name = "yolov9e-seg-coco-torch" + + # model_name = "yolo11n-seg-coco-torch" + # model_name = "yolo11s-seg-coco-torch" + # model_name = "yolo11m-seg-coco-torch" + # model_name = "yolo11l-seg-coco-torch" + # model_name = "yolo11x-seg-coco-torch" + model = foz.load_zoo_model(model_name, label_field="yolo_seg") From 144f43d68bcf937971e43b786353af582f865090 Mon Sep 17 00:00:00 2001 From: brimoor Date: Thu, 24 Oct 2024 12:16:35 -0400 Subject: [PATCH 21/67] fix model zoo docs build From b6a53448b87097f2fa8b1438b4506e98aaa4f0e6 Mon Sep 17 00:00:00 2001 From: prernadh Date: Thu, 24 Oct 2024 16:31:01 +0530 Subject: [PATCH 22/67] Adding SAM2.1 checkpoints --- docs/scripts/make_model_zoo_docs.py | 12 +- fiftyone/zoo/models/manifest-torch.json | 279 ++++++++++++++++++++++++ 2 files changed, 285 insertions(+), 6 deletions(-) diff --git a/docs/scripts/make_model_zoo_docs.py b/docs/scripts/make_model_zoo_docs.py index 57d7bf32b9..cd3882e7a7 100644 --- a/docs/scripts/make_model_zoo_docs.py +++ b/docs/scripts/make_model_zoo_docs.py @@ -147,25 +147,25 @@ dataset.apply_model(model, label_field="auto") session = fo.launch_app(dataset) -{% elif 'segment-anything' in tags and 'video' in tags and 'med-SAM' not in tags %} +{% elif 'med-sam' in name %} model = foz.load_zoo_model("{{ name }}") # Segment inside boxes and propagate to all frames dataset.apply_model( model, - label_field="segmentations", - prompt_field="frames.detections", # can contain Detections or Keypoints + label_field="pred_segmentations", + prompt_field="frames.gt_detections", ) session = fo.launch_app(dataset) -{% elif 'med-sam' in name %} +{% elif 'segment-anything' in tags and 'video' in tags %} model = foz.load_zoo_model("{{ name }}") # Segment inside boxes and propagate to all frames dataset.apply_model( model, - label_field="pred_segmentations", - prompt_field="frames.gt_detections", + label_field="segmentations", + prompt_field="frames.detections", # can contain Detections or Keypoints ) session = fo.launch_app(dataset) diff --git a/fiftyone/zoo/models/manifest-torch.json b/fiftyone/zoo/models/manifest-torch.json index 8d00620325..64611db432 100644 --- a/fiftyone/zoo/models/manifest-torch.json +++ b/fiftyone/zoo/models/manifest-torch.json @@ -486,6 +486,285 @@ ], "date_added": "2024-08-17 14:48:00" }, + { + "base_name": "segment-anything-2-1-hiera-tiny-image-torch", + "base_filename": "sam2.1_hiera_tiny_image.pt", + "version": null, + "description": "Segment Anything Model 2 (SAM2) from `SAM2: Segment Anything in Images and Videos `_", + "source": "https://ai.meta.com/sam2/", + "size_bytes": 155906050, + "manager": { + "type": "fiftyone.core.models.ModelManager", + "config": { + "url": "https://dl.fbaipublicfiles.com/segment_anything_2/092824/sam2.1_hiera_tiny.pt" + } + }, + "default_deployment_config_dict": { + "type": "fiftyone.utils.sam2.SegmentAnything2ImageModel", + "config": { + "entrypoint_fcn": "sam2.build_sam.build_sam2", + "entrypoint_args": { + "model_cfg": "configs/sam2.1/sam2.1_hiera_t.yaml" + }, + "output_processor_cls": "fiftyone.utils.torch.SemanticSegmenterOutputProcessor" + } + }, + "requirements": { + "packages": ["torch", "torchvision"], + "cpu": { + "support": true + }, + "gpu": { + "support": true + } + }, + "tags": ["segment-anything", "torch", "zero-shot"], + "date_added": "2024-08-05 14:38:20" + }, + { + "base_name": "segment-anything-2-1-hiera-small-image-torch", + "base_filename": "sam2.1_hiera_small_image.pt", + "version": null, + "description": "Segment Anything Model 2 (SAM2) from `SAM2: Segment Anything in Images and Videos `_", + "source": "https://ai.meta.com/sam2/", + "size_bytes": 155906050, + "manager": { + "type": "fiftyone.core.models.ModelManager", + "config": { + "url": "https://dl.fbaipublicfiles.com/segment_anything_2/092824/sam2.1_hiera_small.pt" + } + }, + "default_deployment_config_dict": { + "type": "fiftyone.utils.sam2.SegmentAnything2ImageModel", + "config": { + "entrypoint_fcn": "sam2.build_sam.build_sam2", + "entrypoint_args": { + "model_cfg": "configs/sam2.1/sam2.1_hiera_s.yaml" + }, + "output_processor_cls": "fiftyone.utils.torch.SemanticSegmenterOutputProcessor" + } + }, + "requirements": { + "packages": ["torch", "torchvision"], + "cpu": { + "support": true + }, + "gpu": { + "support": true + } + }, + "tags": ["segment-anything", "torch", "zero-shot"], + "date_added": "2024-08-05 14:38:20" + }, + { + "base_name": "segment-anything-2-1-hiera-base-plus-image-torch", + "base_filename": "sam2.1_hiera_base_plus_image.pt", + "version": null, + "description": "Segment Anything Model 2 (SAM2) from `SAM2: Segment Anything in Images and Videos `_", + "source": "https://ai.meta.com/sam2/", + "size_bytes": 155906050, + "manager": { + "type": "fiftyone.core.models.ModelManager", + "config": { + "url": "https://dl.fbaipublicfiles.com/segment_anything_2/092824/sam2.1_hiera_base_plus.pt" + } + }, + "default_deployment_config_dict": { + "type": "fiftyone.utils.sam2.SegmentAnything2ImageModel", + "config": { + "entrypoint_fcn": "sam2.build_sam.build_sam2", + "entrypoint_args": { + "model_cfg": "configs/sam2.1/sam2.1_hiera_b+.yaml" + }, + "output_processor_cls": "fiftyone.utils.torch.SemanticSegmenterOutputProcessor" + } + }, + "requirements": { + "packages": ["torch", "torchvision"], + "cpu": { + "support": true + }, + "gpu": { + "support": true + } + }, + "tags": ["segment-anything", "torch", "zero-shot"], + "date_added": "2024-08-05 14:38:20" + }, + { + "base_name": "segment-anything-2-1-hiera-large-image-torch", + "base_filename": "sam2.1_hiera_large_image.pt", + "version": null, + "description": "Segment Anything Model 2 (SAM2) from `SAM2: Segment Anything in Images and Videos `_", + "source": "https://ai.meta.com/sam2/", + "size_bytes": 155906050, + "manager": { + "type": "fiftyone.core.models.ModelManager", + "config": { + "url": "https://dl.fbaipublicfiles.com/segment_anything_2/092824/sam2.1_hiera_large.pt" + } + }, + "default_deployment_config_dict": { + "type": "fiftyone.utils.sam2.SegmentAnything2ImageModel", + "config": { + "entrypoint_fcn": "sam2.build_sam.build_sam2", + "entrypoint_args": { + "model_cfg": "configs/sam2.1/sam2.1_hiera_l.yaml" + }, + "output_processor_cls": "fiftyone.utils.torch.SemanticSegmenterOutputProcessor" + } + }, + "requirements": { + "packages": ["torch", "torchvision"], + "cpu": { + "support": true + }, + "gpu": { + "support": true + } + }, + "tags": ["segment-anything", "torch", "zero-shot"], + "date_added": "2024-08-05 14:38:20" + }, + { + "base_name": "segment-anything-2-1-hiera-tiny-video-torch", + "base_filename": "sam2.1_hiera_tiny_video.pt", + "version": null, + "description": "Segment Anything Model 2 (SAM2) from `SAM2: Segment Anything in Images and Videos `_", + "source": "https://ai.meta.com/sam2/", + "size_bytes": 155906050, + "manager": { + "type": "fiftyone.core.models.ModelManager", + "config": { + "url": "https://dl.fbaipublicfiles.com/segment_anything_2/092824/sam2.1_hiera_tiny.pt" + } + }, + "default_deployment_config_dict": { + "type": "fiftyone.utils.sam2.SegmentAnything2VideoModel", + "config": { + "entrypoint_fcn": "sam2.build_sam.build_sam2_video_predictor", + "entrypoint_args": { + "model_cfg": "configs/sam2.1/sam2.1_hiera_t.yaml" + } + } + }, + "requirements": { + "packages": ["torch", "torchvision"], + "cpu": { + "support": true + }, + "gpu": { + "support": true + } + }, + "tags": ["segment-anything", "torch", "zero-shot", "video"], + "date_added": "2024-08-05 14:38:20" + }, + { + "base_name": "segment-anything-2-1-hiera-small-video-torch", + "base_filename": "sam2.1_hiera_small_video.pt", + "version": null, + "description": "Segment Anything Model 2 (SAM2) from `SAM2: Segment Anything in Images and Videos `_", + "source": "https://ai.meta.com/sam2/", + "size_bytes": 155906050, + "manager": { + "type": "fiftyone.core.models.ModelManager", + "config": { + "url": "https://dl.fbaipublicfiles.com/segment_anything_2/092824/sam2.1_hiera_small.pt" + } + }, + "default_deployment_config_dict": { + "type": "fiftyone.utils.sam2.SegmentAnything2VideoModel", + "config": { + "entrypoint_fcn": "sam2.build_sam.build_sam2_video_predictor", + "entrypoint_args": { + "model_cfg": "configs/sam2.1/sam2.1_hiera_s.yaml" + }, + "output_processor_cls": "fiftyone.utils.torch.SemanticSegmenterOutputProcessor" + } + }, + "requirements": { + "packages": ["torch", "torchvision"], + "cpu": { + "support": true + }, + "gpu": { + "support": true + } + }, + "tags": ["segment-anything", "torch", "zero-shot", "video"], + "date_added": "2024-08-05 14:38:20" + }, + { + "base_name": "segment-anything-2-1-hiera-base-plus-video-torch", + "base_filename": "sam2.1_hiera_base_plus_video.pt", + "version": null, + "description": "Segment Anything Model 2 (SAM2) from `SAM2: Segment Anything in Images and Videos `_", + "source": "https://ai.meta.com/sam2/", + "size_bytes": 155906050, + "manager": { + "type": "fiftyone.core.models.ModelManager", + "config": { + "url": "https://dl.fbaipublicfiles.com/segment_anything_2/092824/sam2.1_hiera_base_plus.pt" + } + }, + "default_deployment_config_dict": { + "type": "fiftyone.utils.sam2.SegmentAnything2VideoModel", + "config": { + "entrypoint_fcn": "sam2.build_sam.build_sam2_video_predictor", + "entrypoint_args": { + "model_cfg": "configs/sam2.1/sam2.1_hiera_b+.yaml" + }, + "output_processor_cls": "fiftyone.utils.torch.SemanticSegmenterOutputProcessor" + } + }, + "requirements": { + "packages": ["torch", "torchvision"], + "cpu": { + "support": true + }, + "gpu": { + "support": true + } + }, + "tags": ["segment-anything", "torch", "zero-shot", "video"], + "date_added": "2024-08-05 14:38:20" + }, + { + "base_name": "segment-anything-2-1-hiera-large-video-torch", + "base_filename": "sam2.1_hiera_large_video.pt", + "version": null, + "description": "Segment Anything Model 2 (SAM2) from `SAM2: Segment Anything in Images and Videos `_", + "source": "https://ai.meta.com/sam2/", + "size_bytes": 155906050, + "manager": { + "type": "fiftyone.core.models.ModelManager", + "config": { + "url": "https://dl.fbaipublicfiles.com/segment_anything_2/092824/sam2.1_hiera_large.pt" + } + }, + "default_deployment_config_dict": { + "type": "fiftyone.utils.sam2.SegmentAnything2VideoModel", + "config": { + "entrypoint_fcn": "sam2.build_sam.build_sam2_video_predictor", + "entrypoint_args": { + "model_cfg": "configs/sam2.1/sam2.1_hiera_l.yaml" + }, + "output_processor_cls": "fiftyone.utils.torch.SemanticSegmenterOutputProcessor" + } + }, + "requirements": { + "packages": ["torch", "torchvision"], + "cpu": { + "support": true + }, + "gpu": { + "support": true + } + }, + "tags": ["segment-anything", "torch", "zero-shot", "video"], + "date_added": "2024-08-05 14:38:20" + }, { "base_name": "deeplabv3-resnet50-coco-torch", "base_filename": "deeplabv3_resnet50_coco-cd0a2569.pth", From 6a6a7592c4535148ce9fd182a0c93e1cdf701f15 Mon Sep 17 00:00:00 2001 From: brimoor Date: Thu, 24 Oct 2024 15:37:29 -0400 Subject: [PATCH 23/67] use 2.1 --- fiftyone/zoo/models/manifest-torch.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/fiftyone/zoo/models/manifest-torch.json b/fiftyone/zoo/models/manifest-torch.json index 64611db432..ee2b3a2caf 100644 --- a/fiftyone/zoo/models/manifest-torch.json +++ b/fiftyone/zoo/models/manifest-torch.json @@ -487,7 +487,7 @@ "date_added": "2024-08-17 14:48:00" }, { - "base_name": "segment-anything-2-1-hiera-tiny-image-torch", + "base_name": "segment-anything-2.1-hiera-tiny-image-torch", "base_filename": "sam2.1_hiera_tiny_image.pt", "version": null, "description": "Segment Anything Model 2 (SAM2) from `SAM2: Segment Anything in Images and Videos `_", @@ -522,7 +522,7 @@ "date_added": "2024-08-05 14:38:20" }, { - "base_name": "segment-anything-2-1-hiera-small-image-torch", + "base_name": "segment-anything-2.1-hiera-small-image-torch", "base_filename": "sam2.1_hiera_small_image.pt", "version": null, "description": "Segment Anything Model 2 (SAM2) from `SAM2: Segment Anything in Images and Videos `_", @@ -557,7 +557,7 @@ "date_added": "2024-08-05 14:38:20" }, { - "base_name": "segment-anything-2-1-hiera-base-plus-image-torch", + "base_name": "segment-anything-2.1-hiera-base-plus-image-torch", "base_filename": "sam2.1_hiera_base_plus_image.pt", "version": null, "description": "Segment Anything Model 2 (SAM2) from `SAM2: Segment Anything in Images and Videos `_", @@ -592,7 +592,7 @@ "date_added": "2024-08-05 14:38:20" }, { - "base_name": "segment-anything-2-1-hiera-large-image-torch", + "base_name": "segment-anything-2.1-hiera-large-image-torch", "base_filename": "sam2.1_hiera_large_image.pt", "version": null, "description": "Segment Anything Model 2 (SAM2) from `SAM2: Segment Anything in Images and Videos `_", @@ -627,7 +627,7 @@ "date_added": "2024-08-05 14:38:20" }, { - "base_name": "segment-anything-2-1-hiera-tiny-video-torch", + "base_name": "segment-anything-2.1-hiera-tiny-video-torch", "base_filename": "sam2.1_hiera_tiny_video.pt", "version": null, "description": "Segment Anything Model 2 (SAM2) from `SAM2: Segment Anything in Images and Videos `_", @@ -661,7 +661,7 @@ "date_added": "2024-08-05 14:38:20" }, { - "base_name": "segment-anything-2-1-hiera-small-video-torch", + "base_name": "segment-anything-2.1-hiera-small-video-torch", "base_filename": "sam2.1_hiera_small_video.pt", "version": null, "description": "Segment Anything Model 2 (SAM2) from `SAM2: Segment Anything in Images and Videos `_", @@ -696,7 +696,7 @@ "date_added": "2024-08-05 14:38:20" }, { - "base_name": "segment-anything-2-1-hiera-base-plus-video-torch", + "base_name": "segment-anything-2.1-hiera-base-plus-video-torch", "base_filename": "sam2.1_hiera_base_plus_video.pt", "version": null, "description": "Segment Anything Model 2 (SAM2) from `SAM2: Segment Anything in Images and Videos `_", @@ -731,7 +731,7 @@ "date_added": "2024-08-05 14:38:20" }, { - "base_name": "segment-anything-2-1-hiera-large-video-torch", + "base_name": "segment-anything-2.1-hiera-large-video-torch", "base_filename": "sam2.1_hiera_large_video.pt", "version": null, "description": "Segment Anything Model 2 (SAM2) from `SAM2: Segment Anything in Images and Videos `_", From 6681ad991221e499e506923f7ad4403396523620 Mon Sep 17 00:00:00 2001 From: brimoor Date: Thu, 24 Oct 2024 15:54:11 -0400 Subject: [PATCH 24/67] fixing links --- docs/scripts/make_model_zoo_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/scripts/make_model_zoo_docs.py b/docs/scripts/make_model_zoo_docs.py index cd3882e7a7..472e0b31db 100644 --- a/docs/scripts/make_model_zoo_docs.py +++ b/docs/scripts/make_model_zoo_docs.py @@ -354,7 +354,7 @@ def _render_card_model_content(template, model_name): tags = ",".join(tags) - link = "models.html#%s" % zoo_model.name + link = "models.html#%s" % zoo_model.name.replace(".", "-") description = zoo_model.description From 1eaf7846842bf91a28374bd872a106ad2a898a83 Mon Sep 17 00:00:00 2001 From: Ritchie Martori Date: Mon, 21 Oct 2024 14:02:44 -0700 Subject: [PATCH 25/67] fix overlay z-indax for panel menus --- .../plugins/SchemaIO/components/ContainerizedComponent.tsx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/packages/core/src/plugins/SchemaIO/components/ContainerizedComponent.tsx b/app/packages/core/src/plugins/SchemaIO/components/ContainerizedComponent.tsx index 4caadbd7f3..78527693a3 100644 --- a/app/packages/core/src/plugins/SchemaIO/components/ContainerizedComponent.tsx +++ b/app/packages/core/src/plugins/SchemaIO/components/ContainerizedComponent.tsx @@ -7,6 +7,7 @@ import { overlayToSx, } from "../utils"; import { ViewPropsType } from "../utils/types"; +import { has } from "lodash"; export default function ContainerizedComponent(props: ContainerizedComponent) { const { schema, children } = props; @@ -22,7 +23,11 @@ export default function ContainerizedComponent(props: ContainerizedComponent) { } if (isCompositeView(schema)) { + const hasOverlay = !!schema?.view?.overlay; const sxForOverlay = overlayToSx[schema?.view?.overlay] || {}; + if (hasOverlay) { + sxForOverlay.zIndex = 999; + } return ( {containerizedChildren} From 2f1cd845538a17ca078b90624dd010033f3f7b60 Mon Sep 17 00:00:00 2001 From: Sashank Aryal <66688606+sashankaryal@users.noreply.github.com> Date: Thu, 24 Oct 2024 11:33:03 +0545 Subject: [PATCH 26/67] fix bug where timeline name wasn't being forwarded in seek utils (#4975) * fix eslint * forward timeline name --- app/packages/playback/eslint.config.mjs | 11 +++++++++-- app/packages/playback/package.json | 1 + app/packages/playback/src/views/Timeline.tsx | 12 +++++++----- app/yarn.lock | 10 ++++++++++ 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/app/packages/playback/eslint.config.mjs b/app/packages/playback/eslint.config.mjs index 2281b87778..5fd106b853 100644 --- a/app/packages/playback/eslint.config.mjs +++ b/app/packages/playback/eslint.config.mjs @@ -1,7 +1,8 @@ +import { fixupConfigRules } from "@eslint/compat"; +import hooksPlugin from "eslint-plugin-react-hooks"; +import pluginReactConfig from "eslint-plugin-react/configs/recommended.js"; import globals from "globals"; import tseslint from "typescript-eslint"; -import pluginReactConfig from "eslint-plugin-react/configs/recommended.js"; -import { fixupConfigRules } from "@eslint/compat"; export default [ { files: ["lib/**/*.{js,mjs,cjs,ts,jsx,tsx}"] }, @@ -9,4 +10,10 @@ export default [ { languageOptions: { globals: globals.browser } }, ...tseslint.configs.recommended, ...fixupConfigRules(pluginReactConfig), + { + plugins: { + "react-hooks": hooksPlugin, + }, + rules: hooksPlugin.configs.recommended.rules, + }, ]; diff --git a/app/packages/playback/package.json b/app/packages/playback/package.json index 081b41a9b4..3bde5e8218 100644 --- a/app/packages/playback/package.json +++ b/app/packages/playback/package.json @@ -6,6 +6,7 @@ "@eslint/compat": "^1.1.1", "eslint": "9.7.0", "eslint-plugin-react": "^7.35.0", + "eslint-plugin-react-hooks": "rc", "globals": "^15.8.0", "prettier": "^3.3.3", "typescript": "^5.5.4", diff --git a/app/packages/playback/src/views/Timeline.tsx b/app/packages/playback/src/views/Timeline.tsx index 62c7b278d4..7112b6111b 100644 --- a/app/packages/playback/src/views/Timeline.tsx +++ b/app/packages/playback/src/views/Timeline.tsx @@ -26,14 +26,16 @@ interface TimelineProps { */ export const Timeline = React.memo( React.forwardRef( - ({ name, style, controlsStyle }, ref) => { + (timelineProps: TimelineProps, ref) => { + const { name, style, controlsStyle } = timelineProps; + const { playHeadState, config, play, pause, setSpeed } = useTimeline(name); const frameNumber = useFrameNumber(name); - const { getSeekValue, seekTo } = useTimelineVizUtils(); + const { getSeekValue, seekTo } = useTimelineVizUtils(name); - const seekBarValue = React.useMemo(() => getSeekValue(), [frameNumber]); + const seekBarValue = React.useMemo(() => getSeekValue(), [getSeekValue]); const { loaded, loading } = useTimelineBuffers(name); @@ -52,7 +54,7 @@ export const Timeline = React.memo( detail: { timelineName: name, start: true }, }) ); - }, [pause]); + }, [pause, name]); const onSeekEnd = React.useCallback(() => { dispatchEvent( @@ -60,7 +62,7 @@ export const Timeline = React.memo( detail: { timelineName: name, start: false }, }) ); - }, []); + }, [name]); const [isHoveringSeekBar, setIsHoveringSeekBar] = React.useState(false); diff --git a/app/yarn.lock b/app/yarn.lock index 86e96c90f8..6b8fb59394 100644 --- a/app/yarn.lock +++ b/app/yarn.lock @@ -1942,6 +1942,7 @@ __metadata: "@eslint/compat": ^1.1.1 eslint: 9.7.0 eslint-plugin-react: ^7.35.0 + eslint-plugin-react-hooks: rc globals: ^15.8.0 jotai: ^2.9.3 jotai-optics: ^0.4.0 @@ -8372,6 +8373,15 @@ __metadata: languageName: node linkType: hard +"eslint-plugin-react-hooks@npm:rc": + version: 5.1.0-rc-28668d39-20241023 + resolution: "eslint-plugin-react-hooks@npm:5.1.0-rc-28668d39-20241023" + peerDependencies: + eslint: ^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0 + checksum: 6ad29212fa76b96488a6eeb9941a9a6111420092cc309417f5569f917e4e40b15ed282172842ca8611466387c3d750ceee07e9e739e4c28e808065eaf9ed2307 + languageName: node + linkType: hard + "eslint-plugin-react@npm:^7.31.11": version: 7.34.1 resolution: "eslint-plugin-react@npm:7.34.1" From b38cc7d70099c0a009dec93428c7a22d70282883 Mon Sep 17 00:00:00 2001 From: Ritchie Martori Date: Tue, 22 Oct 2024 09:45:38 -0700 Subject: [PATCH 27/67] fix panel overflow From df488775e19d0ef62547e1f9e0fe6b22d8615e6c Mon Sep 17 00:00:00 2001 From: Ritchie Martori Date: Thu, 24 Oct 2024 15:47:04 -0700 Subject: [PATCH 28/67] add label_count link --- docs/source/plugins/developing_plugins.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/source/plugins/developing_plugins.rst b/docs/source/plugins/developing_plugins.rst index a4a6c4e464..47c95314a5 100644 --- a/docs/source/plugins/developing_plugins.rst +++ b/docs/source/plugins/developing_plugins.rst @@ -2185,6 +2185,11 @@ The ``surfaces`` key defines the panel's scope: :ref:`modal view `, which allows you to build interactions that focus on individual samples and scenarios +.. note:: + + For an example of a modal panel, refer to the + `label count panel `_. + .. _panel-execution-context: Execution context From 7c803ed5e6f9795ad3e5ba9dd08d09fc209ad26b Mon Sep 17 00:00:00 2001 From: Ritchie Martori Date: Tue, 22 Oct 2024 16:37:57 -0700 Subject: [PATCH 29/67] add timeline view --- .../SchemaIO/components/PlotlyView.tsx | 1 - .../SchemaIO/components/TimelineView.tsx | 48 +++++++++++++++++++ .../src/plugins/SchemaIO/components/index.ts | 1 + fiftyone/operators/types.py | 13 +++++ 4 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 app/packages/core/src/plugins/SchemaIO/components/TimelineView.tsx diff --git a/app/packages/core/src/plugins/SchemaIO/components/PlotlyView.tsx b/app/packages/core/src/plugins/SchemaIO/components/PlotlyView.tsx index d7a18b33c8..186a3a1cf7 100644 --- a/app/packages/core/src/plugins/SchemaIO/components/PlotlyView.tsx +++ b/app/packages/core/src/plugins/SchemaIO/components/PlotlyView.tsx @@ -212,7 +212,6 @@ export default function PlotlyView(props: ViewPropsType) { return ( diff --git a/app/packages/core/src/plugins/SchemaIO/components/TimelineView.tsx b/app/packages/core/src/plugins/SchemaIO/components/TimelineView.tsx new file mode 100644 index 0000000000..103a9d5012 --- /dev/null +++ b/app/packages/core/src/plugins/SchemaIO/components/TimelineView.tsx @@ -0,0 +1,48 @@ +import React from "react"; +import { Timeline, useCreateTimeline, useTimeline } from "@fiftyone/playback"; +import { ViewPropsType } from "../utils/types"; + +export default function TimelineView(props: ViewPropsType) { + const { schema } = props; + const { view = {} } = schema; + const { timeline_name, loop, total_frames } = view; + + const providedcConfig = { + loop, + totalFrames: total_frames, + }; + + const defaultConfig = { + loop: false, + }; + const finalConfig = { + ...defaultConfig, + ...providedcConfig, + }; + + const requiredParams = ["timeline_name", "total_frames"]; + + // for (const param of requiredParams) { + // if (!finalConfig[param]) { + // throw new Error(`Missing required parameter: ${param}`); + // } + // } + + return ; +} + +export const TimelineCreator = ({ timelineName, totalFrames, loop }) => { + const { isTimelineInitialized } = useCreateTimeline({ + name: timelineName, + config: { + totalFrames: totalFrames, + loop, + }, + }); + + if (!isTimelineInitialized) { + return
initializing timeline...
; + } + + return ; +}; diff --git a/app/packages/core/src/plugins/SchemaIO/components/index.ts b/app/packages/core/src/plugins/SchemaIO/components/index.ts index bb0fca6f6e..2b1cfcc042 100644 --- a/app/packages/core/src/plugins/SchemaIO/components/index.ts +++ b/app/packages/core/src/plugins/SchemaIO/components/index.ts @@ -49,3 +49,4 @@ export { default as TextFieldView } from "./TextFieldView"; export { default as TupleView } from "./TupleView"; export { default as UnsupportedView } from "./UnsupportedView"; export { default as FrameLoaderView } from "./FrameLoaderView"; +export { default as TimelineView } from "./TimelineView"; diff --git a/fiftyone/operators/types.py b/fiftyone/operators/types.py index a22bd01230..d2884b81b9 100644 --- a/fiftyone/operators/types.py +++ b/fiftyone/operators/types.py @@ -2413,6 +2413,19 @@ def __init__(self, **kwargs): super().__init__(**kwargs) +class TimelineView(View): + """Represents a timeline for playing animations. + + Args: + timeline_name (None): the name of the timeline + total_frames (None): the total number of frames in the timeline + loop (False): whether to loop the timeline + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + class Container(BaseType): """Represents a base container for a container types.""" From 19c4e014b020b569a8f0683c4f3efe7d135109cf Mon Sep 17 00:00:00 2001 From: Ritchie Martori Date: Wed, 23 Oct 2024 09:29:42 -0700 Subject: [PATCH 30/67] timeline view polish --- .../SchemaIO/components/FrameLoaderView.tsx | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx index ff41d1274e..1f504614bb 100644 --- a/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx +++ b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx @@ -19,15 +19,16 @@ export default function FrameLoaderView(props: ViewPropsType) { const setPanelState = useSetPanelStateById(true); const localIdRef = React.useRef(); const bufm = useRef(new BufferManager()); + const frameDataRef = useRef(); useEffect(() => { localIdRef.current = Math.random().toString(36).substring(7); - if (data?.frames) - window.dispatchEvent( - new CustomEvent(`frames-loaded`, { - detail: { localId: localIdRef.current }, - }) - ); + if (data?.frames) frameDataRef.current = data.frames; + window.dispatchEvent( + new CustomEvent(`frames-loaded`, { + detail: { localId: localIdRef.current }, + }) + ); }, [data?.signature]); const loadRange = React.useCallback( @@ -44,15 +45,22 @@ export default function FrameLoaderView(props: ViewPropsType) { } return new Promise((resolve) => { - window.addEventListener(`frames-loaded`, (e) => { - if ( - e instanceof CustomEvent && - e.detail.localId === localIdRef.current - ) { - bufm.current.addNewRange(range); - resolve(); - } - }); + if (frameDataRef.current) { + bufm.current.addNewRange(range); + resolve(); + } else { + window.addEventListener(`frames-loaded`, (e) => { + if ( + e instanceof CustomEvent && + e.detail.localId === localIdRef.current + ) { + try { + bufm.current.addNewRange(range); + } catch (e) {} + resolve(); + } + }); + } }); } }, From 5e611c12a49c58a963ca955d9d930c810701d0cd Mon Sep 17 00:00:00 2001 From: Ritchie Martori Date: Wed, 23 Oct 2024 09:31:39 -0700 Subject: [PATCH 31/67] timeline view validation --- .../plugins/SchemaIO/components/TimelineView.tsx | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/app/packages/core/src/plugins/SchemaIO/components/TimelineView.tsx b/app/packages/core/src/plugins/SchemaIO/components/TimelineView.tsx index 103a9d5012..03aa2ee934 100644 --- a/app/packages/core/src/plugins/SchemaIO/components/TimelineView.tsx +++ b/app/packages/core/src/plugins/SchemaIO/components/TimelineView.tsx @@ -20,13 +20,12 @@ export default function TimelineView(props: ViewPropsType) { ...providedcConfig, }; - const requiredParams = ["timeline_name", "total_frames"]; - - // for (const param of requiredParams) { - // if (!finalConfig[param]) { - // throw new Error(`Missing required parameter: ${param}`); - // } - // } + if (!timeline_name) { + throw new Error("Timeline name is required"); + } + if (!finalConfig.totalFrames) { + throw new Error("Total frames is required"); + } return ; } From 28ab8a3999a95718399b66fe30686b20697fd5f6 Mon Sep 17 00:00:00 2001 From: Ritchie Martori Date: Wed, 23 Oct 2024 09:32:11 -0700 Subject: [PATCH 32/67] remove timline loading indicator --- .../core/src/plugins/SchemaIO/components/TimelineView.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/packages/core/src/plugins/SchemaIO/components/TimelineView.tsx b/app/packages/core/src/plugins/SchemaIO/components/TimelineView.tsx index 03aa2ee934..020cc3132f 100644 --- a/app/packages/core/src/plugins/SchemaIO/components/TimelineView.tsx +++ b/app/packages/core/src/plugins/SchemaIO/components/TimelineView.tsx @@ -40,7 +40,7 @@ export const TimelineCreator = ({ timelineName, totalFrames, loop }) => { }); if (!isTimelineInitialized) { - return
initializing timeline...
; + return null; } return ; From e9f664335c28132965ce44ff4477b12c444c0e80 Mon Sep 17 00:00:00 2001 From: Ritchie Martori Date: Wed, 23 Oct 2024 12:16:39 -0700 Subject: [PATCH 33/67] Update app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .../core/src/plugins/SchemaIO/components/FrameLoaderView.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx index 1f504614bb..940d9bee09 100644 --- a/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx +++ b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx @@ -19,7 +19,7 @@ export default function FrameLoaderView(props: ViewPropsType) { const setPanelState = useSetPanelStateById(true); const localIdRef = React.useRef(); const bufm = useRef(new BufferManager()); - const frameDataRef = useRef(); + const frameDataRef = useRef(null); useEffect(() => { localIdRef.current = Math.random().toString(36).substring(7); From 2c3b284bf1b6f16597b19aea0b25c739cf245b41 Mon Sep 17 00:00:00 2001 From: Ritchie Martori Date: Wed, 23 Oct 2024 12:18:53 -0700 Subject: [PATCH 34/67] Update app/packages/core/src/plugins/SchemaIO/components/TimelineView.tsx Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .../core/src/plugins/SchemaIO/components/TimelineView.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/packages/core/src/plugins/SchemaIO/components/TimelineView.tsx b/app/packages/core/src/plugins/SchemaIO/components/TimelineView.tsx index 020cc3132f..e7c0d7df63 100644 --- a/app/packages/core/src/plugins/SchemaIO/components/TimelineView.tsx +++ b/app/packages/core/src/plugins/SchemaIO/components/TimelineView.tsx @@ -7,7 +7,7 @@ export default function TimelineView(props: ViewPropsType) { const { view = {} } = schema; const { timeline_name, loop, total_frames } = view; - const providedcConfig = { + const providedConfig = { loop, totalFrames: total_frames, }; From e61c145062be505a7a6f7e47b26f3456ee7de092 Mon Sep 17 00:00:00 2001 From: Ritchie Martori Date: Wed, 23 Oct 2024 12:20:50 -0700 Subject: [PATCH 35/67] pr comments timelineview --- .../SchemaIO/components/FrameLoaderView.tsx | 14 +++++++----- .../SchemaIO/components/TimelineView.tsx | 22 ++++++++----------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx index 940d9bee09..83f594b5e6 100644 --- a/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx +++ b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx @@ -10,6 +10,8 @@ import { usePanelId, useSetPanelStateById } from "@fiftyone/spaces"; import { useTimeline } from "@fiftyone/playback/src/lib/use-timeline"; import _ from "lodash"; +const FRAME_LOADED_EVENT = "frames-loaded"; + export default function FrameLoaderView(props: ViewPropsType) { const { schema, path, data } = props; const { view = {} } = schema; @@ -25,7 +27,7 @@ export default function FrameLoaderView(props: ViewPropsType) { localIdRef.current = Math.random().toString(36).substring(7); if (data?.frames) frameDataRef.current = data.frames; window.dispatchEvent( - new CustomEvent(`frames-loaded`, { + new CustomEvent(FRAME_LOADED_EVENT, { detail: { localId: localIdRef.current }, }) ); @@ -49,17 +51,17 @@ export default function FrameLoaderView(props: ViewPropsType) { bufm.current.addNewRange(range); resolve(); } else { - window.addEventListener(`frames-loaded`, (e) => { + const onFramesLoaded = (e) => { if ( e instanceof CustomEvent && e.detail.localId === localIdRef.current ) { - try { - bufm.current.addNewRange(range); - } catch (e) {} + window.removeEventListener(FRAME_LOADED_EVENT, onFramesLoaded); + bufm.current.addNewRange(range); resolve(); } - }); + }; + window.addEventListener(FRAME_LOADED_EVENT, onFramesLoaded); } }); } diff --git a/app/packages/core/src/plugins/SchemaIO/components/TimelineView.tsx b/app/packages/core/src/plugins/SchemaIO/components/TimelineView.tsx index e7c0d7df63..a5c2ad198c 100644 --- a/app/packages/core/src/plugins/SchemaIO/components/TimelineView.tsx +++ b/app/packages/core/src/plugins/SchemaIO/components/TimelineView.tsx @@ -1,7 +1,9 @@ -import React from "react"; +import React, { useMemo } from "react"; import { Timeline, useCreateTimeline, useTimeline } from "@fiftyone/playback"; import { ViewPropsType } from "../utils/types"; +const DEFAULT_CONFIG = { loop: false }; + export default function TimelineView(props: ViewPropsType) { const { schema } = props; const { view = {} } = schema; @@ -12,14 +14,10 @@ export default function TimelineView(props: ViewPropsType) { totalFrames: total_frames, }; - const defaultConfig = { - loop: false, - }; - const finalConfig = { - ...defaultConfig, - ...providedcConfig, - }; - + const finalConfig = useMemo( + () => ({ ...DEFAULT_CONFIG, ...providedConfig }), + [providedConfig] + ); if (!timeline_name) { throw new Error("Timeline name is required"); } @@ -31,12 +29,10 @@ export default function TimelineView(props: ViewPropsType) { } export const TimelineCreator = ({ timelineName, totalFrames, loop }) => { + const config = useMemo(() => ({ totalFrames, loop }), [totalFrames, loop]); const { isTimelineInitialized } = useCreateTimeline({ name: timelineName, - config: { - totalFrames: totalFrames, - loop, - }, + config, }); if (!isTimelineInitialized) { From 6daea1ed522d9bdce2440657c0b98fcf74692dbf Mon Sep 17 00:00:00 2001 From: Ritchie Martori Date: Thu, 24 Oct 2024 19:51:20 -0700 Subject: [PATCH 36/67] fix imports --- app/packages/core/src/plugins/SchemaIO/components/index.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/packages/core/src/plugins/SchemaIO/components/index.ts b/app/packages/core/src/plugins/SchemaIO/components/index.ts index 2b1cfcc042..10c4e44a9e 100644 --- a/app/packages/core/src/plugins/SchemaIO/components/index.ts +++ b/app/packages/core/src/plugins/SchemaIO/components/index.ts @@ -33,6 +33,7 @@ export { default as LoadingView } from "./LoadingView"; export { default as MapView } from "./MapView"; export { default as MarkdownView } from "./MarkdownView"; export { default as MediaPlayerView } from "./MediaPlayerView"; +export { default as ModalView } from "./ModalView"; export { default as ObjectView } from "./ObjectView"; export { default as OneOfView } from "./OneOfView"; export { default as PlotlyView } from "./PlotlyView"; @@ -46,7 +47,8 @@ export { default as TableView } from "./TableView"; export { default as TabsView } from "./TabsView"; export { default as TagsView } from "./TagsView"; export { default as TextFieldView } from "./TextFieldView"; +export { default as TextView } from "./TextView"; +export { default as TimelineView } from "./TimelineView"; +export { default as ToastView } from "./ToastView"; export { default as TupleView } from "./TupleView"; export { default as UnsupportedView } from "./UnsupportedView"; -export { default as FrameLoaderView } from "./FrameLoaderView"; -export { default as TimelineView } from "./TimelineView"; From 586159b0df1ab41092f62c4f0fdf40c2987d3440 Mon Sep 17 00:00:00 2001 From: brimoor Date: Fri, 25 Oct 2024 00:05:42 -0400 Subject: [PATCH 37/67] release notes --- docs/source/release-notes.rst | 39 +++++++++++++++++++++++++++++++++++ setup.py | 2 +- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/docs/source/release-notes.rst b/docs/source/release-notes.rst index d78f155c1c..417e4568b0 100644 --- a/docs/source/release-notes.rst +++ b/docs/source/release-notes.rst @@ -3,6 +3,45 @@ FiftyOne Release Notes .. default-role:: code +FiftyOne Teams 2.1.2 +-------------------- +*Released October 25, 2024* + +Includes all updates from :ref:`FiftyOne 1.0.2 `, plus: + +- Fixed an issue that prevented `delegation_target` from being properly set when + running delegated operations with orchestrator registration enabled +- Resolved a `RuntimeError` that could occur due to concurrency issues when + applying operations that download data from cloud buckets + +.. _release-notes-v1.0.2: + +FiftyOne 1.0.2 +-------------- +*Released October 25, 2024* + +Zoo + +- Added :ref:`SAM 2.1 ` + to the :ref:`Model Zoo ` + `#4979 `_ +- Added :ref:`YOLO11 ` to the + :ref:`Model Zoo ` + `#4899 `_ +- Added generic model architecture and backbone tags to all relevant models + :ref:`in the zoo ` for easier navigation + `#4899 `_ + +App + +- Added a new :ref:`TimelineView ` for + building custom animations + `#4965 `_ +- Fixed overlay z-index and overflow for panels + `#4956 `_ +- Fixed bug where timeline name wasn't being forwarded in seek utils + `#4975 `_ + FiftyOne Teams 2.1.1 -------------------- *Released October 14, 2024* diff --git a/setup.py b/setup.py index 3652dd44c5..11aed71b06 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ from setuptools import setup, find_packages -VERSION = "1.0.1" +VERSION = "1.0.2" def get_version(): From 389557ce28d91057c745547db4adc2ba5caffb14 Mon Sep 17 00:00:00 2001 From: Ritchie Martori Date: Thu, 24 Oct 2024 22:04:28 -0700 Subject: [PATCH 38/67] remove unused imports --- app/packages/core/src/plugins/SchemaIO/components/index.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/app/packages/core/src/plugins/SchemaIO/components/index.ts b/app/packages/core/src/plugins/SchemaIO/components/index.ts index 10c4e44a9e..0716b2405e 100644 --- a/app/packages/core/src/plugins/SchemaIO/components/index.ts +++ b/app/packages/core/src/plugins/SchemaIO/components/index.ts @@ -33,7 +33,6 @@ export { default as LoadingView } from "./LoadingView"; export { default as MapView } from "./MapView"; export { default as MarkdownView } from "./MarkdownView"; export { default as MediaPlayerView } from "./MediaPlayerView"; -export { default as ModalView } from "./ModalView"; export { default as ObjectView } from "./ObjectView"; export { default as OneOfView } from "./OneOfView"; export { default as PlotlyView } from "./PlotlyView"; @@ -47,8 +46,6 @@ export { default as TableView } from "./TableView"; export { default as TabsView } from "./TabsView"; export { default as TagsView } from "./TagsView"; export { default as TextFieldView } from "./TextFieldView"; -export { default as TextView } from "./TextView"; export { default as TimelineView } from "./TimelineView"; -export { default as ToastView } from "./ToastView"; export { default as TupleView } from "./TupleView"; export { default as UnsupportedView } from "./UnsupportedView"; From 3d92aab46f91e08c9bedd07cb2bfd883b2a61d07 Mon Sep 17 00:00:00 2001 From: Stuart Wheaton Date: Fri, 25 Oct 2024 21:49:41 -0400 Subject: [PATCH 39/67] fix: quickstart3d load due to dict changed size during iteration --- fiftyone/core/odm/utils.py | 3 ++- requirements/common.txt | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fiftyone/core/odm/utils.py b/fiftyone/core/odm/utils.py index 14f709ac2d..98e294bb73 100644 --- a/fiftyone/core/odm/utils.py +++ b/fiftyone/core/odm/utils.py @@ -658,7 +658,8 @@ def __getitem__(self, name): pass # Then full module list - for module in sys.modules.values(): + all_modules = sys.modules.copy().values() + for module in all_modules: try: cls = self._get_cls(module, name) self._cache[name] = cls diff --git a/requirements/common.txt b/requirements/common.txt index b0a2a3b338..283bffd11d 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -17,7 +17,6 @@ plotly==5.17.0 pprintpp==0.4.0 psutil>=5.7.0 pymongo>=3.12,<4.9 -pydantic==2.6.4 pytz==2022.1 PyYAML==6.0.1 regex==2022.8.17 From 5e7b209fb895fe793960ac31f3d0b0c3cce39305 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Wed, 30 Oct 2024 20:45:24 +0545 Subject: [PATCH 40/67] debounced nav --- .../src/components/Modal/ModalNavigation.tsx | 99 +++++++++++++++++-- .../state/src/hooks/useExpandSample.ts | 14 +-- 2 files changed, 97 insertions(+), 16 deletions(-) diff --git a/app/packages/core/src/components/Modal/ModalNavigation.tsx b/app/packages/core/src/components/Modal/ModalNavigation.tsx index d993eb2002..d35baacba3 100644 --- a/app/packages/core/src/components/Modal/ModalNavigation.tsx +++ b/app/packages/core/src/components/Modal/ModalNavigation.tsx @@ -3,7 +3,7 @@ import { LookerArrowRightIcon, } from "@fiftyone/components"; import * as fos from "@fiftyone/state"; -import React, { useCallback, useRef } from "react"; +import React, { useCallback, useEffect, useRef } from "react"; import { useRecoilValue, useRecoilValueLoadable } from "recoil"; import styled from "styled-components"; @@ -63,17 +63,92 @@ const ModalNavigation = ({ onNavigate }: { onNavigate: () => void }) => { const modal = useRecoilValue(fos.modalSelector); const navigation = useRecoilValue(fos.modalNavigation); - const navigateNext = useCallback(async () => { - onNavigate(); - const result = await navigation?.next(); - setModal(result); + const nextTimeoutRef = useRef(null); + const accumulatedNextOffsetRef = useRef(0); + + const previousTimeoutRef = useRef(null); + const accumulatedPreviousOffsetRef = useRef(0); + + const modalRef = useRef(modal); + + modalRef.current = modal; + + const navigateNext = useCallback(() => { + if (!modalRef.current?.hasNext) { + return; + } + + if (!nextTimeoutRef.current) { + // First click: navigate immediately + onNavigate(); + navigation?.next(1).then(setModal); + accumulatedNextOffsetRef.current = 0; + console.log(">!>Immediate next execution"); + } else { + // Subsequent clicks: accumulate offset + accumulatedNextOffsetRef.current += 1; + console.log(">!>Debouncing next"); + } + + // Reset debounce timer + if (nextTimeoutRef.current) { + clearTimeout(nextTimeoutRef.current); + } + + nextTimeoutRef.current = setTimeout(() => { + if (accumulatedNextOffsetRef.current > 0) { + onNavigate(); + navigation?.next(accumulatedNextOffsetRef.current).then(setModal); + accumulatedNextOffsetRef.current = 0; + } + nextTimeoutRef.current = null; + }, 200); }, [navigation, onNavigate, setModal]); - const navigatePrevious = useCallback(async () => { - onNavigate(); - const result = await navigation?.previous(); - setModal(result); - }, [onNavigate, navigation, setModal]); + const navigatePrevious = useCallback(() => { + if (!modalRef.current?.hasPrevious) { + return; + } + + if (!previousTimeoutRef.current) { + // First click: navigate immediately + onNavigate(); + navigation?.previous(1).then(setModal); + accumulatedPreviousOffsetRef.current = 0; + console.log(">!>Immediate previous execution"); + } else { + // Subsequent clicks: accumulate offset + accumulatedPreviousOffsetRef.current += 1; + console.log(">!>Debouncing previous"); + } + + // Reset debounce timer + if (previousTimeoutRef.current) { + clearTimeout(previousTimeoutRef.current); + } + + previousTimeoutRef.current = setTimeout(() => { + if (accumulatedPreviousOffsetRef.current > 0) { + onNavigate(); + navigation + ?.previous(accumulatedPreviousOffsetRef.current) + .then(setModal); + accumulatedPreviousOffsetRef.current = 0; + } + previousTimeoutRef.current = null; + }, 200); + }, [navigation, onNavigate, setModal]); + + useEffect(() => { + return () => { + if (nextTimeoutRef.current) { + clearTimeout(nextTimeoutRef.current); + } + if (previousTimeoutRef.current) { + clearTimeout(previousTimeoutRef.current); + } + }; + }, []); const keyboardHandler = useCallback( (e: KeyboardEvent) => { @@ -122,6 +197,10 @@ const ModalNavigation = ({ onNavigate }: { onNavigate: () => void }) => { onClick={navigateNext} > +
oi
+ {accumulatedNextOffsetRef.current > 0 && ( +
{accumulatedNextOffsetRef.current}
+ )} )} diff --git a/app/packages/state/src/hooks/useExpandSample.ts b/app/packages/state/src/hooks/useExpandSample.ts index b80bb0c4d6..97426efba3 100644 --- a/app/packages/state/src/hooks/useExpandSample.ts +++ b/app/packages/state/src/hooks/useExpandSample.ts @@ -8,6 +8,7 @@ import * as atoms from "../recoil/atoms"; import * as groupAtoms from "../recoil/groups"; import useSetExpandedSample from "./useSetExpandedSample"; import useSetModalState from "./useSetModalState"; +import { useCallback } from "react"; export type Sample = Exclude< Exclude< @@ -22,6 +23,7 @@ export type Sample = Exclude< export default (store: WeakMap) => { const setExpandedSample = useSetExpandedSample(); const setModalState = useSetModalState(); + return useRecoilCallback( ({ snapshot, set }) => async ({ @@ -64,20 +66,20 @@ export default (store: WeakMap) => { return { id: id.description, groupId }; }; - const next = async () => { - const result = await iter(cursor.next(1)); + const next = async (skip: number) => { + const result = await iter(cursor.next(skip)); return { - hasNext: Boolean(await cursor.next(1, true)), + hasNext: Boolean(await cursor.next(skip, true)), hasPrevious: true, ...result, }; }; - const previous = async () => { - const result = await iter(cursor.next(-1)); + const previous = async (skip: number) => { + const result = await iter(cursor.next(-1 * skip)); return { hasNext: true, - hasPrevious: Boolean(await cursor.next(-1, true)), + hasPrevious: Boolean(await cursor.next(-1 * skip, true)), ...result, }; }; From 94ba70526b44f7e82e8da4d1361946f9ad8f445f Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Wed, 30 Oct 2024 21:15:31 +0545 Subject: [PATCH 41/67] break things up and add unit tests --- .../src/components/Modal/ModalNavigation.tsx | 119 +++--------- .../Modal/debouncedNavigator.test.ts | 181 ++++++++++++++++++ .../components/Modal/debouncedNavigator.ts | 70 +++++++ 3 files changed, 282 insertions(+), 88 deletions(-) create mode 100644 app/packages/core/src/components/Modal/debouncedNavigator.test.ts create mode 100644 app/packages/core/src/components/Modal/debouncedNavigator.ts diff --git a/app/packages/core/src/components/Modal/ModalNavigation.tsx b/app/packages/core/src/components/Modal/ModalNavigation.tsx index d35baacba3..7631a37009 100644 --- a/app/packages/core/src/components/Modal/ModalNavigation.tsx +++ b/app/packages/core/src/components/Modal/ModalNavigation.tsx @@ -3,9 +3,10 @@ import { LookerArrowRightIcon, } from "@fiftyone/components"; import * as fos from "@fiftyone/state"; -import React, { useCallback, useEffect, useRef } from "react"; +import React, { useCallback, useEffect, useMemo, useRef } from "react"; import { useRecoilValue, useRecoilValueLoadable } from "recoil"; import styled from "styled-components"; +import { createDebouncedNavigator } from "./debouncedNavigator"; const Arrow = styled.span<{ $isRight?: boolean; @@ -63,92 +64,38 @@ const ModalNavigation = ({ onNavigate }: { onNavigate: () => void }) => { const modal = useRecoilValue(fos.modalSelector); const navigation = useRecoilValue(fos.modalNavigation); - const nextTimeoutRef = useRef(null); - const accumulatedNextOffsetRef = useRef(0); - - const previousTimeoutRef = useRef(null); - const accumulatedPreviousOffsetRef = useRef(0); - const modalRef = useRef(modal); modalRef.current = modal; - const navigateNext = useCallback(() => { - if (!modalRef.current?.hasNext) { - return; - } - - if (!nextTimeoutRef.current) { - // First click: navigate immediately - onNavigate(); - navigation?.next(1).then(setModal); - accumulatedNextOffsetRef.current = 0; - console.log(">!>Immediate next execution"); - } else { - // Subsequent clicks: accumulate offset - accumulatedNextOffsetRef.current += 1; - console.log(">!>Debouncing next"); - } - - // Reset debounce timer - if (nextTimeoutRef.current) { - clearTimeout(nextTimeoutRef.current); - } - - nextTimeoutRef.current = setTimeout(() => { - if (accumulatedNextOffsetRef.current > 0) { - onNavigate(); - navigation?.next(accumulatedNextOffsetRef.current).then(setModal); - accumulatedNextOffsetRef.current = 0; - } - nextTimeoutRef.current = null; - }, 200); - }, [navigation, onNavigate, setModal]); - - const navigatePrevious = useCallback(() => { - if (!modalRef.current?.hasPrevious) { - return; - } - - if (!previousTimeoutRef.current) { - // First click: navigate immediately - onNavigate(); - navigation?.previous(1).then(setModal); - accumulatedPreviousOffsetRef.current = 0; - console.log(">!>Immediate previous execution"); - } else { - // Subsequent clicks: accumulate offset - accumulatedPreviousOffsetRef.current += 1; - console.log(">!>Debouncing previous"); - } - - // Reset debounce timer - if (previousTimeoutRef.current) { - clearTimeout(previousTimeoutRef.current); - } - - previousTimeoutRef.current = setTimeout(() => { - if (accumulatedPreviousOffsetRef.current > 0) { - onNavigate(); - navigation - ?.previous(accumulatedPreviousOffsetRef.current) - .then(setModal); - accumulatedPreviousOffsetRef.current = 0; - } - previousTimeoutRef.current = null; - }, 200); - }, [navigation, onNavigate, setModal]); + const nextNavigator = useMemo( + () => + createDebouncedNavigator({ + isNavigationIllegalWhen: () => modalRef.current?.hasNext === false, + navigateFn: (offset) => navigation?.next(offset).then(setModal), + onNavigationStart: onNavigate, + debounceTime: 200, + }), + [navigation, onNavigate, setModal] + ); + + const previousNavigator = useMemo( + () => + createDebouncedNavigator({ + isNavigationIllegalWhen: () => modalRef.current?.hasPrevious === false, + navigateFn: (offset) => navigation?.previous(offset).then(setModal), + onNavigationStart: onNavigate, + debounceTime: 200, + }), + [navigation, onNavigate, setModal] + ); useEffect(() => { return () => { - if (nextTimeoutRef.current) { - clearTimeout(nextTimeoutRef.current); - } - if (previousTimeoutRef.current) { - clearTimeout(previousTimeoutRef.current); - } + nextNavigator.cleanup(); + previousNavigator.cleanup(); }; - }, []); + }, [nextNavigator, previousNavigator]); const keyboardHandler = useCallback( (e: KeyboardEvent) => { @@ -164,12 +111,12 @@ const ModalNavigation = ({ onNavigate }: { onNavigate: () => void }) => { } if (e.key === "ArrowLeft") { - navigatePrevious(); + previousNavigator.navigate(); } else if (e.key === "ArrowRight") { - navigateNext(); + nextNavigator.navigate(); } }, - [navigateNext, navigatePrevious] + [nextNavigator, previousNavigator] ); fos.useEventHandler(document, "keyup", keyboardHandler); @@ -184,7 +131,7 @@ const ModalNavigation = ({ onNavigate }: { onNavigate: () => void }) => { @@ -194,13 +141,9 @@ const ModalNavigation = ({ onNavigate }: { onNavigate: () => void }) => { $isRight $isSidebarVisible={isSidebarVisible} $sidebarWidth={sidebarwidth} - onClick={navigateNext} + onClick={nextNavigator.navigate} > -
oi
- {accumulatedNextOffsetRef.current > 0 && ( -
{accumulatedNextOffsetRef.current}
- )} )} diff --git a/app/packages/core/src/components/Modal/debouncedNavigator.test.ts b/app/packages/core/src/components/Modal/debouncedNavigator.test.ts new file mode 100644 index 0000000000..91062edfc1 --- /dev/null +++ b/app/packages/core/src/components/Modal/debouncedNavigator.test.ts @@ -0,0 +1,181 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { createDebouncedNavigator } from "./debouncedNavigator"; + +describe("createDebouncedNavigator", () => { + let navigateFn: ReturnType; + let onNavigationStart: ReturnType; + let isNavigationIllegalWhen: ReturnType; + let debouncedNavigator: ReturnType; + + beforeEach(() => { + vi.useFakeTimers(); + navigateFn = vi.fn(); + onNavigationStart = vi.fn(); + isNavigationIllegalWhen = vi.fn().mockReturnValue(false); + debouncedNavigator = createDebouncedNavigator({ + isNavigationIllegalWhen, + navigateFn, + onNavigationStart, + debounceTime: 100, + }); + }); + + afterEach(() => { + vi.clearAllTimers(); + vi.restoreAllMocks(); + }); + + it("should navigate immediately on the first call", () => { + debouncedNavigator.navigate(); + + expect(isNavigationIllegalWhen).toHaveBeenCalled(); + expect(onNavigationStart).toHaveBeenCalledTimes(1); + expect(navigateFn).toHaveBeenCalledWith(1); + }); + + it("should debounce subsequent calls and accumulate offset", () => { + // immediate call + debouncedNavigator.navigate(); + // accumulated + debouncedNavigator.navigate(); + // accumulated + debouncedNavigator.navigate(); + + // only the first call + expect(onNavigationStart).toHaveBeenCalledTimes(1); + + // advance time less than debounceTime + vi.advanceTimersByTime(50); + // another accumulated call + debouncedNavigator.navigate(); + + // advance time to trigger debounce after the last navigate + // need to advance full debounceTime after last call + vi.advanceTimersByTime(100); + + // first immediate call + after debounce + expect(onNavigationStart).toHaveBeenCalledTimes(2); + // immediate call + expect(navigateFn).toHaveBeenCalledWith(1); + // accumulated calls + expect(navigateFn).toHaveBeenCalledWith(3); + }); + + it("should reset after debounce period", () => { + // immediate call + debouncedNavigator.navigate(); + // accumulated + debouncedNavigator.navigate(); + + vi.advanceTimersByTime(100); + + // next navigate call should be immediate again + debouncedNavigator.navigate(); + + expect(onNavigationStart).toHaveBeenCalledTimes(3); + expect(navigateFn).toHaveBeenNthCalledWith(1, 1); + // accumulated offset + expect(navigateFn).toHaveBeenNthCalledWith(2, 1); + expect(navigateFn).toHaveBeenNthCalledWith(3, 1); + }); + + it("should not navigate when isNavigationIllegalWhen returns true", () => { + isNavigationIllegalWhen.mockReturnValueOnce(true); + + debouncedNavigator.navigate(); + + expect(isNavigationIllegalWhen).toHaveBeenCalled(); + expect(onNavigationStart).not.toHaveBeenCalled(); + expect(navigateFn).not.toHaveBeenCalled(); + }); + + it("should cancel pending navigation when cleanup is called", () => { + // immediate call + debouncedNavigator.navigate(); + // accumulated + debouncedNavigator.navigate(); + debouncedNavigator.cleanup(); + + vi.advanceTimersByTime(200); + + // only the immediate call + expect(onNavigationStart).toHaveBeenCalledTimes(1); + expect(navigateFn).toHaveBeenCalledTimes(1); + expect(navigateFn).toHaveBeenCalledWith(1); + }); + + it("should clear timeout when isNavigationIllegalWhen returns true during debounce", () => { + // immediate call + debouncedNavigator.navigate(); + // accumulated + debouncedNavigator.navigate(); + + isNavigationIllegalWhen.mockReturnValue(true); + // should not accumulate further + debouncedNavigator.navigate(); + + vi.advanceTimersByTime(100); + + // only the initial navigation + expect(onNavigationStart).toHaveBeenCalledTimes(1); // + // only immediate call + expect(navigateFn).toHaveBeenCalledTimes(1); + expect(navigateFn).toHaveBeenCalledWith(1); + + // reset mock to allow navigation + isNavigationIllegalWhen.mockReturnValue(false); + + // should navigate immediately + debouncedNavigator.navigate(); + + expect(onNavigationStart).toHaveBeenCalledTimes(2); + expect(navigateFn).toHaveBeenCalledTimes(2); + expect(navigateFn).toHaveBeenCalledWith(1); + }); + + it("should handle multiple sequences correctly", () => { + // first sequence + // immediate + debouncedNavigator.navigate(); + // accumulated + debouncedNavigator.navigate(); + debouncedNavigator.navigate(); + + vi.advanceTimersByTime(100); + + expect(onNavigationStart).toHaveBeenCalledTimes(2); + expect(navigateFn).toHaveBeenNthCalledWith(1, 1); + expect(navigateFn).toHaveBeenNthCalledWith(2, 2); + + // second sequence + // immediate call + debouncedNavigator.navigate(); + // accumulated + debouncedNavigator.navigate(); + + vi.advanceTimersByTime(100); + + expect(onNavigationStart).toHaveBeenCalledTimes(4); + expect(navigateFn).toHaveBeenNthCalledWith(3, 1); + expect(navigateFn).toHaveBeenNthCalledWith(4, 1); + }); + + it("should reset accumulatedOffset when isNavigationIllegalWhen returns true", () => { + // immediate call + debouncedNavigator.navigate(); + // accumulated + debouncedNavigator.navigate(); + + isNavigationIllegalWhen.mockReturnValueOnce(true); + + // should not accumulate further + debouncedNavigator.navigate(); + + vi.advanceTimersByTime(100); + + // only the immediate call + expect(onNavigationStart).toHaveBeenCalledTimes(1); + expect(navigateFn).toHaveBeenCalledTimes(1); + expect(navigateFn).toHaveBeenCalledWith(1); + }); +}); diff --git a/app/packages/core/src/components/Modal/debouncedNavigator.ts b/app/packages/core/src/components/Modal/debouncedNavigator.ts new file mode 100644 index 0000000000..70dfa4524a --- /dev/null +++ b/app/packages/core/src/components/Modal/debouncedNavigator.ts @@ -0,0 +1,70 @@ +interface DebouncedNavigatorOptions { + isNavigationIllegalWhen: () => boolean; + navigateFn: (offset: number) => Promise | void; + onNavigationStart: () => void; + debounceTime?: number; +} + +export function createDebouncedNavigator({ + isNavigationIllegalWhen, + navigateFn, + onNavigationStart, + debounceTime = 100, +}: DebouncedNavigatorOptions) { + let timeout: ReturnType | null = null; + let accumulatedOffset = 0; + let isFirstCall = true; + + const cleanup = () => { + if (timeout) { + clearTimeout(timeout); + timeout = null; + } + accumulatedOffset = 0; + isFirstCall = true; + }; + + const navigate = () => { + if (isNavigationIllegalWhen()) { + if (timeout) { + clearTimeout(timeout); + timeout = null; + } + // Reset state variables + isFirstCall = true; + accumulatedOffset = 0; + return; + } + + if (isFirstCall) { + // first invocation: navigate immediately + onNavigationStart(); + navigateFn(1); + accumulatedOffset = 0; + isFirstCall = false; + } else { + // subsequently, accumulate offset + accumulatedOffset += 1; + } + + // reset debounce timer + if (timeout) { + clearTimeout(timeout); + } + + timeout = setTimeout(() => { + if (accumulatedOffset > 0) { + onNavigationStart(); + navigateFn(accumulatedOffset); + accumulatedOffset = 0; + } + timeout = null; + isFirstCall = true; + }, debounceTime); + }; + + return { + navigate, + cleanup, + }; +} From 65d0ff6f171c42246663f7b773e1ee3410b82781 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Wed, 30 Oct 2024 22:13:42 +0545 Subject: [PATCH 42/67] fix json panel and help panel ref integrity --- app/packages/core/src/components/Modal/hooks.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/app/packages/core/src/components/Modal/hooks.ts b/app/packages/core/src/components/Modal/hooks.ts index 2cdb6d6310..13c8105cbe 100644 --- a/app/packages/core/src/components/Modal/hooks.ts +++ b/app/packages/core/src/components/Modal/hooks.ts @@ -1,16 +1,25 @@ import * as fos from "@fiftyone/state"; import { useHelpPanel, useJSONPanel } from "@fiftyone/state"; -import { useCallback, useContext } from "react"; +import { useCallback, useContext, useRef } from "react"; import { useRecoilCallback } from "recoil"; import { modalContext } from "./modal-context"; export const useLookerHelpers = () => { const jsonPanel = useJSONPanel(); const helpPanel = useHelpPanel(); + + // todo: jsonPanel and helpPanel are not referentially stable + // so use refs here + const jsonPanelRef = useRef(jsonPanel); + const helpPanelRef = useRef(helpPanel); + + jsonPanelRef.current = jsonPanel; + helpPanelRef.current = helpPanel; + const onNavigate = useCallback(() => { - jsonPanel.close(); - helpPanel.close(); - }, [helpPanel, jsonPanel]); + jsonPanelRef.current?.close(); + helpPanelRef.current?.close(); + }, []); return { jsonPanel, From c21cb2b03833512d350ea47a64fb067d4b5a8978 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Wed, 30 Oct 2024 22:14:07 +0545 Subject: [PATCH 43/67] add typing for offset --- app/packages/state/src/hooks/useExpandSample.ts | 13 ++++++------- app/packages/state/src/recoil/modal.ts | 4 ++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/app/packages/state/src/hooks/useExpandSample.ts b/app/packages/state/src/hooks/useExpandSample.ts index 97426efba3..77431333c0 100644 --- a/app/packages/state/src/hooks/useExpandSample.ts +++ b/app/packages/state/src/hooks/useExpandSample.ts @@ -8,7 +8,6 @@ import * as atoms from "../recoil/atoms"; import * as groupAtoms from "../recoil/groups"; import useSetExpandedSample from "./useSetExpandedSample"; import useSetModalState from "./useSetModalState"; -import { useCallback } from "react"; export type Sample = Exclude< Exclude< @@ -66,20 +65,20 @@ export default (store: WeakMap) => { return { id: id.description, groupId }; }; - const next = async (skip: number) => { - const result = await iter(cursor.next(skip)); + const next = async (offset?: number) => { + const result = await iter(cursor.next(offset ?? 1)); return { - hasNext: Boolean(await cursor.next(skip, true)), + hasNext: Boolean(await cursor.next(offset ?? 1, true)), hasPrevious: true, ...result, }; }; - const previous = async (skip: number) => { - const result = await iter(cursor.next(-1 * skip)); + const previous = async (offset: number) => { + const result = await iter(cursor.next(-1 * (offset ?? 1))); return { hasNext: true, - hasPrevious: Boolean(await cursor.next(-1 * skip, true)), + hasPrevious: Boolean(await cursor.next(-1 * (offset ?? 1), true)), ...result, }; }; diff --git a/app/packages/state/src/recoil/modal.ts b/app/packages/state/src/recoil/modal.ts index 52baab87c2..3c30927f5b 100644 --- a/app/packages/state/src/recoil/modal.ts +++ b/app/packages/state/src/recoil/modal.ts @@ -119,8 +119,8 @@ export const isModalActive = selector({ }); export type ModalNavigation = { - next: () => Promise; - previous: () => Promise; + next: (offset?: number) => Promise; + previous: (offset?: number) => Promise; }; export const modalNavigation = atom({ From da97121ac9e638cbb13e317bdd20a4c662dbaac0 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Wed, 30 Oct 2024 22:14:34 +0545 Subject: [PATCH 44/67] change debounce time to 150 --- app/packages/core/src/components/Modal/ModalNavigation.tsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/packages/core/src/components/Modal/ModalNavigation.tsx b/app/packages/core/src/components/Modal/ModalNavigation.tsx index 7631a37009..4b5dc4459b 100644 --- a/app/packages/core/src/components/Modal/ModalNavigation.tsx +++ b/app/packages/core/src/components/Modal/ModalNavigation.tsx @@ -68,13 +68,15 @@ const ModalNavigation = ({ onNavigate }: { onNavigate: () => void }) => { modalRef.current = modal; + // important: make sure all dependencies of the navigators are referentially stable, + // or else the debouncing mechanism won't work const nextNavigator = useMemo( () => createDebouncedNavigator({ isNavigationIllegalWhen: () => modalRef.current?.hasNext === false, navigateFn: (offset) => navigation?.next(offset).then(setModal), onNavigationStart: onNavigate, - debounceTime: 200, + debounceTime: 150, }), [navigation, onNavigate, setModal] ); @@ -85,7 +87,7 @@ const ModalNavigation = ({ onNavigate }: { onNavigate: () => void }) => { isNavigationIllegalWhen: () => modalRef.current?.hasPrevious === false, navigateFn: (offset) => navigation?.previous(offset).then(setModal), onNavigationStart: onNavigate, - debounceTime: 200, + debounceTime: 150, }), [navigation, onNavigate, setModal] ); From ec42f0dc61dc0aeb3a71138415a00ce6357d0df4 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Thu, 31 Oct 2024 00:11:47 +0545 Subject: [PATCH 45/67] more typing and cleanup --- .../src/components/Modal/debouncedNavigator.ts | 2 +- app/packages/core/src/components/Modal/hooks.ts | 4 ++-- app/packages/state/src/hooks/useExpandSample.ts | 16 +++++++++++----- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/app/packages/core/src/components/Modal/debouncedNavigator.ts b/app/packages/core/src/components/Modal/debouncedNavigator.ts index 70dfa4524a..136795bcc5 100644 --- a/app/packages/core/src/components/Modal/debouncedNavigator.ts +++ b/app/packages/core/src/components/Modal/debouncedNavigator.ts @@ -1,6 +1,6 @@ interface DebouncedNavigatorOptions { isNavigationIllegalWhen: () => boolean; - navigateFn: (offset: number) => Promise | void; + navigateFn: (offset: number) => Promise; onNavigationStart: () => void; debounceTime?: number; } diff --git a/app/packages/core/src/components/Modal/hooks.ts b/app/packages/core/src/components/Modal/hooks.ts index 13c8105cbe..ac61249265 100644 --- a/app/packages/core/src/components/Modal/hooks.ts +++ b/app/packages/core/src/components/Modal/hooks.ts @@ -10,8 +10,8 @@ export const useLookerHelpers = () => { // todo: jsonPanel and helpPanel are not referentially stable // so use refs here - const jsonPanelRef = useRef(jsonPanel); - const helpPanelRef = useRef(helpPanel); + const jsonPanelRef = useRef(jsonPanel); + const helpPanelRef = useRef(helpPanel); jsonPanelRef.current = jsonPanel; helpPanelRef.current = helpPanel; diff --git a/app/packages/state/src/hooks/useExpandSample.ts b/app/packages/state/src/hooks/useExpandSample.ts index 77431333c0..414a9587d1 100644 --- a/app/packages/state/src/hooks/useExpandSample.ts +++ b/app/packages/state/src/hooks/useExpandSample.ts @@ -65,20 +65,26 @@ export default (store: WeakMap) => { return { id: id.description, groupId }; }; - const next = async (offset?: number) => { - const result = await iter(cursor.next(offset ?? 1)); + const next = async (offset: number = 1) => { + const nextId = await cursor.next(offset); + const nextCheckId = await cursor.next(offset, true); + + const result = await iter(Promise.resolve(nextId)); return { - hasNext: Boolean(await cursor.next(offset ?? 1, true)), + hasNext: Boolean(nextCheckId), hasPrevious: true, ...result, }; }; const previous = async (offset: number) => { - const result = await iter(cursor.next(-1 * (offset ?? 1))); + const prevId = await cursor.next(-1 * offset); + const prevCheckId = await cursor.next(-1 * offset, true); + + const result = await iter(Promise.resolve(prevId)); return { hasNext: true, - hasPrevious: Boolean(await cursor.next(-1 * (offset ?? 1), true)), + hasPrevious: Boolean(prevCheckId), ...result, }; }; From 20c1c628cd70be284ead1eb5c0e42715b43ea5dd Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Thu, 31 Oct 2024 01:08:49 +0545 Subject: [PATCH 46/67] fix e2e --- .../app/src/components/AnalyticsConsent.tsx | 14 ++++++++++---- e2e-pw/src/oss/constants/index.ts | 1 + e2e-pw/src/oss/fixtures/loader.ts | 18 ++++++++++++++++++ e2e-pw/src/oss/poms/selector.ts | 2 -- .../src/oss/specs/plugins/histograms.spec.ts | 12 +++++------- 5 files changed, 34 insertions(+), 13 deletions(-) diff --git a/app/packages/app/src/components/AnalyticsConsent.tsx b/app/packages/app/src/components/AnalyticsConsent.tsx index a5370a9f31..34ed4d3d25 100644 --- a/app/packages/app/src/components/AnalyticsConsent.tsx +++ b/app/packages/app/src/components/AnalyticsConsent.tsx @@ -85,17 +85,23 @@ export default function AnalyticsConsent({ Help us improve FiftyOne - We use cookies to understand how FiftyOne is used and improve the product. - You can help us by enabling anonymous analytics. + We use cookies to understand how FiftyOne is used and improve the + product. You can help us by enabling anonymous analytics. - + Disable - + diff --git a/e2e-pw/src/oss/constants/index.ts b/e2e-pw/src/oss/constants/index.ts index f1dd4a3808..f070b5bf7c 100644 --- a/e2e-pw/src/oss/constants/index.ts +++ b/e2e-pw/src/oss/constants/index.ts @@ -2,6 +2,7 @@ import { Duration } from "src/oss/utils"; // time to wait for fiftyone app to load export const DEFAULT_APP_LOAD_TIMEOUT = Duration.Minutes(2); +export const POPUP_DISMISS_TIMEOUT = Duration.Seconds(5); export const DEFAULT_APP_PORT = 8787; export const DEFAULT_APP_HOSTNAME = "0.0.0.0"; diff --git a/e2e-pw/src/oss/fixtures/loader.ts b/e2e-pw/src/oss/fixtures/loader.ts index b794922ef8..5a3f7002ea 100644 --- a/e2e-pw/src/oss/fixtures/loader.ts +++ b/e2e-pw/src/oss/fixtures/loader.ts @@ -8,6 +8,7 @@ import { import { PythonRunner } from "src/shared/python-runner/python-runner"; import kill from "tree-kill"; import waitOn from "wait-on"; +import { POPUP_DISMISS_TIMEOUT } from "../constants"; import { Duration } from "../utils"; type WebServerProcessConfig = { @@ -188,5 +189,22 @@ export class OssLoader extends AbstractFiftyoneLoader { {}, { timeout: Duration.Seconds(10) } ); + + // close all pop-ups (cookies, new feature annoucement, etc.) + try { + await page + .getByTestId("btn-dismiss-query-performance-toast") + .click({ timeout: POPUP_DISMISS_TIMEOUT }); + } catch { + console.log("No query performance toast to dismiss"); + } + + try { + await page + .getByTestId("btn-disable-cookies") + .click({ timeout: POPUP_DISMISS_TIMEOUT }); + } catch { + console.log("No cookies button to disable"); + } } } diff --git a/e2e-pw/src/oss/poms/selector.ts b/e2e-pw/src/oss/poms/selector.ts index 71dbb7c1d8..0e81cbb96a 100644 --- a/e2e-pw/src/oss/poms/selector.ts +++ b/e2e-pw/src/oss/poms/selector.ts @@ -60,7 +60,5 @@ class SelectorAsserter { ) ).toBeVisible(); } - - await this.selectorPom.closeResults(); } } diff --git a/e2e-pw/src/oss/specs/plugins/histograms.spec.ts b/e2e-pw/src/oss/specs/plugins/histograms.spec.ts index 28c6a619b2..2fa715e8eb 100644 --- a/e2e-pw/src/oss/specs/plugins/histograms.spec.ts +++ b/e2e-pw/src/oss/specs/plugins/histograms.spec.ts @@ -75,15 +75,13 @@ test("histograms panel", async ({ histogram, panel }) => { "str", "tags", ]); - await expect(await histogram.locator).toHaveScreenshot("bool-histogram.png", { + await expect(histogram.locator).toHaveScreenshot("bool-histogram.png", { animations: "allow", }); + await histogram.selector.closeResults(); await histogram.selectField("float"); - await expect(await histogram.locator).toHaveScreenshot( - "float-histogram.png", - { - animations: "allow", - } - ); + await expect(histogram.locator).toHaveScreenshot("float-histogram.png", { + animations: "allow", + }); }); From 42086ac7b38400f6def6885447c1efa7b272809e Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Thu, 31 Oct 2024 01:23:41 +0545 Subject: [PATCH 47/67] make e2e required again --- .github/workflows/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index bee364d1c9..9e66fa3da4 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -47,7 +47,7 @@ jobs: all-tests: runs-on: ubuntu-latest - needs: [build, lint, test] + needs: [build, lint, test, e2e] if: always() steps: - run: sh -c ${{ From fe27f764a2e7ae5363915d25f7e070280ebb7274 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Thu, 31 Oct 2024 01:59:22 +0545 Subject: [PATCH 48/67] no need to block until canvas-loaded event in grid modal e2e --- e2e-pw/src/oss/specs/regression-tests/media-field.spec.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/e2e-pw/src/oss/specs/regression-tests/media-field.spec.ts b/e2e-pw/src/oss/specs/regression-tests/media-field.spec.ts index 9d54cd2ee1..bda2c4c94c 100644 --- a/e2e-pw/src/oss/specs/regression-tests/media-field.spec.ts +++ b/e2e-pw/src/oss/specs/regression-tests/media-field.spec.ts @@ -64,13 +64,8 @@ test.beforeAll(async ({ fiftyoneLoader }) => { dataset.save()`); }); -test("grid media field", async ({ eventUtils, fiftyoneLoader, grid, page }) => { +test("grid media field", async ({ fiftyoneLoader, grid, page }) => { await fiftyoneLoader.waitUntilGridVisible(page, datasetName); - const wait = eventUtils.getEventReceivedPromiseForPredicate( - "canvas-loaded", - ({ detail }) => detail.sampleFilepath === "/tmp/empty.png" - ); - await wait; await expect(grid.getNthLooker(0)).toHaveScreenshot("grid-media-field.png"); }); From 8758186a95dcc32157fcbe5501bfd98485133365 Mon Sep 17 00:00:00 2001 From: Sashank Aryal <66688606+sashankaryal@users.noreply.github.com> Date: Thu, 31 Oct 2024 02:05:45 +0545 Subject: [PATCH 49/67] cleanup event handlers in looker2d (#4988) --- .../core/src/components/Modal/Modal.tsx | 11 ++- app/packages/looker/src/lookers/abstract.ts | 33 +++++++-- .../lookers/imavid/ima-vid-frame-samples.ts | 68 +++++++++++-------- app/packages/looker/src/util.ts | 50 +++++++++----- .../state/src/hooks/useCreateLooker.ts | 20 ++++-- 5 files changed, 119 insertions(+), 63 deletions(-) diff --git a/app/packages/core/src/components/Modal/Modal.tsx b/app/packages/core/src/components/Modal/Modal.tsx index c7c2b196cf..5b4b0d9ee0 100644 --- a/app/packages/core/src/components/Modal/Modal.tsx +++ b/app/packages/core/src/components/Modal/Modal.tsx @@ -104,6 +104,10 @@ const Modal = () => { } clearModal(); + activeLookerRef.current?.removeEventListener( + "close", + modalCloseHandler + ); }, [clearModal, jsonPanel, helpPanel] ); @@ -183,13 +187,6 @@ const Modal = () => { looker.addEventListener("close", modalCloseHandler); }, []); - // cleanup effect - useEffect(() => { - return () => { - activeLookerRef.current?.removeEventListener("close", modalCloseHandler); - }; - }, []); - const setActiveLookerRef = useCallback( (looker: fos.Lookers) => { activeLookerRef.current = looker; diff --git a/app/packages/looker/src/lookers/abstract.ts b/app/packages/looker/src/lookers/abstract.ts index ad8c70c528..8c253e07bc 100644 --- a/app/packages/looker/src/lookers/abstract.ts +++ b/app/packages/looker/src/lookers/abstract.ts @@ -60,11 +60,17 @@ const getLabelsWorker = (() => { let workers: Worker[]; let next = -1; - return (dispatchEvent) => { + return (dispatchEvent, abortController) => { if (!workers) { workers = []; for (let i = 0; i < numWorkers; i++) { - workers.push(createWorker(LookerUtils.workerCallbacks, dispatchEvent)); + workers.push( + createWorker( + LookerUtils.workerCallbacks, + dispatchEvent, + abortController + ) + ); } } @@ -102,11 +108,14 @@ export abstract class AbstractLooker< private isBatching = false; private isCommittingBatchUpdates = false; + private readonly abortController: AbortController; + constructor( sample: S, config: State["config"], options: Partial = {} ) { + this.abortController = new AbortController(); this.eventTarget = new EventTarget(); this.subscriptions = {}; this.updater = this.makeUpdate(); @@ -383,9 +392,19 @@ export abstract class AbstractLooker< addEventListener( eventType: string, handler: EventListenerOrEventListenerObject | null, - ...args: any[] + optionsOrUseCapture?: boolean | AddEventListenerOptions ) { - this.eventTarget.addEventListener(eventType, handler, ...args); + const argsWithSignal: AddEventListenerOptions = + typeof optionsOrUseCapture === "boolean" + ? { + signal: this.abortController.signal, + capture: optionsOrUseCapture, + } + : { + ...(optionsOrUseCapture ?? {}), + signal: this.abortController.signal, + }; + this.eventTarget.addEventListener(eventType, handler, argsWithSignal); } removeEventListener( @@ -489,6 +508,7 @@ export abstract class AbstractLooker< this.lookerElement.element.parentNode.removeChild( this.lookerElement.element ); + this.abortController.abort(); } abstract updateOptions(options: Partial): void; @@ -674,8 +694,9 @@ export abstract class AbstractLooker< private loadSample(sample: Sample) { const messageUUID = uuid(); - const labelsWorker = getLabelsWorker((event, detail) => - this.dispatchEvent(event, detail) + const labelsWorker = getLabelsWorker( + (event, detail) => this.dispatchEvent(event, detail), + this.abortController ); const listener = ({ data: { sample, coloring, uuid } }) => { if (uuid === messageUUID) { diff --git a/app/packages/looker/src/lookers/imavid/ima-vid-frame-samples.ts b/app/packages/looker/src/lookers/imavid/ima-vid-frame-samples.ts index 641d7f9582..df3fc41e8d 100644 --- a/app/packages/looker/src/lookers/imavid/ima-vid-frame-samples.ts +++ b/app/packages/looker/src/lookers/imavid/ima-vid-frame-samples.ts @@ -22,8 +22,11 @@ export class ImaVidFrameSamples { private readonly storeBufferManager: BufferManager; + private readonly abortController: AbortController; + constructor(storeBufferManager: BufferManager) { this.storeBufferManager = storeBufferManager; + this.abortController = new AbortController(); this.samples = new LRUCache({ max: MAX_FRAME_SAMPLES_CACHE_SIZE, @@ -65,37 +68,45 @@ export class ImaVidFrameSamples { const source = getSampleSrc(standardizedUrls[mediaField]); return new Promise((resolve) => { - image.addEventListener("load", () => { - const sample = this.samples.get(sampleId); - - if (!sample) { - // sample was removed from the cache, this shouldn't happen... - // but if it does, it might be because the cache was cleared - // todo: handle this case better + image.addEventListener( + "load", + () => { + const sample = this.samples.get(sampleId); + + if (!sample) { + // sample was removed from the cache, this shouldn't happen... + // but if it does, it might be because the cache was cleared + // todo: handle this case better + console.error( + "Sample was removed from cache before image loaded", + sampleId + ); + image.src = BASE64_BLACK_IMAGE; + return; + } + + sample.image = image; + resolve(sampleId); + }, + { signal: this.abortController.signal } + ); + + image.addEventListener( + "error", + () => { console.error( - "Sample was removed from cache before image loaded", - sampleId + "Failed to load image for sample with id", + sampleId, + "at url", + source ); - image.src = BASE64_BLACK_IMAGE; - return; - } - sample.image = image; - resolve(sampleId); - }); - - image.addEventListener("error", () => { - console.error( - "Failed to load image for sample with id", - sampleId, - "at url", - source - ); - - // use a placeholder blank black image to not block animation - // setting src should trigger the load event - image.src = BASE64_BLACK_IMAGE; - }); + // use a placeholder blank black image to not block animation + // setting src should trigger the load event + image.src = BASE64_BLACK_IMAGE; + }, + { signal: this.abortController.signal } + ); image.src = source; }); @@ -124,5 +135,6 @@ export class ImaVidFrameSamples { this.reverseFrameIndex.clear(); this.samples.clear(); this.storeBufferManager.reset(); + this.abortController.abort(); } } diff --git a/app/packages/looker/src/util.ts b/app/packages/looker/src/util.ts index fb8cbd9a00..0209ec3a13 100644 --- a/app/packages/looker/src/util.ts +++ b/app/packages/looker/src/util.ts @@ -446,7 +446,8 @@ export const createWorker = ( listeners?: { [key: string]: ((worker: Worker, args: any) => void)[]; }, - dispatchEvent?: DispatchEvent + dispatchEvent?: DispatchEvent, + abortController?: AbortController ): Worker => { let worker: Worker = null; @@ -456,17 +457,26 @@ export const createWorker = ( worker = new Worker(new URL("./worker/index.ts", import.meta.url)); } - worker.onerror = (error) => { - dispatchEvent("error", error); - }; - worker.addEventListener("message", ({ data }) => { - if (data.error) { - const error = !ERRORS[data.error.cls] - ? new Error(data.error.message) - : new ERRORS[data.error.cls](data.error.data, data.error.message); - dispatchEvent("error", new ErrorEvent("error", { error })); - } - }); + worker.addEventListener( + "error", + (error) => { + dispatchEvent("error", error); + }, + { signal: abortController.signal } + ); + + worker.addEventListener( + "message", + ({ data }) => { + if (data.error) { + const error = !ERRORS[data.error.cls] + ? new Error(data.error.message) + : new ERRORS[data.error.cls](data.error.data, data.error.message); + dispatchEvent("error", new ErrorEvent("error", { error })); + } + }, + { signal: abortController.signal } + ); worker.postMessage({ method: "init", @@ -477,13 +487,17 @@ export const createWorker = ( return worker; } - worker.addEventListener("message", ({ data: { method, ...args } }) => { - if (!(method in listeners)) { - return; - } + worker.addEventListener( + "message", + ({ data: { method, ...args } }) => { + if (!(method in listeners)) { + return; + } - listeners[method].forEach((callback) => callback(worker, args)); - }); + listeners[method].forEach((callback) => callback(worker, args)); + }, + { signal: abortController.signal } + ); return worker; }; diff --git a/app/packages/state/src/hooks/useCreateLooker.ts b/app/packages/state/src/hooks/useCreateLooker.ts index 1fc1b748e3..924348c54c 100644 --- a/app/packages/state/src/hooks/useCreateLooker.ts +++ b/app/packages/state/src/hooks/useCreateLooker.ts @@ -18,7 +18,7 @@ import { isNullish, } from "@fiftyone/utilities"; import { get } from "lodash"; -import { useRef } from "react"; +import { useEffect, useRef } from "react"; import { useErrorHandler } from "react-error-boundary"; import { useRelayEnvironment } from "react-relay"; import { useRecoilCallback, useRecoilValue } from "recoil"; @@ -39,6 +39,7 @@ export default >( highlight?: (sample: Sample) => boolean, enableTimeline?: boolean ) => { + const abortControllerRef = useRef(new AbortController()); const environment = useRelayEnvironment(); const selected = useRecoilValue(selectedSamples); const isClip = useRecoilValue(viewAtoms.isClipsView); @@ -68,6 +69,13 @@ export default >( [] ); + useEffect(() => { + return () => { + // sending abort signal to clean up all event handlers + return abortControllerRef.current.abort(); + }; + }, []); + const create = useRecoilCallback( ({ snapshot }) => ( @@ -248,9 +256,13 @@ export default >( } ); - looker.addEventListener("error", (event) => { - handleError(event.error); - }); + looker.addEventListener( + "error", + (event) => { + handleError(event.error); + }, + { signal: abortControllerRef.current.signal } + ); return looker; }, From 54d0912610516c37b1c4829243a1396c57f4f91d Mon Sep 17 00:00:00 2001 From: brimoor Date: Wed, 30 Oct 2024 16:58:34 -0400 Subject: [PATCH 50/67] updating release notes --- docs/source/release-notes.rst | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/source/release-notes.rst b/docs/source/release-notes.rst index 417e4568b0..e50522e87f 100644 --- a/docs/source/release-notes.rst +++ b/docs/source/release-notes.rst @@ -3,22 +3,17 @@ FiftyOne Release Notes .. default-role:: code -FiftyOne Teams 2.1.2 +FiftyOne Teams 2.1.3 -------------------- -*Released October 25, 2024* +*Released November XX, 2024* -Includes all updates from :ref:`FiftyOne 1.0.2 `, plus: +Includes all updates from :ref:`FiftyOne 1.0.2 ` -- Fixed an issue that prevented `delegation_target` from being properly set when - running delegated operations with orchestrator registration enabled -- Resolved a `RuntimeError` that could occur due to concurrency issues when - applying operations that download data from cloud buckets - -.. _release-notes-v1.0.2: +.. _release-notes-v1.0.3: FiftyOne 1.0.2 -------------- -*Released October 25, 2024* +*Released November XX, 2024* Zoo @@ -42,6 +37,13 @@ App - Fixed bug where timeline name wasn't being forwarded in seek utils `#4975 `_ +FiftyOne Teams 2.1.2 +-------------------- +*Released November XX, 2024* + +- Fixed an issue that prevented `delegation_target` from being properly set when + running delegated operations with orchestrator registration enabled + FiftyOne Teams 2.1.1 -------------------- *Released October 14, 2024* From 8f20094eded2cc2412db204e62c903aaed8a0841 Mon Sep 17 00:00:00 2001 From: Benjamin Kane Date: Thu, 31 Oct 2024 10:50:24 -0400 Subject: [PATCH 51/67] Add modal routing optimization (#5014) * remove modal navigation from router transactions * cache gathered paths * e2e updates * update selection test * lint, add comments --- app/packages/app/src/routing/RouterContext.ts | 211 +++++++++++------- app/packages/app/src/routing/matchPath.ts | 2 +- .../app/src/useEvents/useSetGroupSlice.ts | 6 +- .../app/src/useWriters/onSetGroupSlice.ts | 6 +- .../core/src/components/Grid/useRefreshers.ts | 9 +- .../state/src/recoil/pathData/counts.ts | 17 +- .../state/src/recoil/sidebarExpanded.ts | 14 -- e2e-pw/src/oss/poms/grid/index.ts | 8 +- e2e-pw/src/oss/poms/modal/index.ts | 14 +- e2e-pw/src/oss/specs/selection.spec.ts | 2 +- 10 files changed, 167 insertions(+), 122 deletions(-) diff --git a/app/packages/app/src/routing/RouterContext.ts b/app/packages/app/src/routing/RouterContext.ts index e684acaf8d..92a2114e16 100644 --- a/app/packages/app/src/routing/RouterContext.ts +++ b/app/packages/app/src/routing/RouterContext.ts @@ -69,6 +69,8 @@ export const createRouter = ( ): Router => { const history = isNotebook() ? createMemoryHistory() : createBrowserHistory(); + const getEntryResource = makeGetEntryResource(); + let currentEntryResource: Resource>; let nextCurrentEntryResource: Resource>; @@ -79,17 +81,27 @@ export const createRouter = ( >(); const update = (location: FiftyOneLocation, action?: Action) => { - requestAnimationFrame(() => { - for (const [_, [__, onPending]] of subscribers) onPending?.(); - }); currentEntryResource.load().then(({ cleanup }) => { - nextCurrentEntryResource = getEntryResource( - environment, - routes, - location as FiftyOneLocation, - false, - handleError - ); + try { + nextCurrentEntryResource = getEntryResource({ + environment, + routes, + location: location as FiftyOneLocation, + hard: false, + handleError, + }); + } catch (e) { + if (e instanceof Resource) { + // skip the page change if a resource is thrown + return; + } + + throw e; + } + + requestAnimationFrame(() => { + for (const [_, [__, onPending]] of subscribers) onPending?.(); + }); const loadingResource = nextCurrentEntryResource; loadingResource.load().then((entry) => { @@ -135,13 +147,13 @@ export const createRouter = ( load(hard = false) { const runUpdate = !currentEntryResource || hard; if (!currentEntryResource || hard) { - currentEntryResource = getEntryResource( + currentEntryResource = getEntryResource({ environment, - routes, - history.location as FiftyOneLocation, hard, - handleError - ); + handleError, + location: history.location as FiftyOneLocation, + routes, + }); } runUpdate && update(history.location as FiftyOneLocation); return currentEntryResource.load(); @@ -171,79 +183,114 @@ export const createRouter = ( }; }; -const getEntryResource = ( - environment: Environment, - routes: RouteDefinition[], - location: FiftyOneLocation, - hard = false, - handleError?: (error: unknown) => void -): Resource> => { - let route: RouteDefinition; - let matchResult: MatchPathResult | undefined = undefined; - for (let index = 0; index < routes.length; index++) { - route = routes[index]; - const match = matchPath( - location.pathname, - route, - location.search, - location.state - ); - - if (match) { - matchResult = match; - break; +const SKIP_EVENTS = new Set(["modal", "slice"]); + +const makeGetEntryResource = () => { + let currentLocation: FiftyOneLocation; + let currentResource: Resource>; + + const isReusable = (location: FiftyOneLocation) => { + if (currentLocation) { + return ( + SKIP_EVENTS.has(location.state.event || "") || + SKIP_EVENTS.has(currentLocation?.state.event || "") + ); } - } - if (matchResult == null) { - throw new NotFoundError({ path: location.pathname }); - } + return false; + }; - const fetchPolicy = hard ? "network-only" : "store-or-network"; + const getEntryResource = ({ + environment, + handleError, + hard = false, + location, + routes, + }: { + current?: FiftyOneLocation; + environment: Environment; + routes: RouteDefinition[]; + location: FiftyOneLocation; + hard: boolean; + handleError?: (error: unknown) => void; + }): Resource> => { + if (isReusable(location)) { + // throw the current resource (page) if it can be reused + throw currentResource; + } - return new Resource(() => { - return Promise.all([route.component.load(), route.query.load()]).then( - ([component, concreteRequest]) => { - const preloadedQuery = loadQuery( - environment, - concreteRequest, - matchResult.variables || {}, - { - fetchPolicy, - } - ); - - let resolveEntry: (entry: Entry) => void; - const promise = new Promise>((resolve) => { - resolveEntry = resolve; - }); - const subscription = fetchQuery( - environment, - concreteRequest, - matchResult.variables || {}, - { fetchPolicy } - ).subscribe({ - next: (data) => { - const { state: _, ...rest } = location; - resolveEntry({ - state: matchResult.variables as LocationState, - ...rest, - component, - data, - concreteRequest, - preloadedQuery, - cleanup: () => { - subscription?.unsubscribe(); - }, - }); - }, - error: (error) => handleError?.(error), - }); + let route: RouteDefinition; + let matchResult: MatchPathResult | undefined = undefined; + for (let index = 0; index < routes.length; index++) { + route = routes[index]; + const match = matchPath( + location.pathname, + route, + location.search, + location.state + ); - return promise; + if (match) { + matchResult = match; + break; } - ); - }); + } + + if (matchResult == null) { + throw new NotFoundError({ path: location.pathname }); + } + + const fetchPolicy = hard ? "network-only" : "store-or-network"; + + currentLocation = location; + currentResource = new Resource(() => { + return Promise.all([route.component.load(), route.query.load()]).then( + ([component, concreteRequest]) => { + const preloadedQuery = loadQuery( + environment, + concreteRequest, + matchResult.variables || {}, + { + fetchPolicy, + } + ); + + let resolveEntry: (entry: Entry) => void; + const promise = new Promise>((resolve) => { + resolveEntry = resolve; + }); + const subscription = fetchQuery( + environment, + concreteRequest, + matchResult.variables || {}, + { fetchPolicy } + ).subscribe({ + next: (data) => { + const { state: _, ...rest } = location; + resolveEntry({ + state: matchResult.variables as LocationState, + ...rest, + component, + data, + concreteRequest, + preloadedQuery, + cleanup: () => { + subscription?.unsubscribe(); + }, + }); + }, + error: (error) => handleError?.(error), + }); + + return promise; + } + ); + }); + + return currentResource; + }; + + return getEntryResource; }; export const RouterContext = React.createContext< diff --git a/app/packages/app/src/routing/matchPath.ts b/app/packages/app/src/routing/matchPath.ts index b8c7533359..1223647081 100644 --- a/app/packages/app/src/routing/matchPath.ts +++ b/app/packages/app/src/routing/matchPath.ts @@ -10,7 +10,7 @@ const compilePath = (path: string) => }); export type LocationState = { - event?: "modal"; + event?: "modal" | "slice"; fieldVisibility?: State.FieldVisibilityStage; groupSlice?: string; modalSelector?: ModalSelector; diff --git a/app/packages/app/src/useEvents/useSetGroupSlice.ts b/app/packages/app/src/useEvents/useSetGroupSlice.ts index 9ef152cea5..a58ba62754 100644 --- a/app/packages/app/src/useEvents/useSetGroupSlice.ts +++ b/app/packages/app/src/useEvents/useSetGroupSlice.ts @@ -15,7 +15,11 @@ const useSetGroupSlice: EventHandlerHook = ({ router, session }) => { session.current.sessionGroupSlice = slice; }); - router.push(pathname, { ...router.location.state, groupSlice: slice }); + router.push(pathname, { + ...router.location.state, + event: "slice", + groupSlice: slice, + }); }, [router, session] ); diff --git a/app/packages/app/src/useWriters/onSetGroupSlice.ts b/app/packages/app/src/useWriters/onSetGroupSlice.ts index f2a58d37ba..4045cac469 100644 --- a/app/packages/app/src/useWriters/onSetGroupSlice.ts +++ b/app/packages/app/src/useWriters/onSetGroupSlice.ts @@ -13,7 +13,11 @@ const onSetGroupSlice: RegisteredWriter<"sessionGroupSlice"> = const pathname = router.history.location.pathname + string; - router.push(pathname, { ...router.location.state, groupSlice: slice }); + router.push(pathname, { + ...router.location.state, + event: "slice", + groupSlice: slice, + }); if (env().VITE_NO_STATE) return; commitMutation(environment, { diff --git a/app/packages/core/src/components/Grid/useRefreshers.ts b/app/packages/core/src/components/Grid/useRefreshers.ts index f811d36b1a..10922ab09f 100644 --- a/app/packages/core/src/components/Grid/useRefreshers.ts +++ b/app/packages/core/src/components/Grid/useRefreshers.ts @@ -63,13 +63,8 @@ export default function useRefreshers() { useEffect( () => - subscribe(({ event }, { reset }, previous) => { - if ( - event === "fieldVisibility" || - event === "modal" || - previous?.event === "modal" - ) - return; + subscribe(({ event }, { reset }) => { + if (event === "fieldVisibility") return; // if not a modal page change, reset the grid location reset(gridAt); diff --git a/app/packages/state/src/recoil/pathData/counts.ts b/app/packages/state/src/recoil/pathData/counts.ts index 6059375fcd..1de0e8d8eb 100644 --- a/app/packages/state/src/recoil/pathData/counts.ts +++ b/app/packages/state/src/recoil/pathData/counts.ts @@ -146,6 +146,21 @@ export const counts = selectorFamily({ }, }); +export const gatheredPaths = selectorFamily({ + key: "gatheredPaths", + get: + ({ + embeddedDocType, + ftype, + }: { + embeddedDocType?: string | string[]; + ftype: string | string[]; + }) => + ({ get }) => { + return [...new Set(gatherPaths(get, ftype, embeddedDocType))]; + }, +}); + export const cumulativeCounts = selectorFamily< { [key: string]: number }, { @@ -160,7 +175,7 @@ export const cumulativeCounts = selectorFamily< get: ({ extended, path: key, modal, ftype, embeddedDocType }) => ({ get }) => { - return [...new Set(gatherPaths(get, ftype, embeddedDocType))].reduce( + return get(gatheredPaths({ ftype, embeddedDocType })).reduce( (result, path) => { const data = get(counts({ extended, modal, path: `${path}.${key}` })); for (const value in data) { diff --git a/app/packages/state/src/recoil/sidebarExpanded.ts b/app/packages/state/src/recoil/sidebarExpanded.ts index 2c83c5f8cd..2e3044e9d3 100644 --- a/app/packages/state/src/recoil/sidebarExpanded.ts +++ b/app/packages/state/src/recoil/sidebarExpanded.ts @@ -1,4 +1,3 @@ -import { subscribe } from "@fiftyone/relay"; import { DefaultValue, atom, atomFamily, selectorFamily } from "recoil"; export const sidebarExpandedStore = atomFamily< @@ -7,12 +6,6 @@ export const sidebarExpandedStore = atomFamily< >({ key: "sidebarExpandedStore", default: {}, - effects: [ - ({ node }) => - subscribe(({ event }, { reset }, previous) => { - event !== "modal" && previous?.event !== "modal" && reset(node); - }), - ], }); export const sidebarExpanded = selectorFamily< @@ -36,13 +29,6 @@ export const sidebarExpanded = selectorFamily< export const granularSidebarExpandedStore = atom<{ [key: string]: boolean }>({ key: "granularSidebarExpandedStore", default: {}, - effects: [ - ({ node }) => - subscribe( - ({ event }, { set }, previous) => - event !== "modal" && previous?.event !== "modal" && set(node, {}) - ), - ], }); export const granularSidebarExpanded = selectorFamily({ diff --git a/e2e-pw/src/oss/poms/grid/index.ts b/e2e-pw/src/oss/poms/grid/index.ts index daa42cd6f0..b05ab5e870 100644 --- a/e2e-pw/src/oss/poms/grid/index.ts +++ b/e2e-pw/src/oss/poms/grid/index.ts @@ -1,4 +1,4 @@ -import { expect, Locator, Page } from "src/oss/fixtures"; +import { Locator, Page, expect } from "src/oss/fixtures"; import { EventUtils } from "src/shared/event-utils"; import { GridActionsRowPom } from "../action-row/grid-actions-row"; import { GridSliceSelectorPom } from "../action-row/grid-slice-selector"; @@ -52,9 +52,7 @@ export class GridPom { } async openNthSample(n: number) { - await this.url.pageChange(() => - this.getNthLooker(n).click({ position: { x: 10, y: 80 } }) - ); + await this.getNthLooker(n).click({ position: { x: 10, y: 80 } }); } async openFirstSample() { @@ -80,7 +78,7 @@ export class GridPom { } async selectSlice(slice: string) { - await this.url.pageChange(() => this.sliceSelector.selectSlice(slice)); + await this.sliceSelector.selectSlice(slice); } /** diff --git a/e2e-pw/src/oss/poms/modal/index.ts b/e2e-pw/src/oss/poms/modal/index.ts index 176856107f..33a8e58cdc 100644 --- a/e2e-pw/src/oss/poms/modal/index.ts +++ b/e2e-pw/src/oss/poms/modal/index.ts @@ -1,4 +1,4 @@ -import { expect, Locator, Page } from "src/oss/fixtures"; +import { Locator, Page, expect } from "src/oss/fixtures"; import { EventUtils } from "src/shared/event-utils"; import { Duration } from "../../utils"; import { ModalTaggerPom } from "../action-row/tagger/modal-tagger"; @@ -118,11 +118,9 @@ export class ModalPom { ) { const currentSampleId = await this.sidebar.getSampleId(); - await this.url.pageChange(() => - this.locator - .getByTestId(`nav-${direction === "forward" ? "right" : "left"}-button`) - .click() - ); + await this.locator + .getByTestId(`nav-${direction === "forward" ? "right" : "left"}-button`) + .click(); // wait for sample id to change await this.page.waitForFunction((currentSampleId) => { @@ -219,9 +217,7 @@ export class ModalPom { async close() { // close by clicking outside of modal - await this.url.pageChange(() => - this.page.click("body", { position: { x: 0, y: 0 } }) - ); + await this.page.click("body", { position: { x: 0, y: 0 } }); } async navigateNextSample(allowErrorInfo = false) { diff --git a/e2e-pw/src/oss/specs/selection.spec.ts b/e2e-pw/src/oss/specs/selection.spec.ts index 604b0c015b..858ab29d3f 100644 --- a/e2e-pw/src/oss/specs/selection.spec.ts +++ b/e2e-pw/src/oss/specs/selection.spec.ts @@ -95,7 +95,7 @@ extensionDatasetNamePairs.forEach(([extension, datasetName]) => { await grid.openNthSample(1); await modal.assert.verifySelectionCount(1); - await grid.url.back(); + await modal.close(); await modal.assert.isClosed(); await grid.assert.isSelectionCountEqualTo(1); }); From 1ecde84d19efa5a00d1900e2f15d85a2c9a23f99 Mon Sep 17 00:00:00 2001 From: Benjamin Kane Date: Thu, 31 Oct 2024 10:50:45 -0400 Subject: [PATCH 52/67] freeze tensorflow dep (#5016) Co-authored-by: Br2850 --- requirements/github.txt | 2 +- requirements/test.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/github.txt b/requirements/github.txt index 64ae132326..cc56602780 100644 --- a/requirements/github.txt +++ b/requirements/github.txt @@ -4,7 +4,7 @@ numpy<2 pydicom>=2.2.0 shapely>=1.7.1 -tensorflow +tensorflow==2.17.0 tensorflow-datasets torch torchvision diff --git a/requirements/test.txt b/requirements/test.txt index 9173566d9c..5e619a4ee7 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -7,5 +7,5 @@ pytest-cov==4.0.0 pytest-mock==3.10.0 pytest-asyncio shapely -tensorflow +tensorflow==2.17.0 twine>=3 From 1a9c5d57e425fade23d5be43136fbf3802530461 Mon Sep 17 00:00:00 2001 From: Sashank Aryal <66688606+sashankaryal@users.noreply.github.com> Date: Thu, 31 Oct 2024 20:36:12 +0545 Subject: [PATCH 53/67] don't throw in spotlight.destroy() (#5017) * assert lastVisibleItem before computedHidden * console.error instead of throw when spotlight destroy called and spotlight is not attached --- .../components/src/components/AdaptiveMenu/index.tsx | 6 ++++-- app/packages/spotlight/src/index.ts | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/app/packages/components/src/components/AdaptiveMenu/index.tsx b/app/packages/components/src/components/AdaptiveMenu/index.tsx index 13b6f03077..3d86f7bbe1 100644 --- a/app/packages/components/src/components/AdaptiveMenu/index.tsx +++ b/app/packages/components/src/components/AdaptiveMenu/index.tsx @@ -53,8 +53,10 @@ export default function AdaptiveMenu(props: AdaptiveMenuPropsType) { if (!containerElem) return; hideOverflowingNodes(containerElem, (_: number, lastVisibleItemId) => { const lastVisibleItem = itemsById[lastVisibleItemId]; - const computedHidden = items.length - lastVisibleItem.index - 1; - setHidden(computedHidden); + if (lastVisibleItem?.index) { + const computedHidden = items.length - lastVisibleItem.index - 1; + setHidden(computedHidden); + } }); } diff --git a/app/packages/spotlight/src/index.ts b/app/packages/spotlight/src/index.ts index 689d69e8b1..081f181459 100644 --- a/app/packages/spotlight/src/index.ts +++ b/app/packages/spotlight/src/index.ts @@ -104,7 +104,8 @@ export default class Spotlight extends EventTarget { destroy(): void { if (!this.attached) { - throw new Error("spotlight is not attached"); + console.error("spotlight is not attached"); + return; } this.#backward?.remove(); From 5922bcdc28c8c80f30cba61a3d5fceb97c150009 Mon Sep 17 00:00:00 2001 From: Sashank Aryal <66688606+sashankaryal@users.noreply.github.com> Date: Thu, 31 Oct 2024 20:37:45 +0545 Subject: [PATCH 54/67] fix video regression + e2e (#5020) * add missing await * optional chaining for signal controller --- app/packages/looker/src/lookers/abstract.ts | 4 ++-- .../looker/src/lookers/imavid/ima-vid-frame-samples.ts | 4 ++-- app/packages/looker/src/util.ts | 6 +++--- .../group-video/default-video-slice-group.spec.ts | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/packages/looker/src/lookers/abstract.ts b/app/packages/looker/src/lookers/abstract.ts index 8c253e07bc..c290716c8c 100644 --- a/app/packages/looker/src/lookers/abstract.ts +++ b/app/packages/looker/src/lookers/abstract.ts @@ -397,12 +397,12 @@ export abstract class AbstractLooker< const argsWithSignal: AddEventListenerOptions = typeof optionsOrUseCapture === "boolean" ? { - signal: this.abortController.signal, + signal: this.abortController?.signal, capture: optionsOrUseCapture, } : { ...(optionsOrUseCapture ?? {}), - signal: this.abortController.signal, + signal: this.abortController?.signal, }; this.eventTarget.addEventListener(eventType, handler, argsWithSignal); } diff --git a/app/packages/looker/src/lookers/imavid/ima-vid-frame-samples.ts b/app/packages/looker/src/lookers/imavid/ima-vid-frame-samples.ts index df3fc41e8d..2e92b8f45d 100644 --- a/app/packages/looker/src/lookers/imavid/ima-vid-frame-samples.ts +++ b/app/packages/looker/src/lookers/imavid/ima-vid-frame-samples.ts @@ -88,7 +88,7 @@ export class ImaVidFrameSamples { sample.image = image; resolve(sampleId); }, - { signal: this.abortController.signal } + { signal: this.abortController?.signal } ); image.addEventListener( @@ -105,7 +105,7 @@ export class ImaVidFrameSamples { // setting src should trigger the load event image.src = BASE64_BLACK_IMAGE; }, - { signal: this.abortController.signal } + { signal: this.abortController?.signal } ); image.src = source; diff --git a/app/packages/looker/src/util.ts b/app/packages/looker/src/util.ts index 0209ec3a13..db64ba40ee 100644 --- a/app/packages/looker/src/util.ts +++ b/app/packages/looker/src/util.ts @@ -462,7 +462,7 @@ export const createWorker = ( (error) => { dispatchEvent("error", error); }, - { signal: abortController.signal } + { signal: abortController?.signal } ); worker.addEventListener( @@ -475,7 +475,7 @@ export const createWorker = ( dispatchEvent("error", new ErrorEvent("error", { error })); } }, - { signal: abortController.signal } + { signal: abortController?.signal } ); worker.postMessage({ @@ -496,7 +496,7 @@ export const createWorker = ( listeners[method].forEach((callback) => callback(worker, args)); }, - { signal: abortController.signal } + { signal: abortController?.signal } ); return worker; }; diff --git a/e2e-pw/src/oss/specs/regression-tests/group-video/default-video-slice-group.spec.ts b/e2e-pw/src/oss/specs/regression-tests/group-video/default-video-slice-group.spec.ts index 685514818f..d0b31470c8 100644 --- a/e2e-pw/src/oss/specs/regression-tests/group-video/default-video-slice-group.spec.ts +++ b/e2e-pw/src/oss/specs/regression-tests/group-video/default-video-slice-group.spec.ts @@ -71,7 +71,7 @@ test.describe("default video slice group", () => { }); test.beforeEach(async ({ page, fiftyoneLoader }) => { - fiftyoneLoader.waitUntilGridVisible(page, datasetName); + await fiftyoneLoader.waitUntilGridVisible(page, datasetName); }); test("video as default slice renders", async ({ grid, modal }) => { From 35f8f610f95a1a2697633ff8e3bd9a51d84fbde4 Mon Sep 17 00:00:00 2001 From: Sashank Aryal <66688606+sashankaryal@users.noreply.github.com> Date: Thu, 31 Oct 2024 22:05:22 +0545 Subject: [PATCH 55/67] release tooltip event listener adding func (#5019) Co-authored-by: Benjamin Kane --- .../components/Modal/ModalSamplePlugin.tsx | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/app/packages/core/src/components/Modal/ModalSamplePlugin.tsx b/app/packages/core/src/components/Modal/ModalSamplePlugin.tsx index 249285433f..44e06ec6ea 100644 --- a/app/packages/core/src/components/Modal/ModalSamplePlugin.tsx +++ b/app/packages/core/src/components/Modal/ModalSamplePlugin.tsx @@ -1,6 +1,6 @@ import { ErrorBoundary } from "@fiftyone/components"; import * as fos from "@fiftyone/state"; -import React, { Suspense, useEffect } from "react"; +import React, { Suspense, useCallback, useEffect, useMemo } from "react"; import { useRecoilCallback, useRecoilValue, useSetRecoilState } from "recoil"; import styled from "styled-components"; import Group from "./Group"; @@ -48,18 +48,30 @@ export const ModalSample = React.memo(() => { const { activeLookerRef, onLookerSetSubscribers } = useModalContext(); - useEffect(() => { - onLookerSetSubscribers.current.push((looker) => { + const addTooltipEventListener = useMemo(() => { + return (looker: fos.Lookers) => { looker.addEventListener("tooltip", tooltipEventHandler); - }); + }; + }, []); + + useEffect(() => { + onLookerSetSubscribers.current.push(addTooltipEventListener); return () => { activeLookerRef?.current?.removeEventListener( "tooltip", tooltipEventHandler ); + onLookerSetSubscribers.current = onLookerSetSubscribers.current.filter( + (fn) => fn !== addTooltipEventListener + ); }; - }, [activeLookerRef, onLookerSetSubscribers, tooltipEventHandler]); + }, [ + activeLookerRef, + addTooltipEventListener, + onLookerSetSubscribers, + tooltipEventHandler, + ]); useEffect(() => { // reset tooltip state when modal is closed From 3a2d84c5d65a7471af93a9037f3e5325f5f768a8 Mon Sep 17 00:00:00 2001 From: topher Date: Thu, 31 Oct 2024 19:41:54 -0400 Subject: [PATCH 56/67] fix teams install command (#5024) --- docs/source/teams/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/teams/installation.rst b/docs/source/teams/installation.rst index 277010cd86..42a70cc7b8 100644 --- a/docs/source/teams/installation.rst +++ b/docs/source/teams/installation.rst @@ -46,7 +46,7 @@ private PyPI server as shown below: .. code-block:: shell - pip install --index-url https://{$TOKEN}@pypi.fiftyone.ai fiftyone + pip install --index-url https://${TOKEN}@pypi.fiftyone.ai fiftyone .. note:: From 60df431604395d7009f9dcadf57e581835423788 Mon Sep 17 00:00:00 2001 From: Alexander Foley Date: Fri, 1 Nov 2024 09:42:15 -0400 Subject: [PATCH 57/67] chore: Add deprecation notice --- docs/source/deprecation.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/source/deprecation.rst b/docs/source/deprecation.rst index 277b102b3b..c5637ef0b0 100644 --- a/docs/source/deprecation.rst +++ b/docs/source/deprecation.rst @@ -26,3 +26,12 @@ Python 3.8 `Python 3.8 `_ transitions to `end-of-life` effective October of 2024. FiftyOne releases after September 30, 2024 will no longer support Python 3.8. + +Kubernetes 1.27 +--------------- +*Support ended November 1, 2024* + +`Kubernetes 1.27 `_ +transitioned to `end-of-life` effective July of 2024. FiftyOne Teams +releases after October 31, 2024 might no longer be compatible with +Kubernetes 1.27 and older. From 7b4e73577597db34b5f9e26b2b87f06c338103a8 Mon Sep 17 00:00:00 2001 From: Benjamin Kane Date: Fri, 1 Nov 2024 11:17:53 -0400 Subject: [PATCH 58/67] Spotlight cleanup (#5015) * fix spotlight cleanup * remove modal navigation from router transactions * cache gathered paths * e2e updates * update selection test * lint, add comments * freeze tensorflow dep * add destroy items --------- Co-authored-by: Br2850 --- .../spotlight/src/createScrollReader.ts | 15 ++++-- app/packages/spotlight/src/index.ts | 10 ++-- app/packages/spotlight/src/row.ts | 46 ++++++++++++++----- app/packages/spotlight/src/section.ts | 11 +++-- app/packages/spotlight/src/types.ts | 3 +- 5 files changed, 57 insertions(+), 28 deletions(-) diff --git a/app/packages/spotlight/src/createScrollReader.ts b/app/packages/spotlight/src/createScrollReader.ts index dee64107f2..d164f1b5ab 100644 --- a/app/packages/spotlight/src/createScrollReader.ts +++ b/app/packages/spotlight/src/createScrollReader.ts @@ -15,14 +15,17 @@ export default function createScrollReader( let timeout: ReturnType; let zooming = false; - element.addEventListener("scroll", () => { + const scroll = () => { scrolling = true; - }); + }; + element.addEventListener("scroll", scroll); - element.addEventListener("scrollend", () => { + const scrollEnd = () => { scrolling = false; requestAnimationFrame(() => render(zooming, true)); - }); + }; + + element.addEventListener("scrollend", scrollEnd); const updateScrollStatus = () => { const threshold = getScrollSpeedThreshold(); @@ -68,7 +71,9 @@ export default function createScrollReader( return { destroy: () => { destroyed = true; + element.removeEventListener("scroll", scroll); + element.removeEventListener("scrollend", scrollEnd); }, - zooming: () => zooming + zooming: () => zooming, }; } diff --git a/app/packages/spotlight/src/index.ts b/app/packages/spotlight/src/index.ts index 081f181459..31a4bb6f7c 100644 --- a/app/packages/spotlight/src/index.ts +++ b/app/packages/spotlight/src/index.ts @@ -18,7 +18,7 @@ import { SCROLLBAR_WIDTH, TWO, ZERO, - ZOOMING_COEFFICIENT + ZOOMING_COEFFICIENT, } from "./constants"; import createScrollReader from "./createScrollReader"; import { Load, RowChange } from "./events"; @@ -108,8 +108,8 @@ export default class Spotlight extends EventTarget { return; } - this.#backward?.remove(); - this.#forward?.remove(); + this.#backward?.destroy(); + this.#forward?.destroy(); this.#element?.classList.remove(styles.spotlightLoaded); this.#element?.remove(); this.#scrollReader?.destroy(); @@ -204,7 +204,7 @@ export default class Spotlight extends EventTarget { const backward = this.#forward; this.#forward = section; this.#forward.attach(this.#element); - this.#backward.remove(); + this.#backward.destroy(); this.#backward = backward; offset = before - this.#containerHeight + this.#config.spacing; } @@ -272,7 +272,7 @@ export default class Spotlight extends EventTarget { this.#backward = result.section; this.#backward.attach(this.#element); - this.#forward.remove(); + this.#forward.destroy(); this.#forward = forward; } diff --git a/app/packages/spotlight/src/row.ts b/app/packages/spotlight/src/row.ts index d904e86eb0..7d4242bec9 100644 --- a/app/packages/spotlight/src/row.ts +++ b/app/packages/spotlight/src/row.ts @@ -13,6 +13,7 @@ export default class Row { #from: number; #hidden: boolean; + readonly #aborter: AbortController = new AbortController(); readonly #config: SpotlightConfig; readonly #dangle?: boolean; readonly #container: HTMLDivElement = create(DIV); @@ -47,7 +48,7 @@ export default class Row { element.style.top = pixels(ZERO); if (config.onItemClick) { - element.addEventListener("click", (event) => { + const handler = (event) => { if (event.metaKey || event.shiftKey) { return; } @@ -59,18 +60,13 @@ export default class Row { item, iter, }); + }; + + element.addEventListener("click", handler, { + signal: this.#aborter.signal, }); - element.addEventListener("contextmenu", (event) => { - if (event.metaKey || event.shiftKey) { - return; - } - event.preventDefault(); - focus(item.id); - config.onItemClick({ - event, - item, - iter, - }); + element.addEventListener("contextmenu", handler, { + signal: this.#aborter.signal, }); } @@ -123,6 +119,11 @@ export default class Row { return this.#row[this.#row.length - ONE].item.id; } + destroy() { + this.#destroyItems(); + this.#aborter.abort(); + } + has(item: string) { for (const i of this.#row) { if (i.item.id.description === item) { @@ -138,6 +139,7 @@ export default class Row { } this.#container.remove(); + this.#destroyItems(); } show( @@ -225,4 +227,24 @@ export default class Row { const set = new Set(this.#row.map(({ item }) => item.aspectRatio)); return set.size === ONE ? this.#row[ZERO].item.aspectRatio : null; } + + #destroyItems() { + const destroy = this.#config.destroy; + if (!destroy) { + return; + } + + const errors = []; + for (const item of this.#row) { + try { + destroy(item.item.id); + } catch (e) { + errors.push(e); + } + } + + if (errors.length > 0) { + console.error("Errors occurred during row destruction:", errors); + } + } } diff --git a/app/packages/spotlight/src/section.ts b/app/packages/spotlight/src/section.ts index c495aa3dc3..cd0d39b2b3 100644 --- a/app/packages/spotlight/src/section.ts +++ b/app/packages/spotlight/src/section.ts @@ -108,6 +108,12 @@ export default class Section { : element.appendChild(this.#section); } + destroy() { + this.#section.remove(); + for (const row of this.#rows) row.destroy(); + this.#rows = []; + } + find(item: string): Row | null { for (const row of this.#rows) { if (row.has(item)) { @@ -118,11 +124,6 @@ export default class Section { return null; } - remove() { - this.#section.remove(); - this.#rows = []; - } - render({ config, target, diff --git a/app/packages/spotlight/src/types.ts b/app/packages/spotlight/src/types.ts index 7954fd0bb3..a613eb2632 100644 --- a/app/packages/spotlight/src/types.ts +++ b/app/packages/spotlight/src/types.ts @@ -56,8 +56,9 @@ export type Request = (key: K) => Promise<{ }>; export interface SpotlightConfig { - get: Get; at?: At; + destroy?: (id: ID) => void; + get: Get; key: K; offset?: number; onItemClick?: ItemClick; From d586c4ca2664c36d31496f066c00d8440530d2cb Mon Sep 17 00:00:00 2001 From: Sashank Aryal <66688606+sashankaryal@users.noreply.github.com> Date: Sat, 2 Nov 2024 05:50:17 +0545 Subject: [PATCH 59/67] remove reload_on_navigation modal panel option (#5018) --- app/packages/operators/src/Panel/register.tsx | 1 - app/packages/plugins/src/index.ts | 6 ------ app/packages/spaces/src/components/Panel.tsx | 14 +++----------- fiftyone/operators/operations.py | 7 +------ fiftyone/operators/panel.py | 7 ------- 5 files changed, 4 insertions(+), 31 deletions(-) diff --git a/app/packages/operators/src/Panel/register.tsx b/app/packages/operators/src/Panel/register.tsx index 01e4522690..24a8fa973d 100644 --- a/app/packages/operators/src/Panel/register.tsx +++ b/app/packages/operators/src/Panel/register.tsx @@ -23,7 +23,6 @@ export default function registerPanel(ctx: ExecutionContext) { panelOptions: { allowDuplicates: ctx.params.allow_duplicates, helpMarkdown: ctx.params.help_markdown, - reloadOnNavigation: ctx.params.reload_on_navigation, surfaces: ctx.params.surfaces, }, }); diff --git a/app/packages/plugins/src/index.ts b/app/packages/plugins/src/index.ts index a14e1423e4..cd527a74c8 100644 --- a/app/packages/plugins/src/index.ts +++ b/app/packages/plugins/src/index.ts @@ -325,12 +325,6 @@ type PanelOptions = { * Content displayed on the right side of the label in the panel title bar. */ TabIndicator?: React.ComponentType; - - /** - * If true, the plugin will be remounted when the user navigates to a different sample or group. - * This is only applicable to plugins that are in a modal. - */ - reloadOnNavigation?: boolean; }; type PluginComponentProps = T & { diff --git a/app/packages/spaces/src/components/Panel.tsx b/app/packages/spaces/src/components/Panel.tsx index 3b8c9859ee..6110907f52 100644 --- a/app/packages/spaces/src/components/Panel.tsx +++ b/app/packages/spaces/src/components/Panel.tsx @@ -1,7 +1,7 @@ import { CenteredStack, scrollable } from "@fiftyone/components"; import * as fos from "@fiftyone/state"; import React, { useEffect } from "react"; -import { useRecoilValue, useSetRecoilState } from "recoil"; +import { useSetRecoilState } from "recoil"; import { PANEL_LOADING_TIMEOUT } from "../constants"; import { PanelContext } from "../contexts"; import { useReactivePanel } from "../hooks"; @@ -20,8 +20,6 @@ function Panel(props: PanelProps) { const setPanelIdToScope = useSetRecoilState(panelIdToScopeAtom); const scope = isModalPanel ? "modal" : "grid"; - const thisModalUniqueId = useRecoilValue(fos.currentModalUniqueId); - useEffect(() => { setPanelIdToScope((ids) => ({ ...ids, [node.id]: scope })); }, [scope, setPanelIdToScope, node.id]); @@ -42,9 +40,7 @@ function Panel(props: PanelProps) { ); } - const { component: Component, panelOptions } = panel; - - const shouldKeyComponent = isModalPanel && panelOptions?.reloadOnNavigation; + const { component: Component } = panel; return ( - + ); diff --git a/fiftyone/operators/operations.py b/fiftyone/operators/operations.py index 933f6f3d55..e89ec3bd66 100644 --- a/fiftyone/operators/operations.py +++ b/fiftyone/operators/operations.py @@ -305,7 +305,6 @@ def register_panel( light_icon=None, dark_icon=None, surfaces="grid", - reload_on_navigation=False, on_load=None, on_unload=None, on_change=None, @@ -333,10 +332,7 @@ def register_panel( is in dark mode surfaces ('grid'): surfaces in which to show the panel. Must be one of 'grid', 'modal', or 'grid modal' - reload_on_navigation (False): whether to reload the panel when the - user navigates to a new page. This is only applicable to panels - that are not shown in a modal - on_load (None): an operator to invoke when the panel is loaded + on_load (None): an operator to invoke when the panel is loaded on_unload (None): an operator to invoke when the panel is unloaded on_change (None): an operator to invoke when the panel state changes @@ -367,7 +363,6 @@ def register_panel( "light_icon": light_icon, "dark_icon": dark_icon, "surfaces": surfaces, - "reload_on_navigation": reload_on_navigation, "on_load": on_load, "on_unload": on_unload, "on_change": on_change, diff --git a/fiftyone/operators/panel.py b/fiftyone/operators/panel.py index b9d897c2fd..dec4a2e693 100644 --- a/fiftyone/operators/panel.py +++ b/fiftyone/operators/panel.py @@ -28,9 +28,6 @@ class PanelConfig(OperatorConfig): in dark mode allow_multiple (False): whether to allow multiple instances of the panel to be opened - reload_on_navigation (False): whether to reload the panel when the - user navigates to a new page. This is only applicable to panels - that are not shown in a modal surfaces ("grid"): the surfaces on which the panel can be displayed help_markdown (None): a markdown string to display in the panel's help tooltip @@ -46,7 +43,6 @@ def __init__( dark_icon=None, allow_multiple=False, surfaces: PANEL_SURFACE = "grid", - reload_on_navigation=False, **kwargs ): super().__init__(name) @@ -59,7 +55,6 @@ def __init__( self.allow_multiple = allow_multiple self.unlisted = True self.on_startup = True - self.reload_on_navigation = reload_on_navigation self.surfaces = surfaces self.kwargs = kwargs # unused, placeholder for future extensibility @@ -74,7 +69,6 @@ def to_json(self): "allow_multiple": self.allow_multiple, "on_startup": self.on_startup, "unlisted": self.unlisted, - "reload_on_navigation": self.reload_on_navigation, "surfaces": self.surfaces, } @@ -114,7 +108,6 @@ def on_startup(self, ctx): "dark_icon": self.config.dark_icon, "light_icon": self.config.light_icon, "surfaces": self.config.surfaces, - "reload_on_navigation": self.config.reload_on_navigation, } methods = ["on_load", "on_unload", "on_change"] ctx_change_events = [ From e54c7a708b76eff36d3403f47e5953aa9ecbe55d Mon Sep 17 00:00:00 2001 From: Sashank Aryal <66688606+sashankaryal@users.noreply.github.com> Date: Sat, 2 Nov 2024 06:33:41 +0545 Subject: [PATCH 60/67] fix tooltip race condition (#5030) * fix tooltip race condition * linting --------- Co-authored-by: Benjamin Kane --- .../core/src/components/Modal/Modal.tsx | 34 +++++++----- .../src/components/Modal/ModalNavigation.tsx | 10 ++-- .../components/Modal/ModalSamplePlugin.tsx | 52 +------------------ .../core/src/components/Modal/hooks.ts | 39 +++++++++++++- .../src/components/Modal/modal-context.ts | 1 - 5 files changed, 64 insertions(+), 72 deletions(-) diff --git a/app/packages/core/src/components/Modal/Modal.tsx b/app/packages/core/src/components/Modal/Modal.tsx index 5b4b0d9ee0..6443fa8a2b 100644 --- a/app/packages/core/src/components/Modal/Modal.tsx +++ b/app/packages/core/src/components/Modal/Modal.tsx @@ -1,17 +1,17 @@ import { HelpPanel, JSONPanel } from "@fiftyone/components"; import { OPERATOR_PROMPT_AREAS, OperatorPromptArea } from "@fiftyone/operators"; import * as fos from "@fiftyone/state"; -import React, { useCallback, useEffect, useMemo, useRef } from "react"; +import React, { useCallback, useMemo, useRef } from "react"; import ReactDOM from "react-dom"; import { useRecoilCallback, useRecoilValue } from "recoil"; import styled from "styled-components"; import { ModalActionsRow } from "../Actions"; import Sidebar from "../Sidebar"; -import { useLookerHelpers } from "./hooks"; -import { modalContext } from "./modal-context"; import ModalNavigation from "./ModalNavigation"; import { ModalSpace } from "./ModalSpace"; import { TooltipInfo } from "./TooltipInfo"; +import { useLookerHelpers, useTooltipEventHandler } from "./hooks"; +import { modalContext } from "./modal-context"; import { useModalSidebarRenderEntry } from "./use-sidebar-render-entry"; const ModalWrapper = styled.div` @@ -156,9 +156,9 @@ const Modal = () => { if (activeLookerRef.current) { // we handle close logic in modal + other places return; - } else { - await modalCloseHandler(); } + + await modalCloseHandler(); } }, [] @@ -168,7 +168,7 @@ const Modal = () => { const isFullScreen = useRecoilValue(fos.fullscreen); - const { onNavigate } = useLookerHelpers(); + const { closePanels } = useLookerHelpers(); const screenParams = useMemo(() => { return isFullScreen @@ -178,14 +178,21 @@ const Modal = () => { const activeLookerRef = useRef(); - // this is so that other components can add event listeners to the active looker - const onLookerSetSubscribers = useRef<((looker: fos.Lookers) => void)[]>([]); + const addTooltipEventHandler = useTooltipEventHandler(); + const removeTooltipEventHanlderRef = useRef | null>(null); - const onLookerSet = useCallback((looker: fos.Lookers) => { - onLookerSetSubscribers.current.forEach((sub) => sub(looker)); + const onLookerSet = useCallback( + (looker: fos.Lookers) => { + looker.addEventListener("close", modalCloseHandler); - looker.addEventListener("close", modalCloseHandler); - }, []); + // remove previous event listener + removeTooltipEventHanlderRef.current?.(); + removeTooltipEventHanlderRef.current = addTooltipEventHandler(looker); + }, + [modalCloseHandler, addTooltipEventHandler] + ); const setActiveLookerRef = useCallback( (looker: fos.Lookers) => { @@ -200,7 +207,6 @@ const Modal = () => { value={{ activeLookerRef, setActiveLookerRef, - onLookerSetSubscribers, }} > { - + diff --git a/app/packages/core/src/components/Modal/ModalNavigation.tsx b/app/packages/core/src/components/Modal/ModalNavigation.tsx index 4b5dc4459b..c7c8a4ad8f 100644 --- a/app/packages/core/src/components/Modal/ModalNavigation.tsx +++ b/app/packages/core/src/components/Modal/ModalNavigation.tsx @@ -44,7 +44,7 @@ const Arrow = styled.span<{ } `; -const ModalNavigation = ({ onNavigate }: { onNavigate: () => void }) => { +const ModalNavigation = ({ closePanels }: { closePanels: () => void }) => { const showModalNavigationControls = useRecoilValue( fos.showModalNavigationControls ); @@ -75,10 +75,10 @@ const ModalNavigation = ({ onNavigate }: { onNavigate: () => void }) => { createDebouncedNavigator({ isNavigationIllegalWhen: () => modalRef.current?.hasNext === false, navigateFn: (offset) => navigation?.next(offset).then(setModal), - onNavigationStart: onNavigate, + onNavigationStart: closePanels, debounceTime: 150, }), - [navigation, onNavigate, setModal] + [navigation, closePanels, setModal] ); const previousNavigator = useMemo( @@ -86,10 +86,10 @@ const ModalNavigation = ({ onNavigate }: { onNavigate: () => void }) => { createDebouncedNavigator({ isNavigationIllegalWhen: () => modalRef.current?.hasPrevious === false, navigateFn: (offset) => navigation?.previous(offset).then(setModal), - onNavigationStart: onNavigate, + onNavigationStart: closePanels, debounceTime: 150, }), - [navigation, onNavigate, setModal] + [navigation, closePanels, setModal] ); useEffect(() => { diff --git a/app/packages/core/src/components/Modal/ModalSamplePlugin.tsx b/app/packages/core/src/components/Modal/ModalSamplePlugin.tsx index 44e06ec6ea..60f95b3b77 100644 --- a/app/packages/core/src/components/Modal/ModalSamplePlugin.tsx +++ b/app/packages/core/src/components/Modal/ModalSamplePlugin.tsx @@ -1,10 +1,9 @@ import { ErrorBoundary } from "@fiftyone/components"; import * as fos from "@fiftyone/state"; -import React, { Suspense, useCallback, useEffect, useMemo } from "react"; -import { useRecoilCallback, useRecoilValue, useSetRecoilState } from "recoil"; +import React, { Suspense, useEffect } from "react"; +import { useRecoilValue, useSetRecoilState } from "recoil"; import styled from "styled-components"; import Group from "./Group"; -import { useModalContext } from "./hooks"; import { Sample2D } from "./Sample2D"; import { Sample3d } from "./Sample3d"; @@ -23,56 +22,9 @@ export const ModalSample = React.memo(() => { const isGroup = useRecoilValue(fos.isGroup); const is3D = useRecoilValue(fos.is3DDataset); - const tooltip = fos.useTooltip(); const setIsTooltipLocked = useSetRecoilState(fos.isTooltipLocked); const setTooltipDetail = useSetRecoilState(fos.tooltipDetail); - const tooltipEventHandler = useRecoilCallback( - ({ snapshot, set }) => - (e) => { - const isTooltipLocked = snapshot - .getLoadable(fos.isTooltipLocked) - .getValue(); - - if (e.detail) { - set(fos.tooltipDetail, e.detail); - if (!isTooltipLocked && e.detail?.coordinates) { - tooltip.setCoords(e.detail.coordinates); - } - } else if (!isTooltipLocked) { - set(fos.tooltipDetail, null); - } - }, - [tooltip] - ); - - const { activeLookerRef, onLookerSetSubscribers } = useModalContext(); - - const addTooltipEventListener = useMemo(() => { - return (looker: fos.Lookers) => { - looker.addEventListener("tooltip", tooltipEventHandler); - }; - }, []); - - useEffect(() => { - onLookerSetSubscribers.current.push(addTooltipEventListener); - - return () => { - activeLookerRef?.current?.removeEventListener( - "tooltip", - tooltipEventHandler - ); - onLookerSetSubscribers.current = onLookerSetSubscribers.current.filter( - (fn) => fn !== addTooltipEventListener - ); - }; - }, [ - activeLookerRef, - addTooltipEventListener, - onLookerSetSubscribers, - tooltipEventHandler, - ]); - useEffect(() => { // reset tooltip state when modal is closed setIsTooltipLocked(false); diff --git a/app/packages/core/src/components/Modal/hooks.ts b/app/packages/core/src/components/Modal/hooks.ts index ac61249265..80c9201362 100644 --- a/app/packages/core/src/components/Modal/hooks.ts +++ b/app/packages/core/src/components/Modal/hooks.ts @@ -16,7 +16,7 @@ export const useLookerHelpers = () => { jsonPanelRef.current = jsonPanel; helpPanelRef.current = helpPanel; - const onNavigate = useCallback(() => { + const closePanels = useCallback(() => { jsonPanelRef.current?.close(); helpPanelRef.current?.close(); }, []); @@ -24,7 +24,7 @@ export const useLookerHelpers = () => { return { jsonPanel, helpPanel, - onNavigate, + closePanels, }; }; @@ -78,3 +78,38 @@ export const useModalContext = () => { return ctx; }; + +export const useTooltipEventHandler = () => { + const tooltip = fos.useTooltip(); + + const tooltipEventHandler = useRecoilCallback( + ({ snapshot, set }) => + (e) => { + const isTooltipLocked = snapshot + .getLoadable(fos.isTooltipLocked) + .getValue(); + + if (e.detail) { + set(fos.tooltipDetail, e.detail); + if (!isTooltipLocked && e.detail?.coordinates) { + tooltip.setCoords(e.detail.coordinates); + } + } else if (!isTooltipLocked) { + set(fos.tooltipDetail, null); + } + }, + [tooltip] + ); + + return useCallback( + (looker: fos.Lookers) => { + looker.removeEventListener("tooltip", tooltipEventHandler); + looker.addEventListener("tooltip", tooltipEventHandler); + + return () => { + looker.removeEventListener("tooltip", tooltipEventHandler); + }; + }, + [tooltipEventHandler] + ); +}; diff --git a/app/packages/core/src/components/Modal/modal-context.ts b/app/packages/core/src/components/Modal/modal-context.ts index aa1ae907e5..eef0524d32 100644 --- a/app/packages/core/src/components/Modal/modal-context.ts +++ b/app/packages/core/src/components/Modal/modal-context.ts @@ -4,7 +4,6 @@ import React, { createContext } from "react"; interface ModalContextT { activeLookerRef: React.MutableRefObject; setActiveLookerRef: (looker: Lookers) => void; - onLookerSetSubscribers: React.MutableRefObject<((looker: Lookers) => void)[]>; } export const modalContext = createContext(undefined); From 991d620261e7c9e8b1fa41413b473dc71cf2ea82 Mon Sep 17 00:00:00 2001 From: prerna <163362853+prernadh@users.noreply.github.com> Date: Mon, 4 Nov 2024 08:29:01 -0800 Subject: [PATCH 61/67] Depth estimation input shape update (#5035) * Depth estimation input shape update * Removing print statement * Removing Semantic Seg change --- fiftyone/utils/transformers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fiftyone/utils/transformers.py b/fiftyone/utils/transformers.py index 95bc3b23e0..9ba33468ea 100644 --- a/fiftyone/utils/transformers.py +++ b/fiftyone/utils/transformers.py @@ -950,12 +950,12 @@ def _predict(self, inputs, target_sizes): return to_heatmap(prediction) def predict(self, arg): - target_sizes = [arg.shape[:-1][::-1]] + target_sizes = [arg.shape[:2]] inputs = self.image_processor(arg, return_tensors="pt") return self._predict(inputs, target_sizes) def predict_all(self, args): - target_sizes = [i.shape[:-1][::-1] for i in args] + target_sizes = [i.shape[:2] for i in args] inputs = self.image_processor(args, return_tensors="pt") return self._predict(inputs, target_sizes) From 83f9ede12740f40024e20baf6c8b8c5e39494783 Mon Sep 17 00:00:00 2001 From: Benjamin Kane Date: Mon, 4 Nov 2024 19:50:59 -0500 Subject: [PATCH 62/67] App performance enhancements (#5022) * fix spotlight cleanup * remove modal navigation from router transactions * cache gathered paths * e2e updates * update selection test * lint, add comments * event listener cleanup * freeze tensorflow dep * add < > shortcuts in help modal * add test for shortcuts * details * clear frame stream lru cache * fix shortcutToHelpItems * add destroy items * lint * weak map resets and item deletions * add max stream sizing configurations * linting * bugs * clear stream cache on mouseleave * better spotlight heuristics * cleanup * soft edit * constants tweaks * modal nav enhancements * rm bytes sizing * better aspect ratio thresholds * tweaks * config docs * wait to stream for videos * lru looker cache * cleanup * isReusable fixes * cleanup * move hook, fix e2e * increase lru cache * update screenshots * rm python settings, add to options action --------- Co-authored-by: Br2850 Co-authored-by: Sashank Aryal --- app/packages/app/src/routing/RouterContext.ts | 15 +- app/packages/app/src/routing/matchPath.ts | 2 +- .../app/src/useEvents/useSetSpaces.ts | 4 +- .../app/src/useWriters/onSetSessionSpaces.ts | 4 +- .../src/components/Actions/ActionsRow.tsx | 8 +- .../core/src/components/Actions/Options.tsx | 86 +++- .../core/src/components/Grid/Grid.tsx | 53 +-- .../core/src/components/Grid/recoil.ts | 33 +- .../core/src/components/Grid/useFontSize.ts | 2 +- .../core/src/components/Grid/useRefreshers.ts | 22 +- .../core/src/components/Grid/useSelect.ts | 6 +- .../core/src/components/Grid/useThreshold.ts | 23 +- .../src/components/Modal/ModalNavigation.tsx | 25 +- .../core/src/components/Modal/utils.test.ts | 12 + .../core/src/components/Modal/utils.ts | 16 +- app/packages/looker/src/constants.ts | 1 - app/packages/looker/src/elements/base.ts | 56 +-- .../looker/src/elements/common/actions.ts | 24 ++ .../looker/src/elements/common/index.ts | 2 +- .../looker/src/elements/common/options.ts | 10 +- app/packages/looker/src/elements/image.ts | 7 +- app/packages/looker/src/elements/index.ts | 154 +++----- app/packages/looker/src/elements/util.ts | 55 ++- app/packages/looker/src/elements/video.ts | 75 ++-- app/packages/looker/src/lookers/abstract.ts | 77 ++-- .../looker/src/lookers/frame-reader.ts | 166 ++++++++ app/packages/looker/src/lookers/image.ts | 32 +- .../looker/src/lookers/imavid/controller.ts | 8 +- .../lookers/imavid/ima-vid-frame-samples.ts | 6 +- .../looker/src/lookers/imavid/index.ts | 13 +- app/packages/looker/src/lookers/shared.ts | 9 +- app/packages/looker/src/lookers/three-d.ts | 10 +- app/packages/looker/src/lookers/video.ts | 374 +++++------------- app/packages/looker/src/overlays/base.ts | 7 +- .../looker/src/overlays/classifications.ts | 16 +- app/packages/looker/src/overlays/heatmap.ts | 6 +- .../looker/src/overlays/segmentation.ts | 6 +- app/packages/looker/src/overlays/util.ts | 40 +- app/packages/looker/src/state.ts | 3 + app/packages/looker/src/util.ts | 12 +- .../looker/src/worker/deserializer.ts | 7 +- app/packages/looker/src/worker/index.ts | 8 +- app/packages/spotlight/src/constants.ts | 4 +- app/packages/spotlight/src/index.ts | 4 +- app/packages/spotlight/src/iter.ts | 5 +- app/packages/spotlight/src/row.ts | 9 +- app/packages/spotlight/src/section.ts | 4 +- app/packages/spotlight/src/types.ts | 1 + app/packages/state/src/hooks/index.ts | 2 +- app/packages/state/src/hooks/useClearModal.ts | 1 + .../state/src/hooks/useCreateLooker.ts | 56 ++- .../state/src/hooks/useExpandSample.ts | 2 +- .../state/src/hooks/useSetModalState.ts | 2 +- app/packages/state/src/recoil/modal.ts | 14 +- docs/source/user_guide/config.rst | 2 + .../grid-tagged-chromium-darwin.png | Bin 44076 -> 42131 bytes .../grid-tagged-chromium-linux.png | Bin 36022 -> 36402 bytes .../grid-untagged-chromium-darwin.png | Bin 31224 -> 36755 bytes .../grid-untagged-chromium-linux.png | Bin 27114 -> 31691 bytes ...hide-ship-invisible-cat-chromium-linux.png | Bin 14599 -> 14609 bytes 60 files changed, 933 insertions(+), 668 deletions(-) create mode 100644 app/packages/core/src/components/Modal/utils.test.ts create mode 100644 app/packages/looker/src/lookers/frame-reader.ts diff --git a/app/packages/app/src/routing/RouterContext.ts b/app/packages/app/src/routing/RouterContext.ts index 92a2114e16..e9c625a978 100644 --- a/app/packages/app/src/routing/RouterContext.ts +++ b/app/packages/app/src/routing/RouterContext.ts @@ -11,6 +11,7 @@ import type { Queries } from "../makeRoutes"; import type RouteDefinition from "./RouteDefinition"; import type { LocationState, MatchPathResult } from "./matchPath"; +import { viewsAreEqual } from "@fiftyone/state"; import { NotFoundError, Resource, isNotebook } from "@fiftyone/utilities"; import { createBrowserHistory, createMemoryHistory } from "history"; import React from "react"; @@ -183,13 +184,25 @@ export const createRouter = ( }; }; -const SKIP_EVENTS = new Set(["modal", "slice"]); +const SKIP_EVENTS = new Set(["modal", "slice", "spaces"]); const makeGetEntryResource = () => { let currentLocation: FiftyOneLocation; let currentResource: Resource>; const isReusable = (location: FiftyOneLocation) => { + if (location.pathname !== currentLocation?.pathname) { + return false; + } + + if (location.search !== currentLocation?.search) { + return false; + } + + if (!viewsAreEqual(location.state.view, currentLocation?.state.view)) { + return false; + } + if (currentLocation) { return ( SKIP_EVENTS.has(location.state.event || "") || diff --git a/app/packages/app/src/routing/matchPath.ts b/app/packages/app/src/routing/matchPath.ts index 1223647081..5207cec417 100644 --- a/app/packages/app/src/routing/matchPath.ts +++ b/app/packages/app/src/routing/matchPath.ts @@ -10,7 +10,7 @@ const compilePath = (path: string) => }); export type LocationState = { - event?: "modal" | "slice"; + event?: "modal" | "slice" | "spaces"; fieldVisibility?: State.FieldVisibilityStage; groupSlice?: string; modalSelector?: ModalSelector; diff --git a/app/packages/app/src/useEvents/useSetSpaces.ts b/app/packages/app/src/useEvents/useSetSpaces.ts index f668bcdb4d..b5f19b7e93 100644 --- a/app/packages/app/src/useEvents/useSetSpaces.ts +++ b/app/packages/app/src/useEvents/useSetSpaces.ts @@ -10,7 +10,7 @@ const useSetSpaces: EventHandlerHook = ({ router }) => { (payload) => { setter("sessionSpaces", payload.spaces); const state = router.history.location.state as LocationState; - router.history.replace( + router.replace( resolveURL({ currentPathname: router.history.location.pathname, currentSearch: router.history.location.search, @@ -18,7 +18,7 @@ const useSetSpaces: EventHandlerHook = ({ router }) => { workspace: payload.spaces._name ?? null, }, }), - { ...state, workspace: payload.spaces } + { ...state, event: "spaces", workspace: payload.spaces } ); }, [router, setter] diff --git a/app/packages/app/src/useWriters/onSetSessionSpaces.ts b/app/packages/app/src/useWriters/onSetSessionSpaces.ts index 2c3cebec72..700f6260b7 100644 --- a/app/packages/app/src/useWriters/onSetSessionSpaces.ts +++ b/app/packages/app/src/useWriters/onSetSessionSpaces.ts @@ -8,7 +8,7 @@ const onSetSessionSpaces: RegisteredWriter<"sessionSpaces"> = ({ environment, router, subscription }) => (spaces) => { const state = router.history.location.state as LocationState; - router.history.replace( + router.replace( resolveURL({ currentPathname: router.history.location.pathname, currentSearch: router.history.location.search, @@ -16,7 +16,7 @@ const onSetSessionSpaces: RegisteredWriter<"sessionSpaces"> = workspace: spaces._name || null, }, }), - { ...state, workspace: spaces._name || null } + { ...state, event: "spaces", workspace: spaces._name || null } ); commitMutation(environment, { diff --git a/app/packages/core/src/components/Actions/ActionsRow.tsx b/app/packages/core/src/components/Actions/ActionsRow.tsx index 7816d58381..850259bd2a 100644 --- a/app/packages/core/src/components/Actions/ActionsRow.tsx +++ b/app/packages/core/src/components/Actions/ActionsRow.tsx @@ -8,8 +8,8 @@ import { } from "@fiftyone/components"; import { FrameLooker, ImageLooker, VideoLooker } from "@fiftyone/looker"; import { - OperatorPlacements, OperatorPlacementWithErrorBoundary, + OperatorPlacements, types, useOperatorBrowser, useOperatorPlacements, @@ -277,6 +277,12 @@ const Selected = ({ refresh?.(); }, [samples.size, refresh]); + useEffect(() => { + return () => { + setLoading(false); + }; + }, []); + if (samples.size < 1 && !modal) { return null; } diff --git a/app/packages/core/src/components/Actions/Options.tsx b/app/packages/core/src/components/Actions/Options.tsx index 06c258962e..5deb073b77 100644 --- a/app/packages/core/src/components/Actions/Options.tsx +++ b/app/packages/core/src/components/Actions/Options.tsx @@ -7,10 +7,12 @@ import { import * as fos from "@fiftyone/state"; import { configuredSidebarModeDefault, + frameCacheSize, groupStatistics, sidebarMode, } from "@fiftyone/state"; -import React, { RefObject, useMemo } from "react"; +import type { RefObject } from "react"; +import React, { useMemo } from "react"; import { useRecoilState, useRecoilValue, @@ -20,6 +22,7 @@ import { import { LIGHTNING_MODE, SIDEBAR_MODE } from "../../utils/links"; import Checkbox from "../Common/Checkbox"; import RadioGroup from "../Common/RadioGroup"; +import { lookerGridCaching } from "../Grid/recoil"; import { Button } from "../utils"; import { ActionOption } from "./Common"; import Popout from "./Popout"; @@ -354,6 +357,86 @@ const ShowModalNav = () => { ); }; +const Caching = ({ modal }: { modal?: boolean }) => { + const [caching, setCaching] = useRecoilState(lookerGridCaching); + const [frameStream, setFrameStream] = useRecoilState(frameCacheSize); + const resetFrameCacheSize = useResetRecoilState(frameCacheSize); + const theme = useTheme(); + + return ( + <> + {!modal && ( + <> + + + + )} + { + <> + + { + if (text === "") { + resetFrameCacheSize(); + return "1024"; + } + const value = Number.parseInt(text); + + if (Number.isNaN(value)) { + resetFrameCacheSize(); + return "1024"; + } + + setFrameStream(value); + + return String(value); + }} + inputStyle={{ + fontSize: "1rem", + textAlign: "right", + float: "right", + width: "100%", + }} + value={String(frameStream)} + containerStyle={{ + display: "flex", + justifyContent: "right", + marginBottom: "0.5rem", + }} + /> + + } + + ); +}; + type OptionsProps = { modal?: boolean; anchorRef: RefObject; @@ -375,6 +458,7 @@ const Options = ({ modal, anchorRef }: OptionsProps) => { {!view?.length && } {!modal && } + ); }; diff --git a/app/packages/core/src/components/Grid/Grid.tsx b/app/packages/core/src/components/Grid/Grid.tsx index db454e14d0..4b71823c57 100644 --- a/app/packages/core/src/components/Grid/Grid.tsx +++ b/app/packages/core/src/components/Grid/Grid.tsx @@ -1,9 +1,7 @@ import styles from "./Grid.module.css"; import { freeVideos } from "@fiftyone/looker"; -import type { ID } from "@fiftyone/spotlight"; import Spotlight from "@fiftyone/spotlight"; -import type { Lookers } from "@fiftyone/state"; import * as fos from "@fiftyone/state"; import React, { useEffect, @@ -27,11 +25,9 @@ import useThreshold from "./useThreshold"; function Grid() { const id = useMemo(() => uuid(), []); - const lookerStore = useMemo(() => new WeakMap(), []); const spacing = useRecoilValue(gridSpacing); - const selectSample = useRef>(); - const { pageReset, reset } = useRefreshers(); + const { caching, lookerStore, pageReset, reset } = useRefreshers(); const [resizing, setResizing] = useState(false); const threshold = useThreshold(); @@ -50,19 +46,28 @@ function Grid() { const getFontSize = useFontSize(id); const spotlight = useMemo(() => { + /** SPOTLIGHT REFRESHER */ reset; + /** SPOTLIGHT REFRESHER */ + if (resizing) { return undefined; } return new Spotlight({ ...get(), + destroy: (id) => { + const looker = lookerStore.get(id.description); + looker?.destroy(); + lookerStore.delete(id.description); + }, onItemClick: setSample, + retainItems: caching, rowAspectRatioThreshold: threshold, get: (next) => page(next), render: (id, element, dimensions, soft, hide) => { - if (lookerStore.has(id)) { - const looker = lookerStore.get(id); + if (lookerStore.has(id.description)) { + const looker = lookerStore.get(id.description); hide ? looker?.disable() : looker?.attach(element, dimensions); return; @@ -74,35 +79,33 @@ function Grid() { throw new Error("bad data"); } - const init = (l) => { - l.addEventListener("selectthumbnail", ({ detail }: CustomEvent) => { - selectSample.current?.(detail); - }); - lookerStore.set(id, l); - l.attach(element, dimensions); - }; - - if (!soft) { - init( - createLooker.current?.( - { ...result, symbol: id }, - { - fontSize: getFontSize(), - } - ) - ); + if (soft) { + // we are scrolling fast, skip creation + return; } + + const looker = createLooker.current?.( + { ...result, symbol: id }, + { + fontSize: getFontSize(), + } + ); + looker.addEventListener("selectthumbnail", ({ detail }) => + selectSample.current?.(detail) + ); + lookerStore.set(id.description, looker); + looker.attach(element, dimensions); }, scrollbar: true, spacing, }); }, [ + caching, createLooker, get, getFontSize, lookerStore, page, - records, reset, resizing, setSample, diff --git a/app/packages/core/src/components/Grid/recoil.ts b/app/packages/core/src/components/Grid/recoil.ts index df280227e2..3f3efdaec9 100644 --- a/app/packages/core/src/components/Grid/recoil.ts +++ b/app/packages/core/src/components/Grid/recoil.ts @@ -1,7 +1,38 @@ -import { atom, selector } from "recoil"; +import { DefaultValue, atom, atomFamily, selector } from "recoil"; import * as fos from "@fiftyone/state"; +const lookerGridCachingStore = atomFamily({ + key: "lookerGridCachingStore", + default: true, + effects: (id) => [ + fos.getBrowserStorageEffectForKey(`looker-grid-caching-${id}`, { + valueClass: "boolean", + }), + ], +}); + +export const lookerGridCaching = selector({ + key: "lookerGridCaching", + get: ({ get }) => { + const id = get(fos.datasetId); + if (!id) { + throw new Error("no dataset"); + } + return get(lookerGridCachingStore(id)); + }, + set: ({ get, set }, value) => { + const id = get(fos.datasetId); + if (!id) { + throw new Error("no dataset"); + } + set( + lookerGridCachingStore(id), + value instanceof DefaultValue ? false : value + ); + }, +}); + export const defaultGridZoom = selector({ key: "defaultGridZoom", get: ({ get }) => get(fos.config)?.gridZoom, diff --git a/app/packages/core/src/components/Grid/useFontSize.ts b/app/packages/core/src/components/Grid/useFontSize.ts index 25337c9dc0..4301efdcc3 100644 --- a/app/packages/core/src/components/Grid/useFontSize.ts +++ b/app/packages/core/src/components/Grid/useFontSize.ts @@ -1,7 +1,7 @@ import { useCallback } from "react"; import useThreshold from "./useThreshold"; -const MAX = 32; +const MAX = 14; const MIN = 10; const SCALE_FACTOR = 0.09; diff --git a/app/packages/core/src/components/Grid/useRefreshers.ts b/app/packages/core/src/components/Grid/useRefreshers.ts index 10922ab09f..96d2c0240f 100644 --- a/app/packages/core/src/components/Grid/useRefreshers.ts +++ b/app/packages/core/src/components/Grid/useRefreshers.ts @@ -1,9 +1,10 @@ import { subscribe } from "@fiftyone/relay"; import * as fos from "@fiftyone/state"; +import { LRUCache } from "lru-cache"; import { useEffect, useMemo } from "react"; import uuid from "react-uuid"; import { useRecoilValue } from "recoil"; -import { gridAt, gridOffset, gridPage } from "./recoil"; +import { gridAt, gridOffset, gridPage, lookerGridCaching } from "./recoil"; export default function useRefreshers() { const cropToContent = useRecoilValue(fos.cropToContent(false)); @@ -25,15 +26,17 @@ export default function useRefreshers() { fos.shouldRenderImaVidLooker(false) ); const view = fos.filterView(useRecoilValue(fos.view)); + const caching = useRecoilValue(lookerGridCaching); // only reload, attempt to return to the last grid location const layoutReset = useMemo(() => { + caching; cropToContent; fieldVisibilityStage; mediaField; refresher; return uuid(); - }, [cropToContent, fieldVisibilityStage, mediaField, refresher]); + }, [caching, cropToContent, fieldVisibilityStage, mediaField, refresher]); // the values reset the page, i.e. return to the top const pageReset = useMemo(() => { @@ -74,7 +77,22 @@ export default function useRefreshers() { [] ); + const lookerStore = useMemo(() => { + /** LOOKER STORE REFRESHER */ + reset; + /** LOOKER STORE REFRESHER */ + + return new LRUCache({ + max: 512, + dispose: (looker) => { + looker.destroy(); + }, + }); + }, [reset]); + return { + caching, + lookerStore, pageReset, reset, }; diff --git a/app/packages/core/src/components/Grid/useSelect.ts b/app/packages/core/src/components/Grid/useSelect.ts index 45d0e6c120..9664373992 100644 --- a/app/packages/core/src/components/Grid/useSelect.ts +++ b/app/packages/core/src/components/Grid/useSelect.ts @@ -1,13 +1,13 @@ import type Spotlight from "@fiftyone/spotlight"; -import type { ID } from "@fiftyone/spotlight"; import * as fos from "@fiftyone/state"; +import type { LRUCache } from "lru-cache"; import { useEffect } from "react"; import { useRecoilValue } from "recoil"; export default function useSelect( getFontSize: () => number, options: ReturnType, - store: WeakMap, + store: LRUCache, spotlight?: Spotlight ) { const { init, deferred } = fos.useDeferrer(); @@ -17,7 +17,7 @@ export default function useSelect( deferred(() => { const fontSize = getFontSize(); spotlight?.updateItems((id) => { - store.get(id)?.updateOptions({ + store.get(id.description)?.updateOptions({ ...options, fontSize, selected: selected.has(id.description), diff --git a/app/packages/core/src/components/Grid/useThreshold.ts b/app/packages/core/src/components/Grid/useThreshold.ts index 494a9f1a1c..a226034fd4 100644 --- a/app/packages/core/src/components/Grid/useThreshold.ts +++ b/app/packages/core/src/components/Grid/useThreshold.ts @@ -2,18 +2,27 @@ import { useCallback } from "react"; import { useRecoilValue } from "recoil"; import { gridZoom } from "./recoil"; +const WIDEST = 1200; +const WIDE = 1000; +const NORMAL = 800; + +/** + * Determines a maximium aspect ratio threshold for grid rows based on the + * container width. The smaller the container width, the smaller the maximum + * aspect ratio to prevent a large number of items from rendering on screen + */ export default () => { const zoom = useRecoilValue(gridZoom); return useCallback( (width: number) => { - let min = 1; + let min = 6; - if (width >= 1200) { - min = -5; - } else if (width >= 1000) { - min = -3; - } else if (width >= 800) { - min = -1; + if (width >= WIDEST) { + min = -4; + } else if (width >= WIDE) { + min = -2; + } else if (width >= NORMAL) { + min = 2; } return 11 - Math.max(min, zoom); diff --git a/app/packages/core/src/components/Modal/ModalNavigation.tsx b/app/packages/core/src/components/Modal/ModalNavigation.tsx index c7c8a4ad8f..a02a86d395 100644 --- a/app/packages/core/src/components/Modal/ModalNavigation.tsx +++ b/app/packages/core/src/components/Modal/ModalNavigation.tsx @@ -27,7 +27,7 @@ const Arrow = styled.span<{ left: ${(props) => (props.$isRight ? "initial" : "0.75rem")}; z-index: 99999; padding: 0.75rem; - bottom: 33vh; + top: 50%; width: 3rem; height: 3rem; background-color: var(--fo-palette-background-button); @@ -42,6 +42,10 @@ const Arrow = styled.span<{ transition: box-shadow 0.15s ease-in-out; transition: opacity 0.15s ease-in-out; } + + &:active { + top: calc(50% + 2px); + } `; const ModalNavigation = ({ closePanels }: { closePanels: () => void }) => { @@ -62,7 +66,6 @@ const ModalNavigation = ({ closePanels }: { closePanels: () => void }) => { const setModal = fos.useSetExpandedSample(); const modal = useRecoilValue(fos.modalSelector); - const navigation = useRecoilValue(fos.modalNavigation); const modalRef = useRef(modal); @@ -74,22 +77,32 @@ const ModalNavigation = ({ closePanels }: { closePanels: () => void }) => { () => createDebouncedNavigator({ isNavigationIllegalWhen: () => modalRef.current?.hasNext === false, - navigateFn: (offset) => navigation?.next(offset).then(setModal), + navigateFn: async (offset) => { + const navigation = fos.modalNavigation.get(); + if (navigation) { + return await navigation.next(offset).then(setModal); + } + }, onNavigationStart: closePanels, debounceTime: 150, }), - [navigation, closePanels, setModal] + [closePanels, setModal] ); const previousNavigator = useMemo( () => createDebouncedNavigator({ isNavigationIllegalWhen: () => modalRef.current?.hasPrevious === false, - navigateFn: (offset) => navigation?.previous(offset).then(setModal), + navigateFn: async (offset) => { + const navigation = fos.modalNavigation.get(); + if (navigation) { + return await navigation.previous(offset).then(setModal); + } + }, onNavigationStart: closePanels, debounceTime: 150, }), - [navigation, closePanels, setModal] + [closePanels, setModal] ); useEffect(() => { diff --git a/app/packages/core/src/components/Modal/utils.test.ts b/app/packages/core/src/components/Modal/utils.test.ts new file mode 100644 index 0000000000..d764324867 --- /dev/null +++ b/app/packages/core/src/components/Modal/utils.test.ts @@ -0,0 +1,12 @@ +import { describe, expect, it } from "vitest"; +import { shortcutToHelpItems } from "./utils"; + +describe("shortcut processing test", () => { + it("parses unique shortcuts", () => { + const results = shortcutToHelpItems({ + one: { shortcut: "test" }, + two: { shortcut: "test" }, + }); + expect(results).toStrictEqual([{ shortcut: "test" }]); + }); +}); diff --git a/app/packages/core/src/components/Modal/utils.ts b/app/packages/core/src/components/Modal/utils.ts index 2461570f9a..e20a1ec6a6 100644 --- a/app/packages/core/src/components/Modal/utils.ts +++ b/app/packages/core/src/components/Modal/utils.ts @@ -1,7 +1,13 @@ -export function shortcutToHelpItems(SHORTCUTS) { - const result = {}; - for (const k of SHORTCUTS) { - result[SHORTCUTS[k].shortcut] = SHORTCUTS[k]; +interface ShortcutItem { + shortcut: string; +} + +type Shortcuts = { [key: string]: ShortcutItem }; + +export function shortcutToHelpItems(SHORTCUTS: Shortcuts) { + const uniqueItems = {}; + for (const item of Object.values(SHORTCUTS)) { + uniqueItems[item.shortcut] = item; } - return Object.values(result); + return Object.values(uniqueItems); } diff --git a/app/packages/looker/src/constants.ts b/app/packages/looker/src/constants.ts index 8292bb05e3..13756f1dd9 100644 --- a/app/packages/looker/src/constants.ts +++ b/app/packages/looker/src/constants.ts @@ -22,7 +22,6 @@ export const STROKE_WIDTH = 3; export const FONT_SIZE = 16; export const MIN_PIXELS = 16; export const SCALE_FACTOR = 1.09; -export const MAX_FRAME_CACHE_SIZE_BYTES = 1e9; export const CHUNK_SIZE = 20; export const DATE_TIME = "DateTime"; diff --git a/app/packages/looker/src/elements/base.ts b/app/packages/looker/src/elements/base.ts index 5ce470ef87..3f7fb74c4c 100644 --- a/app/packages/looker/src/elements/base.ts +++ b/app/packages/looker/src/elements/base.ts @@ -2,7 +2,7 @@ * Copyright 2017-2024, Voxel51, Inc. */ -import { BaseState, DispatchEvent, Sample, StateUpdate } from "../state"; +import type { BaseState, DispatchEvent, Sample, StateUpdate } from "../state"; type ElementEvent = (args: { event: E; @@ -24,6 +24,14 @@ type LoadedEvents = { [K in keyof HTMLElementEventMap]?: HTMLElementEventMap[K]; }; +interface BootParams { + abortController: AbortController; + batchUpdate?: (cb: () => unknown) => void; + config: Readonly; + dispatchEvent: (eventType: string, details?: any) => void; + update: StateUpdate; +} + export abstract class BaseElement< State extends BaseState, Element extends HTMLElement = HTMLElement | null @@ -46,12 +54,17 @@ export abstract class BaseElement< protected readonly events: LoadedEvents = {}; - boot( - config: Readonly, - update: StateUpdate, - dispatchEvent: (eventType: string, details?: any) => void, - batchUpdate?: (cb: () => unknown) => void - ) { + applyChildren(children: BaseElement[]) { + this.children = children || []; + } + + boot({ + abortController, + batchUpdate, + config, + dispatchEvent, + update, + }: BootParams) { if (!this.isShown(config)) { return; } @@ -64,12 +77,11 @@ export abstract class BaseElement< for (const [eventType, handler] of Object.entries(this.getEvents(config))) { this.events[eventType] = (event) => handler({ event, update, dispatchEvent }); - this.element?.addEventListener(eventType, this.events[eventType]); + this.element?.addEventListener(eventType, this.events[eventType], { + signal: abortController.signal, + }); } } - applyChildren(children: BaseElement[]) { - this.children = children || []; - } isShown(config: Readonly): boolean { return true; @@ -77,17 +89,17 @@ export abstract class BaseElement< render(state: Readonly, sample: Readonly): Element | null { const self = this.renderSelf(state, sample); - this.children.forEach((child) => { + for (const child of this.children) { if (!child.isShown(state.config)) { - return; + continue; } const element = child.render(state, sample); if (!element || element.parentNode === this.element) { - return; + continue; } - self && self.appendChild(element); - }); + self?.appendChild(element); + } return self; } @@ -105,16 +117,4 @@ export abstract class BaseElement< protected getEvents(config: Readonly): Events { return {}; } - - protected removeEvents() { - for (const eventType in this.events) { - this.element.removeEventListener(eventType, this.events[eventType]); - } - } - - protected attachEvents() { - for (const eventType in this.events) { - this.element.addEventListener(eventType, this.events[eventType]); - } - } } diff --git a/app/packages/looker/src/elements/common/actions.ts b/app/packages/looker/src/elements/common/actions.ts index 5775dda7f7..a8cdbffb84 100644 --- a/app/packages/looker/src/elements/common/actions.ts +++ b/app/packages/looker/src/elements/common/actions.ts @@ -405,6 +405,17 @@ export const nextFrame: Control = { }, }; +export const nextFrameNoOpControl: Control = { + title: "Next frame", + eventKeys: [".", ">"], + shortcut: ">", + detail: "Seek to the next frame", + alwaysHandle: true, + action: () => { + // no-op here, supposed to be implemented elsewhere + }, +}; + export const previousFrame: Control = { title: "Previous frame", eventKeys: [",", "<"], @@ -436,6 +447,17 @@ export const previousFrame: Control = { }, }; +export const previousFrameNoOpControl: Control = { + title: "Previous frame", + eventKeys: [",", "<"], + shortcut: "<", + detail: "Seek to the previous frame", + alwaysHandle: true, + action: () => { + // no-op here, supposed to be implemented elsewhere + }, +}; + export const playPause: Control = { title: "Play / pause", shortcut: "Space", @@ -662,6 +684,8 @@ const VIDEO = { const IMAVID = { ...COMMON, escape: videoEscape, + previousFrame: previousFrameNoOpControl, + nextFrame: nextFrameNoOpControl, }; export const VIDEO_SHORTCUTS = readActions(VIDEO); diff --git a/app/packages/looker/src/elements/common/index.ts b/app/packages/looker/src/elements/common/index.ts index 3e7c00612b..f41c3b8ed7 100644 --- a/app/packages/looker/src/elements/common/index.ts +++ b/app/packages/looker/src/elements/common/index.ts @@ -2,10 +2,10 @@ * Copyright 2017-2024, Voxel51, Inc. */ +export { COMMON_SHORTCUTS, VIDEO_SHORTCUTS } from "./actions"; export { CanvasElement } from "./canvas"; export * from "./controls"; export { ErrorElement } from "./error"; -export { COMMON_SHORTCUTS, VIDEO_SHORTCUTS } from "./actions"; export { LookerElement } from "./looker"; export * from "./options"; export { TagsElement } from "./tags"; diff --git a/app/packages/looker/src/elements/common/options.ts b/app/packages/looker/src/elements/common/options.ts index 68043ee0d6..2d0c201c0d 100644 --- a/app/packages/looker/src/elements/common/options.ts +++ b/app/packages/looker/src/elements/common/options.ts @@ -2,7 +2,7 @@ * Copyright 2017-2024, Voxel51, Inc. */ -import { BaseState, VideoState } from "../../state"; +import { BaseState, ImaVidState, VideoState } from "../../state"; import { BaseElement, Events } from "../base"; import { lookerOptionsInput, lookerOptionsPanel } from "./options.module.css"; @@ -58,11 +58,13 @@ export class OptionsPanelElement< } } -export class LoopVideoOptionElement extends BaseElement { +export class LoopVideoOptionElement< + State extends ImaVidState | VideoState = VideoState +> extends BaseElement { checkbox?: HTMLInputElement; label?: HTMLLabelElement; - getEvents(): Events { + getEvents(): Events { return { click: ({ event, update, dispatchEvent }) => { event.stopPropagation(); @@ -80,7 +82,7 @@ export class LoopVideoOptionElement extends BaseElement { return makeWrapper([this.label]); } - renderSelf({ options: { loop } }: Readonly) { + renderSelf({ options: { loop } }: Readonly) { //@ts-ignore this.checkbox.checked = loop; return this.element; diff --git a/app/packages/looker/src/elements/image.ts b/app/packages/looker/src/elements/image.ts index 590dfd02ef..2df92eb5fd 100644 --- a/app/packages/looker/src/elements/image.ts +++ b/app/packages/looker/src/elements/image.ts @@ -2,11 +2,12 @@ * Copyright 2017-2024, Voxel51, Inc. */ -import { ImageState } from "../state"; -import { BaseElement, Events } from "./base"; +import type { ImageState } from "../state"; +import type { Events } from "./base"; +import { BaseElement } from "./base"; export class ImageElement extends BaseElement { - private src: string = ""; + private src = ""; private imageSource: HTMLImageElement; getEvents(): Events { diff --git a/app/packages/looker/src/elements/index.ts b/app/packages/looker/src/elements/index.ts index a332004b63..ec82844f59 100644 --- a/app/packages/looker/src/elements/index.ts +++ b/app/packages/looker/src/elements/index.ts @@ -1,7 +1,7 @@ /** * Copyright 2017-2024, Voxel51, Inc. */ -import { +import type { BaseState, FrameState, ImaVidState, @@ -10,26 +10,25 @@ import { ThreeDState, VideoState, } from "../state"; +import type { BaseElement } from "./base"; import * as common from "./common"; import * as frame from "./frame"; import * as image from "./image"; import * as imavid from "./imavid"; import * as pcd from "./three-d"; +import type { ElementsTemplate } from "./util"; import { createElementsTree, withEvents } from "./util"; import * as video from "./video"; -export type GetElements = ( - config: Readonly, - update: StateUpdate, - dispatchEvent: (eventType: string, details?: any) => void, - batchUpdate?: (cb: () => unknown) => void | undefined -) => common.LookerElement; +export type GetElements = (params: { + abortController: AbortController; + batchUpdate?: (cb: () => unknown) => void; + config: Readonly; + dispatchEvent: (eventType: string, details?: any) => void; + update: StateUpdate; +}) => common.LookerElement; -export const getFrameElements: GetElements = ( - config, - update, - dispatchEvent -) => { +export const getFrameElements: GetElements = (params) => { const elements = { node: common.LookerElement, children: [ @@ -72,19 +71,13 @@ export const getFrameElements: GetElements = ( ], }; - return createElementsTree>( - config, - elements, - update, - dispatchEvent - ); + return createElementsTree>({ + ...params, + root: elements, + }); }; -export const getImageElements: GetElements = ( - config, - update, - dispatchEvent -) => { +export const getImageElements: GetElements = (params) => { const elements = { node: common.LookerElement, children: [ @@ -127,19 +120,13 @@ export const getImageElements: GetElements = ( ], }; - return createElementsTree>( - config, - elements, - update, - dispatchEvent - ); + return createElementsTree>({ + ...params, + root: elements, + }); }; -export const getVideoElements: GetElements = ( - config, - update, - dispatchEvent -) => { +export const getVideoElements: GetElements = (params) => { const elements = { node: withEvents(common.LookerElement, video.withVideoLookerEvents()), children: [ @@ -193,22 +180,18 @@ export const getVideoElements: GetElements = ( ], }; - return createElementsTree>( - config, - elements, - update, - dispatchEvent - ); + return createElementsTree>({ + ...params, + root: elements, + }); }; -export const getImaVidElements: GetElements = ( - config, - update, - dispatchEvent, - batchUpdate -) => { - const isThumbnail = config.thumbnail; - const children: Array = [ +export const getImaVidElements: GetElements = (params) => { + const isThumbnail = params.config.thumbnail; + const children: ElementsTemplate< + ImaVidState, + BaseElement + >["children"] = [ { node: imavid.ImaVidElement, }, @@ -231,31 +214,29 @@ export const getImaVidElements: GetElements = ( } children.push( - ...[ - { - node: imavid.ImaVidControlsElement, - children: [ - { node: common.PlusElement }, - { node: common.MinusElement }, - { node: common.CropToContentButtonElement }, - { node: common.ToggleOverlaysButtonElement }, - { node: common.JSONButtonElement }, - { node: common.OptionsButtonElement }, - { node: common.HelpButtonElement }, - ], - }, - { - node: common.OptionsPanelElement, - children: [ - { node: common.LoopVideoOptionElement }, - { node: common.OnlyShowHoveredOnLabelOptionElement }, - { node: common.ShowConfidenceOptionElement }, - { node: common.ShowIndexOptionElement }, - { node: common.ShowLabelOptionElement }, - { node: common.ShowTooltipOptionElement }, - ], - }, - ] + { + node: imavid.ImaVidControlsElement, + children: [ + { node: common.PlusElement }, + { node: common.MinusElement }, + { node: common.CropToContentButtonElement }, + { node: common.ToggleOverlaysButtonElement }, + { node: common.JSONButtonElement }, + { node: common.OptionsButtonElement }, + { node: common.HelpButtonElement }, + ], + }, + { + node: common.OptionsPanelElement, + children: [ + { node: common.LoopVideoOptionElement }, + { node: common.OnlyShowHoveredOnLabelOptionElement }, + { node: common.ShowConfidenceOptionElement }, + { node: common.ShowIndexOptionElement }, + { node: common.ShowLabelOptionElement }, + { node: common.ShowTooltipOptionElement }, + ], + } ); const elements = { @@ -263,20 +244,13 @@ export const getImaVidElements: GetElements = ( children, }; - return createElementsTree>( - config, - elements, - update, - dispatchEvent, - batchUpdate - ); + return createElementsTree>({ + ...params, + root: elements, + }); }; -export const get3dElements: GetElements = ( - config, - update, - dispatchEvent -) => { +export const get3dElements: GetElements = (params) => { const elements = { node: common.LookerElement, children: [ @@ -306,10 +280,8 @@ export const get3dElements: GetElements = ( ], }; - return createElementsTree>( - config, - elements, - update, - dispatchEvent - ); + return createElementsTree>({ + ...params, + root: elements, + }); }; diff --git a/app/packages/looker/src/elements/util.ts b/app/packages/looker/src/elements/util.ts index 2853907b5a..c050f0e696 100644 --- a/app/packages/looker/src/elements/util.ts +++ b/app/packages/looker/src/elements/util.ts @@ -2,8 +2,8 @@ * Copyright 2017-2024, Voxel51, Inc. */ -import { BaseState, StateUpdate } from "../state"; -import { BaseElement, Events } from "./base"; +import type { BaseState, StateUpdate } from "../state"; +import type { BaseElement, Events } from "./base"; export const FRAME_ZERO_OFFSET = 1; @@ -31,7 +31,7 @@ type ElementConstructor< Element extends BaseElement > = new () => Element; -interface ElementsTemplate< +export interface ElementsTemplate< State extends BaseState, Element extends BaseElement = BaseElement > { @@ -42,30 +42,25 @@ interface ElementsTemplate< export function createElementsTree< State extends BaseState, Element extends BaseElement = BaseElement ->( - config: Readonly, - root: ElementsTemplate, - update: StateUpdate, - dispatchEvent: (eventType: string, details?: any) => void, - batchUpdate?: (cb: () => unknown) => void -): Element { - const element = new root.node(); - element.boot(config, update, dispatchEvent, batchUpdate); - - if (!element.isShown(config)) { +>(params: { + abortController: AbortController; + batchUpdate?: (cb: () => unknown) => void; + config: Readonly; + dispatchEvent: (eventType: string, details?: any) => void; + root: ElementsTemplate; + update: StateUpdate; +}): Element { + const element = new params.root.node(); + element.boot(params); + + if (!element.isShown(params.config)) { return element; } let children = new Array>(); - children = root.children - ? root.children.map((child) => - createElementsTree( - config, - child, - update, - dispatchEvent, - batchUpdate - ) + children = params.root.children + ? params.root.children.map((child) => + createElementsTree({ ...params, root: child }) ) : children; @@ -74,10 +69,7 @@ export function createElementsTree< return element; } -const stringifyNumber = function ( - number: number, - pad: boolean = false -): string { +const stringifyNumber = (number: number, pad = false): string => { let str = ""; if (pad && number < 10) { str += "0" + number; @@ -94,13 +86,14 @@ export const getFrameNumber = ( duration: number, frameRate: number ): number => { + let stamp = time; const frameDuration = 1 / frameRate; // account for exact end of video if (time === duration) { - time -= 0.1 * frameDuration; + stamp -= 0.1 * frameDuration; } - return Math.floor(time * frameRate + FRAME_ZERO_OFFSET); + return Math.floor(stamp * frameRate + FRAME_ZERO_OFFSET); }; export const getClampedTime = ( @@ -112,9 +105,7 @@ export const getClampedTime = ( }; export const getTime = (frameNumber: number, frameRate: number): number => { - frameNumber -= 1; - - return (frameNumber + 0.01) / frameRate; + return (frameNumber - 1 + 0.01) / frameRate; }; export const getFrameString = ( diff --git a/app/packages/looker/src/elements/video.ts b/app/packages/looker/src/elements/video.ts index 3ea8ab6466..0954e47e01 100644 --- a/app/packages/looker/src/elements/video.ts +++ b/app/packages/looker/src/elements/video.ts @@ -5,20 +5,15 @@ import { playbackRate, volume as volumeIcon, volumeMuted } from "../icons"; import lockIcon from "../icons/lock.svg"; import lockOpenIcon from "../icons/lockOpen.svg"; -import { VideoState } from "../state"; -import { BaseElement, Events } from "./base"; +import type { VideoState } from "../state"; +import type { Events } from "./base"; +import { BaseElement } from "./base"; import { muteUnmute, playPause, resetPlaybackRate, supportLock, } from "./common/actions"; -import { - lookerClickable, - lookerControlActive, - lookerTime, -} from "./common/controls.module.css"; -import { lookerLoader } from "./common/looker.module.css"; import { dispatchTooltipEvent } from "./common/util"; import { acquirePlayer, @@ -28,6 +23,13 @@ import { getFullTimeString, getTime, } from "./util"; + +import { + lookerClickable, + lookerControlActive, + lookerTime, +} from "./common/controls.module.css"; +import { lookerLoader } from "./common/looker.module.css"; import { bufferingCircle, bufferingPath, @@ -39,7 +41,7 @@ import { } from "./video.module.css"; export class LoaderBar extends BaseElement { - private buffering = false; + private shown: boolean = undefined; isShown({ thumbnail }: Readonly) { return thumbnail; @@ -56,25 +58,24 @@ export class LoaderBar extends BaseElement { buffering, hovering, waitingForVideo, + waitingToStream, error, lockedToSupport, config: { frameRate, support }, }: Readonly) { - if ( - (buffering || waitingForVideo) && - hovering && - !error === this.buffering - ) { - return this.element; - } + const shown = + !error && hovering && (waitingForVideo || buffering || waitingToStream); const start = lockedToSupport ? support[0] : 1; const end = lockedToSupport ? support[1] : getFrameNumber(duration, duration, frameRate); - this.buffering = - (buffering || waitingForVideo) && hovering && !error && start !== end; + if (shown === this.shown || start === end) { + return this.element; + } - if (this.buffering) { + this.shown = shown; + + if (this.shown) { this.element.style.display = "block"; } else { this.element.style.display = "none"; @@ -593,6 +594,17 @@ export class VideoElement extends BaseElement { return this.element; } + private attachEvents() { + for (const eventType in this.events) { + this.element.addEventListener(eventType, this.events[eventType]); + } + } + private removeEvents() { + for (const eventType in this.events) { + this.element.removeEventListener(eventType, this.events[eventType]); + } + } + private acquireVideo() { let called = false; @@ -645,7 +657,7 @@ export class VideoElement extends BaseElement { this.removeEvents(); this.element = null; - this.release && this.release(); + this.release?.(); this.release = null; this.update({ @@ -708,7 +720,11 @@ export class VideoElement extends BaseElement { }); } if (loaded && (!playing || seeking || buffering) && !this.element.paused) { - !this.waitingToPlay ? this.element.pause() : (this.waitingToPause = true); + if (this.waitingToPlay) { + this.waitingToPause = true; + } else { + this.element.pause(); + } } if (this.loop !== loop) { @@ -736,24 +752,37 @@ export class VideoElement extends BaseElement { } export function withVideoLookerEvents(): () => Events { - return function () { + return () => { + let timeout: ReturnType = null; return { mouseenter: ({ update }) => { update(({ config: { thumbnail } }) => { if (thumbnail) { + timeout = setTimeout(() => { + update({ + playing: true, + waitingToStream: false, + }); + }, 500); + return { - playing: true, + waitingToStream: true, }; } return {}; }); }, mouseleave: ({ update }) => { + if (timeout) { + clearTimeout(timeout); + timeout = null; + } update(({ config: { thumbnail, support } }) => { if (thumbnail) { return { frameNumber: support ? support[0] : 1, playing: false, + waitingToStream: false, }; } return { diff --git a/app/packages/looker/src/lookers/abstract.ts b/app/packages/looker/src/lookers/abstract.ts index c290716c8c..7d18e8bddb 100644 --- a/app/packages/looker/src/lookers/abstract.ts +++ b/app/packages/looker/src/lookers/abstract.ts @@ -60,17 +60,11 @@ const getLabelsWorker = (() => { let workers: Worker[]; let next = -1; - return (dispatchEvent, abortController) => { + return () => { if (!workers) { workers = []; for (let i = 0; i < numWorkers; i++) { - workers.push( - createWorker( - LookerUtils.workerCallbacks, - dispatchEvent, - abortController - ) - ); + workers.push(createWorker(LookerUtils.workerCallbacks)); } } @@ -97,6 +91,7 @@ export abstract class AbstractLooker< private previousState?: Readonly; private readonly rootEvents: Events; + protected readonly abortController: AbortController; protected sampleOverlays: Overlay[]; protected currentOverlays: Overlay[]; protected pluckedOverlays: Overlay[]; @@ -108,8 +103,6 @@ export abstract class AbstractLooker< private isBatching = false; private isCommittingBatchUpdates = false; - private readonly abortController: AbortController; - constructor( sample: S, config: State["config"], @@ -120,6 +113,7 @@ export abstract class AbstractLooker< this.subscriptions = {}; this.updater = this.makeUpdate(); this.state = this.getInitialState(config, options); + this.loadSample(sample); this.state.options.mimetype = getMimeType(sample); this.pluckedOverlays = []; @@ -320,7 +314,7 @@ export abstract class AbstractLooker< Boolean(this.currentOverlays.length) && this.currentOverlays[0].containsPoint(this.state) > CONTAINS.NONE; - postUpdate && postUpdate(this.state, this.currentOverlays, this.sample); + postUpdate?.(this.state, this.currentOverlays, this.sample); this.dispatchImpliedEvents(this.previousState, this.state); @@ -332,7 +326,9 @@ export abstract class AbstractLooker< } ctx.lineWidth = this.state.strokeWidth; - ctx.font = `bold ${this.state.fontSize.toFixed(2)}px Palanquin`; + if (!this.state.config.thumbnail) { + ctx.font = `bold ${this.state.fontSize.toFixed(2)}px Palanquin`; + } ctx.textAlign = "left"; ctx.textBaseline = "bottom"; ctx.imageSmoothingEnabled = false; @@ -465,6 +461,9 @@ export abstract class AbstractLooker< }; } + /** + * Attaches the instance to the provided HTMLElement and adds event listeners + */ attach(element: HTMLElement | string, dimensions?: Dimensions): void { if (typeof element === "string") { element = document.getElementById(element); @@ -476,17 +475,13 @@ export abstract class AbstractLooker< } if (this.lookerElement.element.parentElement) { - const parent = this.lookerElement.element.parentElement; - this.resizeObserver.disconnect(); - parent.removeChild(this.lookerElement.element); - - for (const eventType in this.rootEvents) { - parent.removeEventListener(eventType, this.rootEvents[eventType]); - } + console.warn("instance is already attached"); } for (const eventType in this.rootEvents) { - element.addEventListener(eventType, this.rootEvents[eventType]); + element.addEventListener(eventType, this.rootEvents[eventType], { + signal: this.abortController.signal, + }); } this.updater({ windowBBox: dimensions ? [0, 0, ...dimensions] : getElementBBox(element), @@ -502,13 +497,14 @@ export abstract class AbstractLooker< }); } + /** + * Detaches the instance from the DOM + */ detach(): void { this.resizeObserver.disconnect(); - this.lookerElement.element.parentNode && - this.lookerElement.element.parentNode.removeChild( - this.lookerElement.element - ); - this.abortController.abort(); + this.lookerElement.element.parentNode?.removeChild( + this.lookerElement.element + ); } abstract updateOptions(options: Partial): void; @@ -532,20 +528,20 @@ export abstract class AbstractLooker< getCurrentSampleLabels(): LabelData[] { const labels: LabelData[] = []; - this.currentOverlays.forEach((overlay) => { + for (const overlay of this.currentOverlays) { if (overlay instanceof ClassificationsOverlay) { - overlay.getFilteredAndFlat(this.state).forEach(([field, label]) => { + for (const [field, label] of overlay.getFilteredAndFlat(this.state)) { labels.push({ field: field, labelId: label.id, sampleId: this.sample.id, }); - }); + } } else { const { id: labelId, field } = overlay.getSelectData(this.state); labels.push({ labelId, field, sampleId: this.sample.id }); } - }); + } return labels; } @@ -554,14 +550,19 @@ export abstract class AbstractLooker< return false; } + /** + * Detaches the instance from the DOM and aborts all associated event + * listeners + * + * This method must be called to avoid memory leaks associated with detached + * elements + */ destroy() { - this.resizeObserver.disconnect(); - this.lookerElement.element.parentElement && - this.lookerElement.element.parentElement.removeChild( - this.lookerElement.element - ); + this.detach(); + this.abortController.abort(); this.updater({ destroyed: true }); } + disable() { this.updater({ disabled: true }); } @@ -694,10 +695,9 @@ export abstract class AbstractLooker< private loadSample(sample: Sample) { const messageUUID = uuid(); - const labelsWorker = getLabelsWorker( - (event, detail) => this.dispatchEvent(event, detail), - this.abortController - ); + + const labelsWorker = getLabelsWorker(); + const listener = ({ data: { sample, coloring, uuid } }) => { if (uuid === messageUUID) { this.sample = sample; @@ -711,6 +711,7 @@ export abstract class AbstractLooker< labelsWorker.removeEventListener("message", listener); } }; + labelsWorker.addEventListener("message", listener); labelsWorker.postMessage({ diff --git a/app/packages/looker/src/lookers/frame-reader.ts b/app/packages/looker/src/lookers/frame-reader.ts new file mode 100644 index 0000000000..8c309c4261 --- /dev/null +++ b/app/packages/looker/src/lookers/frame-reader.ts @@ -0,0 +1,166 @@ +import type { Schema, Stage } from "@fiftyone/utilities"; +import { LRUCache } from "lru-cache"; +import { v4 as uuid } from "uuid"; +import type { Coloring, CustomizeColor } from ".."; +import { CHUNK_SIZE } from "../constants"; +import { loadOverlays } from "../overlays"; +import type { Overlay } from "../overlays/base"; +import type { + BaseConfig, + BufferRange, + FrameChunkResponse, + FrameSample, + StateUpdate, + VideoState, +} from "../state"; +import { createWorker } from "../util"; +import { LookerUtils, withFrames } from "./shared"; + +type RemoveFrame = (frameNumber: number) => void; + +export interface Frame { + sample: FrameSample; + overlays: Overlay[]; +} + +interface AcquireReaderOptions { + addFrame: (frameNumber: number, frame: Frame) => void; + addFrameBuffers: (range: [number, number]) => void; + coloring: Coloring; + customizeColorSetting: CustomizeColor[]; + dispatchEvent: (eventType: "buffering", detail: boolean) => void; + getCurrentFrame: () => number; + dataset: string; + frameNumber: number; + frameCount: number; + group: BaseConfig["group"]; + maxFrameStreamSize?: number; + removeFrame: RemoveFrame; + sampleId: string; + schema: Schema; + update: StateUpdate; + view: Stage[]; +} + +export const { acquireReader, clearReader } = (() => { + const createCache = ( + removeFrame: RemoveFrame, + maxFrameStreamSize?: number + ) => { + console.log(maxFrameStreamSize); + return new LRUCache({ + max: maxFrameStreamSize || 1000, + dispose: (_, key) => { + removeFrame(key); + }, + }); + }; + + let currentOptions: AcquireReaderOptions = null; + let frameCache: LRUCache = null; + let frameReader: Worker; + let nextRange: BufferRange = null; + let requestingFrames = false; + + const setStream = ({ + addFrame, + addFrameBuffers, + frameNumber, + frameCount, + sampleId, + update, + dispatchEvent, + coloring, + customizeColorSetting, + dataset, + view, + group, + schema, + }: AcquireReaderOptions): string => { + nextRange = [frameNumber, Math.min(frameCount, CHUNK_SIZE + frameNumber)]; + const subscription = uuid(); + + frameReader?.terminate(); + frameReader = createWorker({ + ...LookerUtils.workerCallbacks, + frameChunk: [ + (worker, { frames, range: [start, end] }: FrameChunkResponse) => { + addFrameBuffers([start, end]); + for (let i = 0; i < frames.length; i++) { + const frameSample = frames[i]; + const prefixedFrameSample = withFrames(frameSample); + const overlays = loadOverlays(prefixedFrameSample, schema); + const frame = { overlays, sample: frameSample }; + frameCache.set(frameSample.frame_number, frame); + addFrame(frameSample.frame_number, frame); + } + + if (end < frameCount) { + nextRange = [end + 1, Math.min(frameCount, end + 1 + CHUNK_SIZE)]; + requestingFrames = true; + worker.postMessage({ + method: "requestFrameChunk", + uuid: subscription, + }); + } else { + requestingFrames = false; + nextRange = null; + } + + update((state) => { + state.buffering && dispatchEvent("buffering", false); + return { buffering: false }; + }); + }, + ], + }); + + requestingFrames = true; + frameReader.postMessage({ + method: "setStream", + sampleId, + frameCount, + frameNumber, + uuid: subscription, + coloring, + customizeColorSetting, + dataset, + view, + group, + schema, + }); + return subscription; + }; + + return { + acquireReader: ( + options: AcquireReaderOptions + ): ((frameNumber?: number) => void) => { + currentOptions = options; + + let subscription = setStream(currentOptions); + frameCache = createCache(options.removeFrame, options.maxFrameStreamSize); + + return (frameNumber: number) => { + if (!nextRange) { + nextRange = [frameNumber, frameNumber + CHUNK_SIZE]; + subscription = setStream({ ...currentOptions, frameNumber }); + } else if (!requestingFrames) { + frameReader.postMessage({ + method: "requestFrameChunk", + uuid: subscription, + }); + } + requestingFrames = true; + }; + }, + + clearReader: () => { + nextRange = null; + frameCache = null; + currentOptions = null; + frameReader?.terminate(); + frameReader = undefined; + }, + }; +})(); diff --git a/app/packages/looker/src/lookers/image.ts b/app/packages/looker/src/lookers/image.ts index 5e20d9670a..db49b23cac 100644 --- a/app/packages/looker/src/lookers/image.ts +++ b/app/packages/looker/src/lookers/image.ts @@ -1,31 +1,51 @@ import { getImageElements } from "../elements"; import { COMMON_SHORTCUTS } from "../elements/common"; -import { Overlay } from "../overlays/base"; -import { DEFAULT_IMAGE_OPTIONS, ImageState } from "../state"; +import type { Overlay } from "../overlays/base"; +import type { ImageState } from "../state"; +import { DEFAULT_IMAGE_OPTIONS } from "../state"; import { AbstractLooker } from "./abstract"; import { LookerUtils } from "./shared"; +import { + nextFrameNoOpControl, + previousFrameNoOpControl, +} from "../elements/common/actions"; import { zoomToContent } from "../zoom"; export class ImageLooker extends AbstractLooker { getElements(config) { - return getImageElements(config, this.updater, this.getDispatchEvent()); + return getImageElements({ + abortController: this.abortController, + config, + dispatchEvent: this.getDispatchEvent(), + update: this.updater, + }); } getInitialState( config: ImageState["config"], options: ImageState["options"] ) { - options = { + const resolved = { ...this.getDefaultOptions(), ...options, }; + // if in dynamic groups mode, add < > shortcuts, too + let shortcuts = { ...COMMON_SHORTCUTS }; + if (config.isDynamicGroup) { + shortcuts = { + ...COMMON_SHORTCUTS, + previousFrameNoOpControl, + nextFrameNoOpControl, + }; + } + return { ...this.getInitialBaseState(), config: { ...config }, - options, - SHORTCUTS: COMMON_SHORTCUTS, + options: resolved, + SHORTCUTS: shortcuts, }; } diff --git a/app/packages/looker/src/lookers/imavid/controller.ts b/app/packages/looker/src/lookers/imavid/controller.ts index 7ea1b8040e..7428d1392d 100644 --- a/app/packages/looker/src/lookers/imavid/controller.ts +++ b/app/packages/looker/src/lookers/imavid/controller.ts @@ -1,6 +1,6 @@ import * as foq from "@fiftyone/relay"; import { BufferManager } from "@fiftyone/utilities"; -import { Environment, fetchQuery, Subscription } from "relay-runtime"; +import { Environment, Subscription, fetchQuery } from "relay-runtime"; import { BufferRange, ImaVidState, StateUpdate } from "../../state"; import { BUFFERS_REFRESH_TIMEOUT_YIELD, DEFAULT_FRAME_RATE } from "./constants"; import { @@ -26,6 +26,7 @@ export class ImaVidFramesController { constructor( private readonly config: { + maxFrameStreamSize: number; environment: Environment; firstFrameNumber: number; // todo: remove any @@ -173,7 +174,10 @@ export class ImaVidFramesController { if (!ImaVidStore.has(this.key)) { ImaVidStore.set( this.key, - new ImaVidFrameSamples(this.storeBufferManager) + new ImaVidFrameSamples( + this.config.maxFrameStreamSize, + this.storeBufferManager + ) ); } diff --git a/app/packages/looker/src/lookers/imavid/ima-vid-frame-samples.ts b/app/packages/looker/src/lookers/imavid/ima-vid-frame-samples.ts index 2e92b8f45d..e30b158b5b 100644 --- a/app/packages/looker/src/lookers/imavid/ima-vid-frame-samples.ts +++ b/app/packages/looker/src/lookers/imavid/ima-vid-frame-samples.ts @@ -1,7 +1,7 @@ import { + ModalSample, getSampleSrc, getStandardizedUrls, - ModalSample, } from "@fiftyone/state"; import { BufferManager } from "@fiftyone/utilities"; import { LRUCache } from "lru-cache"; @@ -24,12 +24,12 @@ export class ImaVidFrameSamples { private readonly abortController: AbortController; - constructor(storeBufferManager: BufferManager) { + constructor(cacheSize: number, storeBufferManager: BufferManager) { this.storeBufferManager = storeBufferManager; this.abortController = new AbortController(); this.samples = new LRUCache({ - max: MAX_FRAME_SAMPLES_CACHE_SIZE, + max: cacheSize || MAX_FRAME_SAMPLES_CACHE_SIZE, dispose: (_modal, sampleId) => { // remove it from the frame index const frameNumber = this.reverseFrameIndex.get(sampleId); diff --git a/app/packages/looker/src/lookers/imavid/index.ts b/app/packages/looker/src/lookers/imavid/index.ts index cd302395e4..dc75a3b4f2 100644 --- a/app/packages/looker/src/lookers/imavid/index.ts +++ b/app/packages/looker/src/lookers/imavid/index.ts @@ -66,7 +66,7 @@ export class ImaVidLooker extends AbstractLooker { } destroy() { - this.unsubscribe && this.unsubscribe(); + this.unsubscribe?.(); this.frameStoreController.pauseFetch(); this.pause(); super.destroy(); @@ -87,12 +87,13 @@ export class ImaVidLooker extends AbstractLooker { } getElements(config) { - const elements = getImaVidElements( + const elements = getImaVidElements({ + abortController: this.abortController, + batchUpdate: this.batchUpdater.bind(this), config, - this.updater.bind(this), - this.getDispatchEvent(), - this.batchUpdater.bind(this) - ); + update: this.updater.bind(this), + dispatchEvent: this.getDispatchEvent(), + }); this.elements = elements; return elements; } diff --git a/app/packages/looker/src/lookers/shared.ts b/app/packages/looker/src/lookers/shared.ts index 574f999590..2b4cc3d19d 100644 --- a/app/packages/looker/src/lookers/shared.ts +++ b/app/packages/looker/src/lookers/shared.ts @@ -1,6 +1,6 @@ import { ClassificationsOverlay } from "../overlays"; -import { Overlay } from "../overlays/base"; -import { +import type { Overlay } from "../overlays/base"; +import type { BaseState, FrameState, ImaVidState, @@ -109,3 +109,8 @@ export const LookerUtils = { ], }, }; + +export const withFrames = (obj: T): T => + Object.fromEntries( + Object.entries(obj).map(([k, v]) => ["frames." + k, v]) + ) as T; diff --git a/app/packages/looker/src/lookers/three-d.ts b/app/packages/looker/src/lookers/three-d.ts index 12c5ff38d2..5f40557155 100644 --- a/app/packages/looker/src/lookers/three-d.ts +++ b/app/packages/looker/src/lookers/three-d.ts @@ -1,12 +1,18 @@ import { get3dElements } from "../elements"; import { COMMON_SHORTCUTS } from "../elements/common"; -import { DEFAULT_3D_OPTIONS, ThreeDState } from "../state"; +import type { ThreeDState } from "../state"; +import { DEFAULT_3D_OPTIONS } from "../state"; import { AbstractLooker } from "./abstract"; import { LookerUtils } from "./shared"; export class ThreeDLooker extends AbstractLooker { getElements(config) { - return get3dElements(config, this.updater, this.getDispatchEvent()); + return get3dElements({ + abortController: this.abortController, + config, + update: this.updater, + dispatchEvent: this.getDispatchEvent(), + }); } hasDefaultZoom(): boolean { diff --git a/app/packages/looker/src/lookers/video.ts b/app/packages/looker/src/lookers/video.ts index 305bb3ce83..8032d1bf18 100644 --- a/app/packages/looker/src/lookers/video.ts +++ b/app/packages/looker/src/lookers/video.ts @@ -1,214 +1,32 @@ -import { v4 as uuid } from "uuid"; +import { setFrameNumberAtom } from "@fiftyone/playback"; +import { getDefaultStore } from "jotai"; import { getVideoElements } from "../elements"; import { VIDEO_SHORTCUTS } from "../elements/common"; +import { getFrameNumber } from "../elements/util"; import { ClassificationsOverlay, loadOverlays } from "../overlays"; -import { Overlay } from "../overlays/base"; +import type { Overlay } from "../overlays/base"; import processOverlays from "../processOverlays"; -import { - BaseConfig, - BufferRange, +import type { Buffers, - Coloring, - CustomizeColor, - DEFAULT_VIDEO_OPTIONS, - FrameChunkResponse, FrameSample, LabelData, - StateUpdate, VideoConfig, VideoSample, VideoState, } from "../state"; -import { addToBuffers, createWorker, removeFromBuffers } from "../util"; - -import { setFrameNumberAtom } from "@fiftyone/playback"; -import { Schema } from "@fiftyone/utilities"; -import { getDefaultStore } from "jotai"; -import { LRUCache } from "lru-cache"; -import { CHUNK_SIZE, MAX_FRAME_CACHE_SIZE_BYTES } from "../constants"; -import { getFrameNumber } from "../elements/util"; +import { DEFAULT_VIDEO_OPTIONS } from "../state"; +import { addToBuffers, removeFromBuffers } from "../util"; import { AbstractLooker } from "./abstract"; -import { LookerUtils } from "./shared"; - -interface Frame { - sample: FrameSample; - overlays: Overlay[]; -} - -type RemoveFrame = (frameNumber: number) => void; - -interface AcquireReaderOptions { - addFrame: (frameNumber: number, frame: Frame) => void; - addFrameBuffers: (range: [number, number]) => void; - removeFrame: RemoveFrame; - getCurrentFrame: () => number; - dataset: string; - group: BaseConfig["group"]; - view: any[]; - sampleId: string; - frameNumber: number; - frameCount: number; - update: StateUpdate; - dispatchEvent: (eventType: string, detail: any) => void; - coloring: Coloring; - customizeColorSetting: CustomizeColor[]; - schema: Schema; -} - -const { acquireReader, addFrame } = (() => { - const createCache = () => - new LRUCache, Frame>({ - maxSize: MAX_FRAME_CACHE_SIZE_BYTES, - sizeCalculation: (frame) => { - let size = 1; - frame.overlays.forEach((overlay) => { - size += overlay.getSizeBytes(); - }); - return size; - }, - dispose: (frame, removeFrameRef) => { - const removeFrame = removeFrameRef.deref(); - removeFrame && removeFrame(frame.sample.frame_number); - }, - }); - - const frameCache = createCache(); - let frameReader: Worker; - - let streamSize = 0; - let nextRange: BufferRange = null; - - let requestingFrames = false; - let currentOptions: AcquireReaderOptions = null; - - const setStream = ({ - addFrame, - addFrameBuffers, - removeFrame, - frameNumber, - frameCount, - sampleId, - update, - dispatchEvent, - coloring, - customizeColorSetting, - dataset, - view, - group, - schema, - }: AcquireReaderOptions): string => { - streamSize = 0; - nextRange = [frameNumber, Math.min(frameCount, CHUNK_SIZE + frameNumber)]; - const subscription = uuid(); - frameReader && frameReader.terminate(); - frameReader = createWorker(LookerUtils.workerCallbacks, dispatchEvent); - frameReader.onmessage = ({ data }: MessageEvent) => { - if (data.uuid !== subscription || data.method !== "frameChunk") { - return; - } - - const { - frames, - range: [start, end], - } = data; - addFrameBuffers([start, end]); - for (let i = start; i <= end; i++) { - const frame = { - sample: { - frame_number: i, - }, - overlays: [], - }; - frameCache.set(new WeakRef(removeFrame), frame); - addFrame(i, frame); - } - - for (let i = 0; i < frames.length; i++) { - const frameSample = frames[i]; - const prefixedFrameSample = withFrames(frameSample); - - const overlays = loadOverlays(prefixedFrameSample, schema); - overlays.forEach((overlay) => { - streamSize += overlay.getSizeBytes(); - }); - const frame = { sample: frameSample, overlays }; - frameCache.set(new WeakRef(removeFrame), frame); - addFrame(frameSample.frame_number, frame); - } +import { type Frame, acquireReader, clearReader } from "./frame-reader"; +import { LookerUtils, withFrames } from "./shared"; - const requestMore = streamSize < MAX_FRAME_CACHE_SIZE_BYTES; - - if (requestMore && end < frameCount) { - nextRange = [end + 1, Math.min(frameCount, end + 1 + CHUNK_SIZE)]; - requestingFrames = true; - frameReader.postMessage({ - method: "requestFrameChunk", - uuid: subscription, - }); - } else { - requestingFrames = false; - nextRange = null; - } - - update((state) => { - state.buffering && dispatchEvent("buffering", false); - return { buffering: false }; - }); - }; - - requestingFrames = true; - frameReader.postMessage({ - method: "setStream", - sampleId, - frameCount, - frameNumber, - uuid: subscription, - coloring, - customizeColorSetting, - dataset, - view, - group, - schema, - }); - return subscription; - }; - - return { - acquireReader: ( - options: AcquireReaderOptions - ): ((frameNumber?: number) => void) => { - currentOptions = options; - let subscription = setStream(currentOptions); - - return (frameNumber: number, force?: boolean) => { - if ( - force || - !nextRange || - (frameNumber < nextRange[0] && frameNumber > nextRange[1]) - ) { - force && frameCache.clear(); - nextRange = [frameNumber, frameNumber + CHUNK_SIZE]; - subscription = setStream({ ...currentOptions, frameNumber }); - } else if (!requestingFrames) { - frameReader.postMessage({ - method: "requestFrameChunk", - uuid: subscription, - }); - } - requestingFrames = true; - }; - }, - addFrame: (removeFrame: RemoveFrame, frame: Frame): void => { - frameCache.set(new WeakRef(removeFrame), frame); - }, - }; -})(); - -let lookerWithReader: VideoLooker | null = null; +let LOOKER_WITH_READER: VideoLooker | null = null; export class VideoLooker extends AbstractLooker { + private firstFrame: Frame; + private firstFrameNumber: number; private frames: Map> = new Map(); - private requestFrames: (frameNumber: number, force?: boolean) => void; + private requestFrames: (frameNumber: number) => void; get frameNumber() { return this.state.frameNumber; @@ -228,9 +46,13 @@ export class VideoLooker extends AbstractLooker { ); } - destroy() { + detach() { this.pause(); - super.destroy(); + if (LOOKER_WITH_READER === this) { + clearReader(); + LOOKER_WITH_READER = null; + } + super.detach(); } dispatchImpliedEvents( @@ -249,55 +71,62 @@ export class VideoLooker extends AbstractLooker { getCurrentSampleLabels(): LabelData[] { const labels: LabelData[] = []; - processOverlays(this.state, this.sampleOverlays)[0].forEach((overlay) => { + for (const overlay of processOverlays(this.state, this.sampleOverlays)[0]) { if (overlay instanceof ClassificationsOverlay) { - overlay.getFilteredAndFlat(this.state).forEach(([field, label]) => { + for (const [field, label] of overlay.getFilteredAndFlat(this.state)) { labels.push({ field: field, labelId: label.id, sampleId: this.sample.id, }); - }); + } } else { const { id: labelId, field } = overlay.getSelectData(this.state); labels.push({ labelId, field, sampleId: this.sample.id }); } - }); + } return labels; } getCurrentFrameLabels(): LabelData[] { const frame = this.frames.get(this.frameNumber).deref(); + if (!frame) { + return []; + } const labels: LabelData[] = []; - if (frame) { - processOverlays(this.state, frame.overlays)[0].forEach((overlay) => { - if (overlay instanceof ClassificationsOverlay) { - overlay.getFilteredAndFlat(this.state).forEach(([field, label]) => { - labels.push({ - field: field, - labelId: label.id, - frameNumber: this.frameNumber, - sampleId: this.sample.id, - }); - }); - } else { - const { id: labelId, field } = overlay.getSelectData(this.state); + + for (const overlay of processOverlays(this.state, frame.overlays)[0]) { + if (overlay instanceof ClassificationsOverlay) { + for (const [field, label] of overlay.getFilteredAndFlat(this.state)) { labels.push({ - labelId, - field, - sampleId: this.sample.id, + field: field, + labelId: label.id, frameNumber: this.frameNumber, + sampleId: this.sample.id, }); } - }); + } else { + const { id: labelId, field } = overlay.getSelectData(this.state); + labels.push({ + labelId, + field, + sampleId: this.sample.id, + frameNumber: this.frameNumber, + }); + } } return labels; } getElements(config) { - return getVideoElements(config, this.updater, this.getDispatchEvent()); + return getVideoElements({ + abortController: this.abortController, + config, + dispatchEvent: this.getDispatchEvent(), + update: this.updater, + }); } getInitialState( @@ -324,6 +153,7 @@ export class VideoLooker extends AbstractLooker { SHORTCUTS: VIDEO_SHORTCUTS, hasPoster: false, waitingForVideo: false, + waitingToStream: false, lockedToSupport: Boolean(config.support), }; } @@ -347,28 +177,22 @@ export class VideoLooker extends AbstractLooker { this.state.config.fieldSchema, true ); - - const providedFrames = sample.frames?.length + const [firstFrameData] = sample.frames?.length ? sample.frames : [{ frame_number: 1 }]; - - const providedFrameOverlays = providedFrames.map((frameSample) => - loadOverlays(withFrames(frameSample), this.state.config.fieldSchema) + const firstFrameOverlays = loadOverlays( + withFrames(firstFrameData), + this.state.config.fieldSchema ); - - const frames = providedFrames.map((frameSample, i) => ({ - sample: frameSample as FrameSample, - overlays: providedFrameOverlays[i], - })); - frames.forEach((frame) => { - const frameNumber = frame.sample.frame_number; - addFrame( - (frameNumber) => removeFromBuffers(frameNumber, this.state.buffers), - frame - ); - this.frames.set(frame.sample.frame_number, new WeakRef(frame)); - addToBuffers([frameNumber, frameNumber], this.state.buffers); - }); + const firstFrame = { + sample: firstFrameData as FrameSample, + overlays: firstFrameOverlays, + }; + this.firstFrame = firstFrame; + this.firstFrameNumber = firstFrameData.frame_number; + const frameNumber = firstFrame.sample.frame_number; + this.frames.set(firstFrame.sample.frame_number, new WeakRef(firstFrame)); + addToBuffers([frameNumber, frameNumber], this.state.buffers); } pluckOverlays(state: VideoState) { @@ -381,11 +205,9 @@ export class VideoLooker extends AbstractLooker { } let pluckedOverlays = hideSampleOverlays ? [] : this.sampleOverlays; - if (this.hasFrame(state.frameNumber)) { - const frame = this.frames.get(state.frameNumber)?.deref(); - if (frame !== undefined) { - pluckedOverlays = [...pluckedOverlays, ...frame.overlays]; - } + const frame = this.getFrame(state.frameNumber); + if (frame) { + pluckedOverlays = [...pluckedOverlays, ...frame.overlays]; } let frameCount = null; @@ -400,19 +222,19 @@ export class VideoLooker extends AbstractLooker { if ( (!state.config.thumbnail || state.playing) && - lookerWithReader !== this && + LOOKER_WITH_READER !== this && frameCount !== null ) { - lookerWithReader?.pause(); - lookerWithReader = this; + // eslint-disable-next-line @typescript-eslint/no-this-alias + LOOKER_WITH_READER = this; this.setReader(); - } else if (lookerWithReader !== this && frameCount) { + } else if (LOOKER_WITH_READER !== this && frameCount) { this.state.buffering && this.dispatchEvent("buffering", false); this.state.playing = false; this.state.buffering = false; } - if (lookerWithReader === this) { + if (LOOKER_WITH_READER === this) { if (this.hasFrame(Math.min(frameCount, state.frameNumber + 1))) { this.state.buffering && this.dispatchEvent("buffering", false); this.state.buffering = false; @@ -430,26 +252,29 @@ export class VideoLooker extends AbstractLooker { this.requestFrames = acquireReader({ addFrame: (frameNumber, frame) => this.frames.set(frameNumber, new WeakRef(frame)), - addFrameBuffers: (range) => - (this.state.buffers = addToBuffers(range, this.state.buffers)), - removeFrame: (frameNumber) => - removeFromBuffers(frameNumber, this.state.buffers), - getCurrentFrame: () => this.frameNumber, - sampleId: this.state.config.sampleId, + addFrameBuffers: (range) => { + this.state.buffers = addToBuffers(range, this.state.buffers); + }, + coloring: this.state.options.coloring, + customizeColorSetting: this.state.options.customizeColorSetting, + dispatchEvent: (event, detail) => this.dispatchEvent(event, detail), + dataset: this.state.config.dataset, frameCount: getFrameNumber( this.state.duration, this.state.duration, this.state.config.frameRate ), frameNumber: this.state.frameNumber, - update: this.updater, - dispatchEvent: (event, detail) => this.dispatchEvent(event, detail), - coloring: this.state.options.coloring, - customizeColorSetting: this.state.options.customizeColorSetting, - dataset: this.state.config.dataset, + getCurrentFrame: () => this.frameNumber, group: this.state.config.group, - view: this.state.config.view, + maxFrameStreamSize: this.state.config.maxFrameStreamSize, + maxFrameStreamSizeBytes: this.state.config.maxFrameStreamSizeBytes, + removeFrame: (frameNumber) => + removeFromBuffers(frameNumber, this.state.buffers), + sampleId: this.state.config.sampleId, schema: this.state.config.fieldSchema, + update: this.updater, + view: this.state.config.view, }); } @@ -534,6 +359,13 @@ export class VideoLooker extends AbstractLooker { }); } + if (LOOKER_WITH_READER === this) { + if (this.state.config.thumbnail && !this.state.hovering) { + clearReader(); + LOOKER_WITH_READER = null; + } + } + return super.postProcess(); } @@ -549,9 +381,9 @@ export class VideoLooker extends AbstractLooker { } updateSample(sample: VideoSample) { - if (lookerWithReader === this) { - lookerWithReader?.pause(); - lookerWithReader = null; + if (LOOKER_WITH_READER === this) { + LOOKER_WITH_READER?.pause(); + LOOKER_WITH_READER = null; } this.frames = new Map(); @@ -564,19 +396,25 @@ export class VideoLooker extends AbstractLooker { } private hasFrame(frameNumber: number) { + if (frameNumber === this.firstFrameNumber) { + return this.firstFrame; + } return ( this.frames.has(frameNumber) && this.frames.get(frameNumber)?.deref() !== undefined ); } + private getFrame(frameNumber: number) { + if (frameNumber === this.firstFrameNumber) { + return this.firstFrame; + } + + return this.frames.get(frameNumber)?.deref(); + } + private initialBuffers(config: VideoConfig) { const firstFrame = config.support ? config.support[0] : 1; return [[firstFrame, firstFrame]] as Buffers; } } - -const withFrames = (obj: T): T => - Object.fromEntries( - Object.entries(obj).map(([k, v]) => ["frames." + k, v]) - ) as T; diff --git a/app/packages/looker/src/overlays/base.ts b/app/packages/looker/src/overlays/base.ts index 37721e2540..97b270c978 100644 --- a/app/packages/looker/src/overlays/base.ts +++ b/app/packages/looker/src/overlays/base.ts @@ -4,7 +4,7 @@ import { getCls } from "@fiftyone/utilities"; import { BaseState, Coordinates, NONFINITE } from "../state"; -import { getLabelColor, shouldShowLabelTag, sizeBytes } from "./util"; +import { getLabelColor, shouldShowLabelTag } from "./util"; // in numerical order (CONTAINS_BORDER takes precedence over CONTAINS_CONTENT) export enum CONTAINS { @@ -66,7 +66,6 @@ export interface Overlay> { getPointInfo(state: Readonly): any; getSelectData(state: Readonly): SelectData; getPoints(state: Readonly): Coordinates[]; - getSizeBytes(): number; } export abstract class CoordinateOverlay< @@ -126,10 +125,6 @@ export abstract class CoordinateOverlay< abstract getPoints(state: Readonly): Coordinates[]; - getSizeBytes(): number { - return sizeBytes(this.label); - } - getSelectData(state: Readonly): SelectData { return { id: this.label.id, diff --git a/app/packages/looker/src/overlays/classifications.ts b/app/packages/looker/src/overlays/classifications.ts index 5754706040..cce6bc369f 100644 --- a/app/packages/looker/src/overlays/classifications.ts +++ b/app/packages/looker/src/overlays/classifications.ts @@ -3,10 +3,10 @@ */ import { - getCls, REGRESSION, TEMPORAL_DETECTION, TEMPORAL_DETECTIONS, + getCls, } from "@fiftyone/utilities"; import { INFO_COLOR, MOMENT_CLASSIFICATIONS } from "../constants"; @@ -19,13 +19,13 @@ import { } from "../state"; import { CONTAINS, - isShown, Overlay, PointInfo, RegularLabel, SelectData, + isShown, } from "./base"; -import { getLabelColor, sizeBytes } from "./util"; +import { getLabelColor } from "./util"; export type Classification = RegularLabel; @@ -163,16 +163,6 @@ export class ClassificationsOverlay< return []; } - getSizeBytes() { - let bytes = 100; - this.labels.forEach(([_, labels]) => { - labels.forEach((label) => { - bytes += sizeBytes(label); - }); - }); - return bytes; - } - protected getFiltered(state: Readonly): Labels