From 94aa6ea19815ec8e7742e35575b9c9f1f880604d Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Wed, 21 Feb 2024 08:31:39 -0500 Subject: [PATCH 01/36] reintroduce last PR changes --- docs/examples/example.ipynb | 2 +- src/viam/components/camera/camera.py | 13 +++++-------- src/viam/components/camera/client.py | 19 +++++++------------ tests/test_camera.py | 24 ++++++++++++------------ 4 files changed, 25 insertions(+), 33 deletions(-) diff --git a/docs/examples/example.ipynb b/docs/examples/example.ipynb index 73998a711..c94726455 100644 --- a/docs/examples/example.ipynb +++ b/docs/examples/example.ipynb @@ -165,7 +165,7 @@ "robot = await connect_with_channel()\n", "camera = Camera.from_robot(robot, \"camera0\")\n", "image = await camera.get_image(CameraMimeType.JPEG)\n", - "image.save(\"foo.png\")\n", + "image.image.save(\"foo.png\")\n", "\n", "# Don't forget to close the robot when you're done!\n", "await robot.close()" diff --git a/src/viam/components/camera/camera.py b/src/viam/components/camera/camera.py index 2dfb6cc42..fdab7108e 100644 --- a/src/viam/components/camera/camera.py +++ b/src/viam/components/camera/camera.py @@ -1,16 +1,13 @@ import abc import sys -from typing import Any, Dict, Final, List, Optional, Tuple, Union +from typing import Any, Dict, Final, List, Optional, Tuple -from PIL.Image import Image - -from viam.media.video import NamedImage +from viam.media.video import NamedImage, ViamImage from viam.proto.common import ResponseMetadata from viam.proto.component.camera import GetPropertiesResponse from viam.resource.types import RESOURCE_NAMESPACE_RDK, RESOURCE_TYPE_COMPONENT, Subtype from ..component_base import ComponentBase -from . import RawImage if sys.version_info >= (3, 10): from typing import TypeAlias @@ -36,8 +33,8 @@ class Camera(ComponentBase): @abc.abstractmethod async def get_image( self, mime_type: str = "", *, extra: Optional[Dict[str, Any]] = None, timeout: Optional[float] = None, **kwargs - ) -> Union[Image, RawImage]: - """Get the next image from the camera as an Image or RawImage. + ) -> ViamImage: + """Get the next image from the camera as a ViamImage. Be sure to close the image when finished. NOTE: If the mime type is ``image/vnd.viam.dep`` you can use :func:`viam.media.video.ViamImage.bytes_to_depth_array` @@ -47,7 +44,7 @@ async def get_image( mime_type (str): The desired mime type of the image. This does not guarantee output type Returns: - Image | RawImage: The frame + ViamImage: The frame """ ... diff --git a/src/viam/components/camera/client.py b/src/viam/components/camera/client.py index bcf7af0ba..808d9242a 100644 --- a/src/viam/components/camera/client.py +++ b/src/viam/components/camera/client.py @@ -1,10 +1,8 @@ -from io import BytesIO -from typing import Any, Dict, List, Mapping, Optional, Tuple, Union +from typing import Any, Dict, List, Mapping, Optional, Tuple from grpclib.client import Channel -from PIL import Image -from viam.media.video import LIBRARY_SUPPORTED_FORMATS, CameraMimeType, NamedImage +from viam.media.video import CameraMimeType, NamedImage, ViamImage from viam.proto.common import DoCommandRequest, DoCommandResponse, Geometry, ResponseMetadata from viam.proto.component.camera import ( CameraServiceStub, @@ -19,17 +17,14 @@ from viam.resource.rpc_client_base import ReconfigurableResourceRPCClientBase from viam.utils import ValueTypes, dict_to_struct, get_geometries, struct_to_dict -from . import Camera, RawImage +from . import Camera -def get_image_from_response(data: bytes, response_mime_type: str, request_mime_type: str) -> Union[Image.Image, RawImage]: +def get_image_from_response(data: bytes, response_mime_type: str, request_mime_type: str) -> ViamImage: if not request_mime_type: request_mime_type = response_mime_type - mime_type, is_lazy = CameraMimeType.from_lazy(request_mime_type) - if is_lazy or mime_type._should_be_raw: - image = RawImage(data=data, mime_type=response_mime_type) - return image - return Image.open(BytesIO(data), formats=LIBRARY_SUPPORTED_FORMATS) + mime_type, _ = CameraMimeType.from_lazy(request_mime_type) + return ViamImage(data, mime_type) class CameraClient(Camera, ReconfigurableResourceRPCClientBase): @@ -49,7 +44,7 @@ async def get_image( extra: Optional[Dict[str, Any]] = None, timeout: Optional[float] = None, **__, - ) -> Union[Image.Image, RawImage]: + ) -> ViamImage: if extra is None: extra = {} request = GetImageRequest(name=self.name, mime_type=mime_type, extra=dict_to_struct(extra)) diff --git a/tests/test_camera.py b/tests/test_camera.py index ed68aeb8d..2d2c51888 100644 --- a/tests/test_camera.py +++ b/tests/test_camera.py @@ -10,7 +10,7 @@ from viam.components.camera import Camera, CameraClient from viam.components.camera.service import CameraRPCService from viam.components.generic.service import GenericRPCService -from viam.media.video import LIBRARY_SUPPORTED_FORMATS, CameraMimeType, NamedImage, RawImage +from viam.media.video import LIBRARY_SUPPORTED_FORMATS, CameraMimeType, NamedImage, RawImage, ViamImage from viam.proto.common import DoCommandRequest, DoCommandResponse, GetGeometriesRequest, GetGeometriesResponse, ResponseMetadata from viam.proto.component.camera import ( CameraServiceStub, @@ -253,30 +253,30 @@ async def test_get_image(self, camera: MockCamera, service: CameraRPCService, im # Test known mime type png_img = await client.get_image(timeout=1.82, mime_type=CameraMimeType.PNG) - assert isinstance(png_img, Image.Image) - assert png_img.tobytes() == image.tobytes() - assert isinstance(png_img, Image.Image) - assert png_img.tobytes() == image.tobytes() + assert isinstance(png_img.image, Image.Image) + assert png_img.image.tobytes() == image.tobytes() + assert isinstance(png_img.image, Image.Image) + assert png_img.image.tobytes() == image.tobytes() assert camera.timeout == loose_approx(1.82) # Test raw mime type rgba_img = await client.get_image(CameraMimeType.VIAM_RGBA) - assert isinstance(rgba_img, Image.Image) - rgba_bytes = rgba_img.tobytes() - assert isinstance(rgba_img, Image.Image) - rgba_bytes = rgba_img.tobytes() + assert isinstance(rgba_img.image, Image.Image) + rgba_bytes = rgba_img.image.tobytes() assert rgba_bytes == image.copy().convert("RGBA").tobytes() # Test lazy mime type raw_img = await client.get_image(CameraMimeType.PNG.with_lazy_suffix) - assert isinstance(raw_img, RawImage) + assert isinstance(raw_img, ViamImage) + assert raw_img.image is None assert raw_img.data == image.tobytes() assert raw_img.mime_type == CameraMimeType.PNG # Test unknown mime type raw_img = await client.get_image("unknown") - assert isinstance(raw_img, RawImage) - assert raw_img.mime_type == "unknown" + assert isinstance(raw_img, ViamImage) + assert raw_img.image is None + assert raw_img.mime_type == CameraMimeType.UNSUPPORTED @pytest.mark.asyncio async def test_get_images(self, camera: MockCamera, service: CameraRPCService, image: Image.Image, metadata: ResponseMetadata): From 979929d166f3008e90f9548a72e3c6adaf329e88 Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Wed, 21 Feb 2024 11:32:45 -0500 Subject: [PATCH 02/36] use viamimage in vision service --- src/viam/services/vision/client.py | 59 +++++++++++++++-------------- src/viam/services/vision/service.py | 19 +++------- src/viam/services/vision/vision.py | 10 ++--- tests/mocks/services.py | 9 ++--- tests/test_vision_service.py | 22 +++++------ 5 files changed, 52 insertions(+), 67 deletions(-) diff --git a/src/viam/services/vision/client.py b/src/viam/services/vision/client.py index 35c6a9aa1..9aba4fe25 100644 --- a/src/viam/services/vision/client.py +++ b/src/viam/services/vision/client.py @@ -1,10 +1,9 @@ -from io import BytesIO -from typing import Any, List, Mapping, Optional, Union +from typing import Any, List, Mapping, Optional from grpclib.client import Channel +from PIL import UnidentifiedImageError -from viam.media.viam_rgba_plugin import Image -from viam.media.video import CameraMimeType, RawImage +from viam.media.video import CameraMimeType, ViamImage from viam.proto.common import DoCommandRequest, DoCommandResponse, PointCloudObject from viam.proto.service.vision import ( Classification, @@ -55,25 +54,26 @@ async def get_detections_from_camera( async def get_detections( self, - image: Union[Image.Image, RawImage], + image: ViamImage, *, extra: Optional[Mapping[str, Any]] = None, timeout: Optional[float] = None, ) -> List[Detection]: if extra is None: extra = {} - mime_type = CameraMimeType.JPEG - if isinstance(image, RawImage): - image = Image.open(BytesIO(image.data), formats=[mime_type.name]) + image.mime_type = CameraMimeType.JPEG - request = GetDetectionsRequest( - name=self.name, - image=mime_type.encode_image(image), - width=image.width, - height=image.height, - mime_type=mime_type, - extra=dict_to_struct(extra), - ) + if image.image is None: + raise UnidentifiedImageError + else: + request = GetDetectionsRequest( + name=self.name, + image=image.data, + width=image.image.width, + height=image.image.height, + mime_type=image.mime_type, + extra=dict_to_struct(extra), + ) response: GetDetectionsResponse = await self.client.GetDetections(request, timeout=timeout) return list(response.detections) @@ -93,7 +93,7 @@ async def get_classifications_from_camera( async def get_classifications( self, - image: Union[Image.Image, RawImage], + image: ViamImage, count: int, *, extra: Optional[Mapping[str, Any]] = None, @@ -101,19 +101,20 @@ async def get_classifications( ) -> List[Classification]: if extra is None: extra = {} - mime_type = CameraMimeType.JPEG - if isinstance(image, RawImage): - image = Image.open(BytesIO(image.data), formats=[mime_type.name]) + image.mime_type = CameraMimeType.JPEG - request = GetClassificationsRequest( - name=self.name, - image=mime_type.encode_image(image), - width=image.width, - height=image.height, - mime_type=mime_type, - n=count, - extra=dict_to_struct(extra), - ) + if image.image is None: + raise UnidentifiedImageError + else: + request = GetClassificationsRequest( + name=self.name, + image=image.data, + width=image.image.width, + height=image.image.height, + mime_type=image.mime_type, + n=count, + extra=dict_to_struct(extra), + ) response: GetClassificationsResponse = await self.client.GetClassifications(request, timeout=timeout) return list(response.classifications) diff --git a/src/viam/services/vision/service.py b/src/viam/services/vision/service.py index 525b46454..cf15de0eb 100644 --- a/src/viam/services/vision/service.py +++ b/src/viam/services/vision/service.py @@ -1,9 +1,6 @@ -from io import BytesIO - from grpclib.server import Stream -from PIL import Image -from viam.media.video import LIBRARY_SUPPORTED_FORMATS, CameraMimeType, RawImage +from viam.media.video import CameraMimeType, ViamImage from viam.proto.common import DoCommandRequest, DoCommandResponse from viam.proto.service.vision import ( GetClassificationsFromCameraRequest, @@ -48,11 +45,8 @@ async def GetDetections(self, stream: Stream[GetDetectionsRequest, GetDetections extra = struct_to_dict(request.extra) timeout = stream.deadline.time_remaining() if stream.deadline else None - mime_type, is_lazy = CameraMimeType.from_lazy(request.mime_type) - if is_lazy or not (CameraMimeType.is_supported(mime_type)): - image = RawImage(request.image, request.mime_type) - else: - image = Image.open(BytesIO(request.image), formats=LIBRARY_SUPPORTED_FORMATS) + mime_type, _ = CameraMimeType.from_lazy(request.mime_type) + image = ViamImage(request.image, mime_type) result = await vision.get_detections(image, extra=extra, timeout=timeout) response = GetDetectionsResponse(detections=result) @@ -77,11 +71,8 @@ async def GetClassifications(self, stream: Stream[GetClassificationsRequest, Get extra = struct_to_dict(request.extra) timeout = stream.deadline.time_remaining() if stream.deadline else None - mime_type, is_lazy = CameraMimeType.from_lazy(request.mime_type) - if is_lazy or not (CameraMimeType.is_supported(mime_type)): - image = RawImage(request.image, request.mime_type) - else: - image = Image.open(BytesIO(request.image), formats=LIBRARY_SUPPORTED_FORMATS) + mime_type, _ = CameraMimeType.from_lazy(request.mime_type) + image = ViamImage(request.image, mime_type) result = await vision.get_classifications(image, request.n, extra=extra, timeout=timeout) response = GetClassificationsResponse(classifications=result) diff --git a/src/viam/services/vision/vision.py b/src/viam/services/vision/vision.py index dec3b04dc..5b751a70a 100644 --- a/src/viam/services/vision/vision.py +++ b/src/viam/services/vision/vision.py @@ -1,9 +1,7 @@ import abc -from typing import Any, Final, List, Mapping, Optional, Union +from typing import Any, Final, List, Mapping, Optional -from PIL import Image - -from viam.media.video import RawImage +from viam.media.video import ViamImage from viam.proto.common import PointCloudObject from viam.proto.service.vision import Classification, Detection from viam.resource.types import RESOURCE_NAMESPACE_RDK, RESOURCE_TYPE_SERVICE, Subtype @@ -47,7 +45,7 @@ async def get_detections_from_camera( @abc.abstractmethod async def get_detections( self, - image: Union[Image.Image, RawImage], + image: ViamImage, *, extra: Optional[Mapping[str, Any]] = None, timeout: Optional[float] = None, @@ -87,7 +85,7 @@ async def get_classifications_from_camera( @abc.abstractmethod async def get_classifications( self, - image: Union[Image.Image, RawImage], + image: ViamImage, count: int, *, extra: Optional[Mapping[str, Any]] = None, diff --git a/tests/mocks/services.py b/tests/mocks/services.py index 7d44afa6a..4fad663e7 100644 --- a/tests/mocks/services.py +++ b/tests/mocks/services.py @@ -1,12 +1,11 @@ -from typing import Any, Dict, List, Mapping, Optional, Tuple, Union +from typing import Any, Dict, List, Mapping, Optional, Tuple import numpy as np from grpclib.server import Stream from numpy.typing import NDArray -from PIL import Image from viam.app.data_client import DataClient -from viam.media.video import RawImage +from viam.media.video import ViamImage from viam.proto.app import ( AddRoleRequest, AddRoleResponse, @@ -332,7 +331,7 @@ async def get_detections_from_camera( return self.detections async def get_detections( - self, image: Union[Image.Image, RawImage], *, extra: Optional[Mapping[str, Any]] = None, timeout: Optional[float] = None + self, image: ViamImage, *, extra: Optional[Mapping[str, Any]] = None, timeout: Optional[float] = None ) -> List[Detection]: self.extra = extra self.timeout = timeout @@ -346,7 +345,7 @@ async def get_classifications_from_camera( return self.classifications async def get_classifications( - self, image: Union[Image.Image, RawImage], count: int, *, extra: Optional[Mapping[str, Any]] = None, timeout: Optional[float] = None + self, image: ViamImage, count: int, *, extra: Optional[Mapping[str, Any]] = None, timeout: Optional[float] = None ) -> List[Classification]: self.extra = extra self.timeout = timeout diff --git a/tests/test_vision_service.py b/tests/test_vision_service.py index 2a7dfdd8d..8b8994777 100644 --- a/tests/test_vision_service.py +++ b/tests/test_vision_service.py @@ -4,7 +4,7 @@ from grpclib.testing import ChannelFor from PIL import Image -from viam.media.video import CameraMimeType +from viam.media.video import CameraMimeType, ViamImage from viam.proto.common import ( DoCommandRequest, DoCommandResponse, @@ -35,6 +35,8 @@ from .mocks.services import MockVision +PILIMAGE = Image.new("RGB", (100, 100), "#AABBCCDD") +IMAGE = ViamImage(CameraMimeType.JPEG.encode_image(PILIMAGE), CameraMimeType.JPEG) DETECTORS = [ "detector-0", "detector-1", @@ -129,9 +131,8 @@ async def test_get_detections_from_camera(self, vision: MockVision): @pytest.mark.asyncio async def test_get_detections(self, vision: MockVision): - image = Image.new("RGB", (100, 100), "#AABBCCDD") extra = {"foo": "get_detections"} - response = await vision.get_detections(image, extra=extra) + response = await vision.get_detections(IMAGE, extra=extra) assert response == DETECTIONS assert vision.extra == extra @@ -144,9 +145,8 @@ async def test_get_classifications_from_camera(self, vision: MockVision): @pytest.mark.asyncio async def test_get_classifications(self, vision: MockVision): - image = Image.new("RGB", (100, 100), "#AABBCCDD") extra = {"foo": "get_classifications"} - response = await vision.get_classifications(image, 1, extra=extra) + response = await vision.get_classifications(IMAGE, 1, extra=extra) assert response == CLASSIFICATIONS assert vision.extra == extra @@ -179,11 +179,10 @@ async def test_get_detections_from_camera(self, vision: MockVision, service: Vis async def test_get_detections(self, vision: MockVision, service: VisionRPCService): async with ChannelFor([service]) as channel: client = VisionServiceStub(channel) - image = Image.new("RGB", (100, 100), "#AABBCCDD") extra = {"foo": "get_detections"} request = GetDetectionsRequest( name=vision.name, - image=CameraMimeType.JPEG.encode_image(image), + image=IMAGE.data, width=100, height=100, mime_type=CameraMimeType.JPEG, @@ -207,11 +206,10 @@ async def test_get_classifications_from_camera(self, vision: MockVision, service async def test_get_classifications(self, vision: MockVision, service: VisionRPCService): async with ChannelFor([service]) as channel: client = VisionServiceStub(channel) - image = Image.new("RGB", (100, 100), "#AABBCCDD") extra = {"foo": "get_classifications"} request = GetClassificationsRequest( name=vision.name, - image=CameraMimeType.JPEG.encode_image(image), + image=IMAGE.data, width=100, height=100, mime_type=CameraMimeType.JPEG, @@ -261,9 +259,8 @@ async def test_get_detections_from_camera(self, vision: MockVision, service: Vis async def test_get_detections(self, vision: MockVision, service: VisionRPCService): async with ChannelFor([service]) as channel: client = VisionClient(VISION_SERVICE_NAME, channel) - image = Image.new("RGB", (100, 100), "#AABBCCDD") extra = {"foo": "get_detections"} - response = await client.get_detections(image, extra=extra) + response = await client.get_detections(IMAGE, extra=extra) assert response == DETECTIONS assert vision.extra == extra @@ -280,9 +277,8 @@ async def test_get_classifications_from_camera(self, vision: MockVision, service async def test_get_classifications(self, vision: MockVision, service: VisionRPCService): async with ChannelFor([service]) as channel: client = VisionClient(VISION_SERVICE_NAME, channel) - image = Image.new("RGB", (100, 100), "#AABBCCDD") extra = {"foo": "get_classifications"} - response = await client.get_classifications(image, 1, extra=extra) + response = await client.get_classifications(IMAGE, 1, extra=extra) assert response == CLASSIFICATIONS assert vision.extra == extra From 485a85115d3a74ab0b9969c49679fb11c4677174 Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Tue, 26 Mar 2024 07:52:50 -0700 Subject: [PATCH 03/36] remove rawimage from camera files --- src/viam/components/camera/__init__.py | 3 +-- src/viam/components/camera/service.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/viam/components/camera/__init__.py b/src/viam/components/camera/__init__.py index 5d5bab667..c4c4ff8e8 100644 --- a/src/viam/components/camera/__init__.py +++ b/src/viam/components/camera/__init__.py @@ -1,4 +1,4 @@ -from viam.media.video import RawImage, ViamImage +from viam.media.video import ViamImage from viam.proto.component.camera import DistortionParameters, IntrinsicParameters from viam.resource.registry import Registry, ResourceRegistration @@ -10,7 +10,6 @@ "Camera", "IntrinsicParameters", "DistortionParameters", - "RawImage", "ViamImage", ] diff --git a/src/viam/components/camera/service.py b/src/viam/components/camera/service.py index 9fe5f3663..ac56978a5 100644 --- a/src/viam/components/camera/service.py +++ b/src/viam/components/camera/service.py @@ -23,7 +23,7 @@ from viam.resource.rpc_service_base import ResourceRPCServiceBase from viam.utils import dict_to_struct, struct_to_dict -from . import Camera, RawImage +from . import Camera, ViamImage class CameraRPCService(CameraServiceBase, ResourceRPCServiceBase[Camera]): @@ -95,7 +95,7 @@ async def RenderFrame(self, stream: Stream[RenderFrameRequest, HttpBody]) -> Non img = mimetype.encode_image(image) finally: image.close() - response = HttpBody(data=img, content_type=image.mime_type if isinstance(image, RawImage) else mimetype) # type: ignore + response = HttpBody(data=img, content_type=image.mime_type if isinstance(image, ViamImage) else mimetype) # type: ignore await stream.send_message(response) async def GetPointCloud(self, stream: Stream[GetPointCloudRequest, GetPointCloudResponse]) -> None: From a0fd40192f3c3eafd047c67d2e9c9a57d984681b Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Tue, 26 Mar 2024 13:04:10 -0700 Subject: [PATCH 04/36] use viamimage in camera_test --- src/viam/components/camera/service.py | 10 ++-------- tests/mocks/components.py | 14 +++++--------- tests/test_camera.py | 27 ++++++++++----------------- 3 files changed, 17 insertions(+), 34 deletions(-) diff --git a/src/viam/components/camera/service.py b/src/viam/components/camera/service.py index ac56978a5..2b0dd248a 100644 --- a/src/viam/components/camera/service.py +++ b/src/viam/components/camera/service.py @@ -54,11 +54,9 @@ async def GetImage(self, stream: Stream[GetImageRequest, GetImageResponse]) -> N response_mime = mimetype else: response_mime = request.mime_type - response = GetImageResponse(mime_type=response_mime) - img_bytes = mimetype.encode_image(image) + response = GetImageResponse(mime_type=response_mime, image=image.data) finally: image.close() - response.image = img_bytes await stream.send_message(response) async def GetImages(self, stream: Stream[GetImagesRequest, GetImagesResponse]) -> None: @@ -91,11 +89,7 @@ async def RenderFrame(self, stream: Stream[RenderFrameRequest, HttpBody]) -> Non mimetype = CameraMimeType.JPEG timeout = stream.deadline.time_remaining() if stream.deadline else None image = await camera.get_image(mimetype, timeout=timeout, metadata=stream.metadata) - try: - img = mimetype.encode_image(image) - finally: - image.close() - response = HttpBody(data=img, content_type=image.mime_type if isinstance(image, ViamImage) else mimetype) # type: ignore + response = HttpBody(data=image.data, content_type=image.mime_type if isinstance(image, ViamImage) else mimetype) # type: ignore await stream.send_message(response) async def GetPointCloud(self, stream: Stream[GetPointCloudRequest, GetPointCloudResponse]) -> None: diff --git a/tests/mocks/components.py b/tests/mocks/components.py index 482eff31e..f9a9e323f 100644 --- a/tests/mocks/components.py +++ b/tests/mocks/components.py @@ -32,7 +32,7 @@ from viam.errors import ResourceNotFoundError from viam.media import MediaStreamWithIterator from viam.media.audio import Audio, AudioStream -from viam.media.video import CameraMimeType, NamedImage, RawImage +from viam.media.video import CameraMimeType, NamedImage, ViamImage from viam.proto.common import ( AnalogStatus, BoardStatus, @@ -396,25 +396,21 @@ def __init__(self, name: str): async def get_image( self, mime_type: str = "", extra: Optional[Dict[str, Any]] = None, timeout: Optional[float] = None, **kwargs - ) -> Union[Image.Image, RawImage]: + ) -> ViamImage: self.extra = extra self.timeout = timeout mime_type, is_lazy = CameraMimeType.from_lazy(mime_type) if is_lazy or (not CameraMimeType.is_supported(mime_type)): - return RawImage( + return ViamImage( data=self.image.convert("RGBA").tobytes("raw", "RGBA"), mime_type=mime_type, ) - return self.image.copy() + return ViamImage(mime_type.encode_image(self.image), mime_type) async def get_images(self, timeout: Optional[float] = None, **kwargs) -> Tuple[List[NamedImage], ResponseMetadata]: self.timeout = timeout return [ - NamedImage( - name=self.name, - data=CameraMimeType.VIAM_RGBA.encode_image(self.image), - mime_type=CameraMimeType.VIAM_RGBA, - ) + NamedImage(name=self.name, data=CameraMimeType.VIAM_RGBA.encode_image(self.image), mime_type=CameraMimeType.VIAM_RGBA) ], self.metadata async def get_point_cloud( diff --git a/tests/test_camera.py b/tests/test_camera.py index 2d2c51888..097b3ead7 100644 --- a/tests/test_camera.py +++ b/tests/test_camera.py @@ -87,16 +87,15 @@ class TestCamera: @pytest.mark.asyncio async def test_get_image(self, camera: MockCamera, image: Image.Image): img = await camera.get_image(CameraMimeType.PNG) - assert img == image + assert img.data == CameraMimeType.PNG.encode_image(image) + assert img.mime_type == CameraMimeType.PNG img = await camera.get_image(CameraMimeType.PNG, {"1": 1}) assert camera.extra == {"1": 1} img = await camera.get_image(CameraMimeType.VIAM_RGBA) - assert isinstance(img, Image.Image) - - img = await camera.get_image(CameraMimeType.PNG.with_lazy_suffix) - assert isinstance(img, RawImage) + assert img.data == CameraMimeType.VIAM_RGBA.encode_image(image) + assert img.mime_type == CameraMimeType.VIAM_RGBA @pytest.mark.asyncio async def test_get_images(self, camera: Camera, image: Image.Image, metadata: ResponseMetadata): @@ -154,22 +153,19 @@ async def test_get_image(self, camera: MockCamera, service: CameraRPCService, im # Test known mime type request = GetImageRequest(name="camera", mime_type=CameraMimeType.PNG) response: GetImageResponse = await client.GetImage(request, timeout=18.2) - img = Image.open(BytesIO(response.image), formats=["PNG"]) - assert img.tobytes() == image.tobytes() + assert response.image == CameraMimeType.PNG.encode_image(image) assert camera.timeout == loose_approx(18.2) # Test raw mime type request = GetImageRequest(name="camera", mime_type=CameraMimeType.VIAM_RGBA) response: GetImageResponse = await client.GetImage(request) - img = Image.open(BytesIO(response.image), formats=["VIAM_RGBA"]) - assert img.tobytes() == image.tobytes() + assert response.image == CameraMimeType.VIAM_RGBA.encode_image(image) assert response.mime_type == CameraMimeType.VIAM_RGBA # Test unknown mime type request = GetImageRequest(name="camera", mime_type="unknown") response: GetImageResponse = await client.GetImage(request) - img = Image.frombytes("RGBA", (100, 100), response.image, "raw") - assert img == image + assert response.image == image.convert("RGBA").tobytes("raw", "RGBA") assert response.mime_type == "unknown" # Test empty mime type. Empty mime type should default to JPEG for non-depth cameras @@ -199,9 +195,7 @@ async def test_render_frame(self, camera: MockCamera, service: CameraRPCService, request = RenderFrameRequest(name="camera", mime_type=CameraMimeType.PNG) response: HttpBody = await client.RenderFrame(request, timeout=4.4) assert response.content_type == CameraMimeType.PNG - buf = BytesIO(response.data) - img = Image.open(buf, formats=LIBRARY_SUPPORTED_FORMATS) - assert img.tobytes() == image.tobytes() + assert response.data == CameraMimeType.PNG.encode_image(image) assert camera.timeout == loose_approx(4.4) @pytest.mark.asyncio @@ -246,15 +240,14 @@ async def test_get_geometries(self, camera: MockCamera, service: CameraRPCServic class TestClient: @pytest.mark.asyncio - async def test_get_image(self, camera: MockCamera, service: CameraRPCService, image: Image.Image): + async def test_get_image(self, camera: MockCamera, service: CameraRPCService, image: ViamImage): assert camera.timeout is None async with ChannelFor([service]) as channel: client = CameraClient("camera", channel) # Test known mime type png_img = await client.get_image(timeout=1.82, mime_type=CameraMimeType.PNG) - assert isinstance(png_img.image, Image.Image) - assert png_img.image.tobytes() == image.tobytes() + assert png_img.data == CameraMimeType.PNG.encode_image(image) assert isinstance(png_img.image, Image.Image) assert png_img.image.tobytes() == image.tobytes() assert camera.timeout == loose_approx(1.82) From 414fae9225e8e02268f8e765da55f31c60e8376c Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Wed, 27 Mar 2024 08:28:54 -0700 Subject: [PATCH 05/36] fix imports in camera client test --- tests/test_camera.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_camera.py b/tests/test_camera.py index 097b3ead7..275e04320 100644 --- a/tests/test_camera.py +++ b/tests/test_camera.py @@ -1,5 +1,4 @@ from datetime import datetime -from io import BytesIO import pytest from google.api.httpbody_pb2 import HttpBody @@ -10,7 +9,7 @@ from viam.components.camera import Camera, CameraClient from viam.components.camera.service import CameraRPCService from viam.components.generic.service import GenericRPCService -from viam.media.video import LIBRARY_SUPPORTED_FORMATS, CameraMimeType, NamedImage, RawImage, ViamImage +from viam.media.video import CameraMimeType, NamedImage, ViamImage from viam.proto.common import DoCommandRequest, DoCommandResponse, GetGeometriesRequest, GetGeometriesResponse, ResponseMetadata from viam.proto.component.camera import ( CameraServiceStub, @@ -240,7 +239,7 @@ async def test_get_geometries(self, camera: MockCamera, service: CameraRPCServic class TestClient: @pytest.mark.asyncio - async def test_get_image(self, camera: MockCamera, service: CameraRPCService, image: ViamImage): + async def test_get_image(self, camera: MockCamera, service: CameraRPCService, image: Image.Image): assert camera.timeout is None async with ChannelFor([service]) as channel: client = CameraClient("camera", channel) From 34ebe4a1daf69c84937814554acb9001c240bb55 Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Wed, 27 Mar 2024 09:10:36 -0700 Subject: [PATCH 06/36] fix types and imports --- src/viam/services/vision/client.py | 8 ++++---- tests/mocks/components.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/viam/services/vision/client.py b/src/viam/services/vision/client.py index 9aba4fe25..febbccc04 100644 --- a/src/viam/services/vision/client.py +++ b/src/viam/services/vision/client.py @@ -61,7 +61,7 @@ async def get_detections( ) -> List[Detection]: if extra is None: extra = {} - image.mime_type = CameraMimeType.JPEG + mime_type = CameraMimeType.JPEG if image.image is None: raise UnidentifiedImageError @@ -71,7 +71,7 @@ async def get_detections( image=image.data, width=image.image.width, height=image.image.height, - mime_type=image.mime_type, + mime_type=mime_type, extra=dict_to_struct(extra), ) response: GetDetectionsResponse = await self.client.GetDetections(request, timeout=timeout) @@ -101,8 +101,8 @@ async def get_classifications( ) -> List[Classification]: if extra is None: extra = {} - image.mime_type = CameraMimeType.JPEG + mime_type = CameraMimeType.JPEG if image.image is None: raise UnidentifiedImageError else: @@ -111,7 +111,7 @@ async def get_classifications( image=image.data, width=image.image.width, height=image.image.height, - mime_type=image.mime_type, + mime_type=mime_type, n=count, extra=dict_to_struct(extra), ) diff --git a/tests/mocks/components.py b/tests/mocks/components.py index f9a9e323f..ebd9007d6 100644 --- a/tests/mocks/components.py +++ b/tests/mocks/components.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from datetime import datetime, timedelta from secrets import choice -from typing import Any, Dict, List, Mapping, Optional, Tuple, Union +from typing import Any, Dict, List, Mapping, Optional, Tuple from google.protobuf.timestamp_pb2 import Timestamp from PIL import Image From f7e355e74ec8d4c07e11463cd310ec5700fb398f Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Wed, 27 Mar 2024 09:56:48 -0700 Subject: [PATCH 07/36] add height and width properties to viamimage --- src/viam/media/video.py | 23 ++++++++++++++++++++--- src/viam/services/vision/client.py | 6 +++--- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/viam/media/video.py b/src/viam/media/video.py index a74db0cf9..0096c7962 100644 --- a/src/viam/media/video.py +++ b/src/viam/media/video.py @@ -159,6 +159,8 @@ class ViamImage: Provides the raw data and the mime type, as well as lazily loading and caching the PIL.Image representation. """ + height: Optional[int] + width: Optional[int] _data: bytes _mime_type: CameraMimeType _image: Optional[Image.Image] = None @@ -167,6 +169,7 @@ class ViamImage: def __init__(self, data: bytes, mime_type: CameraMimeType) -> None: self._data = data self._mime_type = mime_type + self._get_image_dimensions() @property def data(self) -> bytes: @@ -220,14 +223,28 @@ def bytes_to_depth_array(self) -> List[List[int]]: if self.mime_type != CameraMimeType.VIAM_RAW_DEPTH.value: raise NotSupportedError("Type must be `image/vnd.viam.dep` to use bytes_to_depth_array()") - width = int.from_bytes(self.data[8:16], "big") - height = int.from_bytes(self.data[16:24], "big") + self.width = int.from_bytes(self.data[8:16], "big") + self.height = int.from_bytes(self.data[16:24], "big") depth_arr = array("H", self.data[24:]) depth_arr.byteswap() - depth_arr_2d = [[depth_arr[row * width + col] for col in range(width)] for row in range(height)] + depth_arr_2d = [[depth_arr[row * self.width + col] for col in range(self.width)] for row in range(self.height)] return depth_arr_2d + def _get_image_dimensions(self) -> None: + """Get image dimensions from the data of an image that has the MIME type ``image/jpeg``, ``image/png``, or ``image/vnd.viam.rgba``.""" + + if self.mime_type not in [CameraMimeType.JPEG, CameraMimeType.PNG, CameraMimeType.VIAM_RGBA]: + return + + try: + image = Image.open(BytesIO(self.data), formats=LIBRARY_SUPPORTED_FORMATS) + except UnidentifiedImageError: + return + + self.width = image.width + self.height = image.height + class NamedImage(ViamImage): """An implementation of ViamImage that contains a name attribute.""" diff --git a/src/viam/services/vision/client.py b/src/viam/services/vision/client.py index febbccc04..b53f3db5d 100644 --- a/src/viam/services/vision/client.py +++ b/src/viam/services/vision/client.py @@ -63,14 +63,14 @@ async def get_detections( extra = {} mime_type = CameraMimeType.JPEG - if image.image is None: + if image.width is None or image.height is None: raise UnidentifiedImageError else: request = GetDetectionsRequest( name=self.name, image=image.data, - width=image.image.width, - height=image.image.height, + width=image.width, + height=image.height, mime_type=mime_type, extra=dict_to_struct(extra), ) From 3f5dfc2e99068be57fa33a57449d51061b0bf7db Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Wed, 27 Mar 2024 10:12:01 -0700 Subject: [PATCH 08/36] remove lazy concept from cameramimetype --- src/viam/components/camera/client.py | 2 +- src/viam/components/camera/service.py | 2 +- src/viam/media/video.py | 25 ++++--------------------- src/viam/services/vision/service.py | 4 ++-- tests/mocks/components.py | 4 ++-- tests/test_camera.py | 7 ------- 6 files changed, 10 insertions(+), 34 deletions(-) diff --git a/src/viam/components/camera/client.py b/src/viam/components/camera/client.py index 808d9242a..3112293e5 100644 --- a/src/viam/components/camera/client.py +++ b/src/viam/components/camera/client.py @@ -23,7 +23,7 @@ def get_image_from_response(data: bytes, response_mime_type: str, request_mime_type: str) -> ViamImage: if not request_mime_type: request_mime_type = response_mime_type - mime_type, _ = CameraMimeType.from_lazy(request_mime_type) + mime_type = CameraMimeType.from_string(request_mime_type) return ViamImage(data, mime_type) diff --git a/src/viam/components/camera/service.py b/src/viam/components/camera/service.py index 2b0dd248a..78220b372 100644 --- a/src/viam/components/camera/service.py +++ b/src/viam/components/camera/service.py @@ -49,7 +49,7 @@ async def GetImage(self, stream: Stream[GetImageRequest, GetImageResponse]) -> N request.mime_type = self._camera_mime_types[camera.name] - mimetype, _ = CameraMimeType.from_lazy(request.mime_type) + mimetype = CameraMimeType.from_string(request.mime_type) if CameraMimeType.is_supported(mimetype): response_mime = mimetype else: diff --git a/src/viam/media/video.py b/src/viam/media/video.py index 0096c7962..33a356e82 100644 --- a/src/viam/media/video.py +++ b/src/viam/media/video.py @@ -11,8 +11,6 @@ from .viam_rgba_plugin import RGBA_FORMAT_LABEL -LAZY_SUFFIX = "+lazy" - # Formats that are supported by PIL LIBRARY_SUPPORTED_FORMATS = ["JPEG", "PNG", RGBA_FORMAT_LABEL] @@ -71,19 +69,10 @@ class CameraMimeType(str, Enum): UNSUPPORTED = "unsupported" @classmethod - def from_lazy(cls, value: str) -> Tuple[Self, bool]: - is_lazy = False - mime_type = value - if value.endswith(LAZY_SUFFIX): - mime_type = value[: (len(value) - len(LAZY_SUFFIX))] - is_lazy = True - if not cls.is_supported(value) and not is_lazy: - mime_type = CameraMimeType.UNSUPPORTED - return (cls(mime_type), is_lazy) - - @property - def with_lazy_suffix(self) -> str: - return f"{self.value}{LAZY_SUFFIX}" + def from_string(cls, value: str) -> Self: + if not cls.is_supported(value): + return CameraMimeType.UNSUPPORTED + return cls(value) def encode_image(self, image: Union[Image.Image, RawImage]) -> bytes: if isinstance(image, RawImage): @@ -98,12 +87,6 @@ def encode_image(self, image: Union[Image.Image, RawImage]) -> bytes: else: raise ValueError(f"Cannot encode image to {self}") - @property - def _should_be_raw(self) -> bool: - return self in [CameraMimeType.UNSUPPORTED, CameraMimeType.PCD, CameraMimeType.VIAM_RAW_DEPTH] or not CameraMimeType.is_supported( - self - ) - @classmethod def is_supported(cls, mime_type: str) -> bool: """Check if the provided mime_type is supported. diff --git a/src/viam/services/vision/service.py b/src/viam/services/vision/service.py index cf15de0eb..f745d052b 100644 --- a/src/viam/services/vision/service.py +++ b/src/viam/services/vision/service.py @@ -45,7 +45,7 @@ async def GetDetections(self, stream: Stream[GetDetectionsRequest, GetDetections extra = struct_to_dict(request.extra) timeout = stream.deadline.time_remaining() if stream.deadline else None - mime_type, _ = CameraMimeType.from_lazy(request.mime_type) + mime_type = CameraMimeType.from_string(request.mime_type) image = ViamImage(request.image, mime_type) result = await vision.get_detections(image, extra=extra, timeout=timeout) @@ -71,7 +71,7 @@ async def GetClassifications(self, stream: Stream[GetClassificationsRequest, Get extra = struct_to_dict(request.extra) timeout = stream.deadline.time_remaining() if stream.deadline else None - mime_type, _ = CameraMimeType.from_lazy(request.mime_type) + mime_type = CameraMimeType.from_string(request.mime_type) image = ViamImage(request.image, mime_type) result = await vision.get_classifications(image, request.n, extra=extra, timeout=timeout) diff --git a/tests/mocks/components.py b/tests/mocks/components.py index ebd9007d6..01f1ec711 100644 --- a/tests/mocks/components.py +++ b/tests/mocks/components.py @@ -399,8 +399,8 @@ async def get_image( ) -> ViamImage: self.extra = extra self.timeout = timeout - mime_type, is_lazy = CameraMimeType.from_lazy(mime_type) - if is_lazy or (not CameraMimeType.is_supported(mime_type)): + mime_type = CameraMimeType.from_string(mime_type) + if not CameraMimeType.is_supported(mime_type): return ViamImage( data=self.image.convert("RGBA").tobytes("raw", "RGBA"), mime_type=mime_type, diff --git a/tests/test_camera.py b/tests/test_camera.py index 275e04320..d5af7cd6c 100644 --- a/tests/test_camera.py +++ b/tests/test_camera.py @@ -257,13 +257,6 @@ async def test_get_image(self, camera: MockCamera, service: CameraRPCService, im rgba_bytes = rgba_img.image.tobytes() assert rgba_bytes == image.copy().convert("RGBA").tobytes() - # Test lazy mime type - raw_img = await client.get_image(CameraMimeType.PNG.with_lazy_suffix) - assert isinstance(raw_img, ViamImage) - assert raw_img.image is None - assert raw_img.data == image.tobytes() - assert raw_img.mime_type == CameraMimeType.PNG - # Test unknown mime type raw_img = await client.get_image("unknown") assert isinstance(raw_img, ViamImage) From 2c98d45a4bb5150f972196d4b0a5cbc9f44b1745 Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Wed, 27 Mar 2024 10:18:21 -0700 Subject: [PATCH 09/36] start removing rawimage --- tests/test_media.py | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/tests/test_media.py b/tests/test_media.py index 38158bc17..ab25a8efb 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -4,24 +4,7 @@ from PIL import Image -from viam.media.video import CameraMimeType, NamedImage, RawImage, ViamImage - - -class TestRawImage: - def test_bytes_to_depth_array(self): - with open(f"{os.path.dirname(__file__)}/../data/fakeDM.vnd.viam.dep", "rb") as depth_map: - image = RawImage(depth_map.read(), "image/vnd.viam.dep") - assert isinstance(image, RawImage) - standard_data = image.bytes_to_depth_array() - assert len(standard_data) == 10 - assert len(standard_data[0]) == 20 - data_arr = array("H", image.data[24:]) - data_arr.byteswap() - assert len(data_arr) == 200 - assert standard_data[0][0] == data_arr[0] - assert standard_data[-1][3] == data_arr[183] - assert standard_data[-1][3] == 9 * 3 - assert standard_data[4][4] == 4 * 4 +from viam.media.video import CameraMimeType, NamedImage, ViamImage class TestViamImage: @@ -70,8 +53,8 @@ def test_mime_type_update(self): def test_bytes_to_depth_array(self): with open(f"{os.path.dirname(__file__)}/../data/fakeDM.vnd.viam.dep", "rb") as depth_map: - image = RawImage(depth_map.read(), "image/vnd.viam.dep") - assert isinstance(image, RawImage) + image = ViamImage(depth_map.read(), "image/vnd.viam.dep") + assert isinstance(image, ViamImage) standard_data = image.bytes_to_depth_array() assert len(standard_data) == 10 assert len(standard_data[0]) == 20 From 2ce9dc606b2b172f57881034454f379323ce80c1 Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Wed, 27 Mar 2024 15:42:07 -0700 Subject: [PATCH 10/36] add width and height to viamimage --- src/viam/media/video.py | 24 ++++++++++++++++++++---- src/viam/services/vision/client.py | 21 ++++++++++----------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/src/viam/media/video.py b/src/viam/media/video.py index 33a356e82..2f6b4312c 100644 --- a/src/viam/media/video.py +++ b/src/viam/media/video.py @@ -142,8 +142,8 @@ class ViamImage: Provides the raw data and the mime type, as well as lazily loading and caching the PIL.Image representation. """ - height: Optional[int] - width: Optional[int] + _height: Optional[int] + _width: Optional[int] _data: bytes _mime_type: CameraMimeType _image: Optional[Image.Image] = None @@ -173,6 +173,22 @@ def mime_type(self, value: CameraMimeType): self._image_decoded = False self._image = None + @property + def width(self) -> Union[int, None]: + return self._width + + @width.setter + def width(self, width: int): + self._width = width + + @property + def height(self) -> Union[int, None]: + return self._height + + @height.setter + def height(self, height: int): + self._height = height + @property def image(self) -> Optional[Image.Image]: """The PIL.Image representation of the image. If the mime type is not supported, this will be None.""" @@ -225,8 +241,8 @@ def _get_image_dimensions(self) -> None: except UnidentifiedImageError: return - self.width = image.width - self.height = image.height + self._width = image.width + self._height = image.height class NamedImage(ViamImage): diff --git a/src/viam/services/vision/client.py b/src/viam/services/vision/client.py index b53f3db5d..54eb27595 100644 --- a/src/viam/services/vision/client.py +++ b/src/viam/services/vision/client.py @@ -103,18 +103,17 @@ async def get_classifications( extra = {} mime_type = CameraMimeType.JPEG - if image.image is None: + if image.width is None or image.height is None: raise UnidentifiedImageError - else: - request = GetClassificationsRequest( - name=self.name, - image=image.data, - width=image.image.width, - height=image.image.height, - mime_type=mime_type, - n=count, - extra=dict_to_struct(extra), - ) + request = GetClassificationsRequest( + name=self.name, + image=image.data, + width=image.width, + height=image.height, + mime_type=mime_type, + n=count, + extra=dict_to_struct(extra), + ) response: GetClassificationsResponse = await self.client.GetClassifications(request, timeout=timeout) return list(response.classifications) From bdb2713b6777b97d04282b39f72916b043640a02 Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Thu, 28 Mar 2024 09:00:51 -0700 Subject: [PATCH 11/36] change vision service test --- src/viam/media/video.py | 8 ++++++-- tests/test_vision_service.py | 6 +++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/viam/media/video.py b/src/viam/media/video.py index 2f6b4312c..658787390 100644 --- a/src/viam/media/video.py +++ b/src/viam/media/video.py @@ -1,7 +1,7 @@ from array import array from enum import Enum from io import BytesIO -from typing import List, NamedTuple, Optional, Tuple, Union +from typing import List, NamedTuple, Optional, Union from PIL import Image, UnidentifiedImageError from typing_extensions import Self @@ -175,6 +175,7 @@ def mime_type(self, value: CameraMimeType): @property def width(self) -> Union[int, None]: + """The width of the image""" return self._width @width.setter @@ -183,6 +184,7 @@ def width(self, width: int): @property def height(self) -> Union[int, None]: + """The height of the image""" return self._height @height.setter @@ -231,7 +233,9 @@ def bytes_to_depth_array(self) -> List[List[int]]: return depth_arr_2d def _get_image_dimensions(self) -> None: - """Get image dimensions from the data of an image that has the MIME type ``image/jpeg``, ``image/png``, or ``image/vnd.viam.rgba``.""" + """ + Get image dimensions from the data of an image that has the MIME type ``image/jpeg``, ``image/png``, or ``image/vnd.viam.rgba``. + """ if self.mime_type not in [CameraMimeType.JPEG, CameraMimeType.PNG, CameraMimeType.VIAM_RGBA]: return diff --git a/tests/test_vision_service.py b/tests/test_vision_service.py index 8b8994777..d59e440d6 100644 --- a/tests/test_vision_service.py +++ b/tests/test_vision_service.py @@ -2,7 +2,6 @@ import pytest from grpclib.testing import ChannelFor -from PIL import Image from viam.media.video import CameraMimeType, ViamImage from viam.proto.common import ( @@ -35,8 +34,9 @@ from .mocks.services import MockVision -PILIMAGE = Image.new("RGB", (100, 100), "#AABBCCDD") -IMAGE = ViamImage(CameraMimeType.JPEG.encode_image(PILIMAGE), CameraMimeType.JPEG) +IMAGE = ViamImage(b"data", CameraMimeType.JPEG) +IMAGE.width = 2 +IMAGE.height = 4 DETECTORS = [ "detector-0", "detector-1", From f885a41e25056a7e54f0d4712c152f45f0637660 Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Thu, 28 Mar 2024 11:10:31 -0700 Subject: [PATCH 12/36] remove encode_image --- src/viam/media/video.py | 15 +-------- tests/mocks/components.py | 15 ++------- tests/test_camera.py | 70 +++++++++++---------------------------- 3 files changed, 23 insertions(+), 77 deletions(-) diff --git a/src/viam/media/video.py b/src/viam/media/video.py index 658787390..9048612e0 100644 --- a/src/viam/media/video.py +++ b/src/viam/media/video.py @@ -71,22 +71,9 @@ class CameraMimeType(str, Enum): @classmethod def from_string(cls, value: str) -> Self: if not cls.is_supported(value): - return CameraMimeType.UNSUPPORTED + return cls(CameraMimeType.UNSUPPORTED) return cls(value) - def encode_image(self, image: Union[Image.Image, RawImage]) -> bytes: - if isinstance(image, RawImage): - return image.data - - if self.name in LIBRARY_SUPPORTED_FORMATS: - buf = BytesIO() - if image.mode == "RGBA" and self.name == "JPEG": - image = image.convert("RGB") - image.save(buf, format=self.name) - return buf.getvalue() - else: - raise ValueError(f"Cannot encode image to {self}") - @classmethod def is_supported(cls, mime_type: str) -> bool: """Check if the provided mime_type is supported. diff --git a/tests/mocks/components.py b/tests/mocks/components.py index 01f1ec711..7a928614e 100644 --- a/tests/mocks/components.py +++ b/tests/mocks/components.py @@ -11,7 +11,6 @@ from typing import Any, Dict, List, Mapping, Optional, Tuple from google.protobuf.timestamp_pb2 import Timestamp -from PIL import Image from viam.components.arm import Arm, JointPositions, KinematicsFileFormat from viam.components.audio_input import AudioInput @@ -379,7 +378,7 @@ async def write_analog(self, pin: str, value: int, *, timeout: Optional[float] = class MockCamera(Camera): def __init__(self, name: str): - self.image = Image.new("RGBA", (100, 100), "#AABBCCDD") + self.image = ViamImage(b"data", CameraMimeType.PNG) self.geometries = GEOMETRIES self.point_cloud = b"THIS IS A POINT CLOUD" self.extra = None @@ -399,19 +398,11 @@ async def get_image( ) -> ViamImage: self.extra = extra self.timeout = timeout - mime_type = CameraMimeType.from_string(mime_type) - if not CameraMimeType.is_supported(mime_type): - return ViamImage( - data=self.image.convert("RGBA").tobytes("raw", "RGBA"), - mime_type=mime_type, - ) - return ViamImage(mime_type.encode_image(self.image), mime_type) + return self.image async def get_images(self, timeout: Optional[float] = None, **kwargs) -> Tuple[List[NamedImage], ResponseMetadata]: self.timeout = timeout - return [ - NamedImage(name=self.name, data=CameraMimeType.VIAM_RGBA.encode_image(self.image), mime_type=CameraMimeType.VIAM_RGBA) - ], self.metadata + return [NamedImage(self.name, self.image.data, self.image.mime_type)], self.metadata async def get_point_cloud( self, *, extra: Optional[Dict[str, Any]] = None, timeout: Optional[float] = None, **kwargs diff --git a/tests/test_camera.py b/tests/test_camera.py index d5af7cd6c..5f6a9210b 100644 --- a/tests/test_camera.py +++ b/tests/test_camera.py @@ -4,7 +4,6 @@ from google.api.httpbody_pb2 import HttpBody from google.protobuf.timestamp_pb2 import Timestamp from grpclib.testing import ChannelFor -from PIL import Image from viam.components.camera import Camera, CameraClient from viam.components.camera.service import CameraRPCService @@ -40,8 +39,8 @@ @pytest.fixture(scope="function") -def image() -> Image.Image: - return Image.new("RGBA", (100, 100), "#AABBCCDD") +def image() -> ViamImage: + return ViamImage(b"data", CameraMimeType.PNG) @pytest.fixture(scope="function") @@ -84,24 +83,20 @@ def generic_service(camera: Camera) -> GenericRPCService: class TestCamera: @pytest.mark.asyncio - async def test_get_image(self, camera: MockCamera, image: Image.Image): + async def test_get_image(self, camera: MockCamera, image: ViamImage): img = await camera.get_image(CameraMimeType.PNG) - assert img.data == CameraMimeType.PNG.encode_image(image) - assert img.mime_type == CameraMimeType.PNG + assert img.data == image.data + assert img.mime_type == image.mime_type img = await camera.get_image(CameraMimeType.PNG, {"1": 1}) assert camera.extra == {"1": 1} - img = await camera.get_image(CameraMimeType.VIAM_RGBA) - assert img.data == CameraMimeType.VIAM_RGBA.encode_image(image) - assert img.mime_type == CameraMimeType.VIAM_RGBA - @pytest.mark.asyncio - async def test_get_images(self, camera: Camera, image: Image.Image, metadata: ResponseMetadata): + async def test_get_images(self, camera: Camera, image: ViamImage, metadata: ResponseMetadata): imgs, md = await camera.get_images() assert isinstance(imgs[0], NamedImage) assert imgs[0].name == camera.name - assert imgs[0].data == CameraMimeType.VIAM_RGBA.encode_image(image) + assert imgs[0].data == image.data assert md == metadata @pytest.mark.asyncio @@ -144,7 +139,7 @@ async def test_get_geometries(self, camera: MockCamera): class TestService: @pytest.mark.asyncio - async def test_get_image(self, camera: MockCamera, service: CameraRPCService, image: Image.Image): + async def test_get_image(self, camera: MockCamera, service: CameraRPCService, image: ViamImage): assert camera.timeout is None async with ChannelFor([service]) as channel: client = CameraServiceStub(channel) @@ -152,24 +147,13 @@ async def test_get_image(self, camera: MockCamera, service: CameraRPCService, im # Test known mime type request = GetImageRequest(name="camera", mime_type=CameraMimeType.PNG) response: GetImageResponse = await client.GetImage(request, timeout=18.2) - assert response.image == CameraMimeType.PNG.encode_image(image) + assert response.image == image.data assert camera.timeout == loose_approx(18.2) - # Test raw mime type - request = GetImageRequest(name="camera", mime_type=CameraMimeType.VIAM_RGBA) - response: GetImageResponse = await client.GetImage(request) - assert response.image == CameraMimeType.VIAM_RGBA.encode_image(image) - assert response.mime_type == CameraMimeType.VIAM_RGBA - - # Test unknown mime type - request = GetImageRequest(name="camera", mime_type="unknown") - response: GetImageResponse = await client.GetImage(request) - assert response.image == image.convert("RGBA").tobytes("raw", "RGBA") - assert response.mime_type == "unknown" - # Test empty mime type. Empty mime type should default to JPEG for non-depth cameras request = GetImageRequest(name="camera") response: GetImageResponse = await client.GetImage(request) + assert response.image == image.data assert service._camera_mime_types["camera"] == CameraMimeType.JPEG @pytest.mark.asyncio @@ -181,20 +165,20 @@ async def test_get_images(self, camera: MockCamera, service: CameraRPCService, m request = GetImagesRequest(name="camera") response: GetImagesResponse = await client.GetImages(request, timeout=18.2) raw_img = response.images[0] - assert raw_img.format == Format.FORMAT_RAW_RGBA + assert raw_img.format == Format.FORMAT_PNG assert raw_img.source_name == camera.name assert response.response_metadata == metadata assert camera.timeout == loose_approx(18.2) @pytest.mark.asyncio - async def test_render_frame(self, camera: MockCamera, service: CameraRPCService, image: Image.Image): + async def test_render_frame(self, camera: MockCamera, service: CameraRPCService, image: ViamImage): assert camera.timeout is None async with ChannelFor([service]) as channel: client = CameraServiceStub(channel) request = RenderFrameRequest(name="camera", mime_type=CameraMimeType.PNG) response: HttpBody = await client.RenderFrame(request, timeout=4.4) assert response.content_type == CameraMimeType.PNG - assert response.data == CameraMimeType.PNG.encode_image(image) + assert response.data == image.data assert camera.timeout == loose_approx(4.4) @pytest.mark.asyncio @@ -239,32 +223,17 @@ async def test_get_geometries(self, camera: MockCamera, service: CameraRPCServic class TestClient: @pytest.mark.asyncio - async def test_get_image(self, camera: MockCamera, service: CameraRPCService, image: Image.Image): + async def test_get_image(self, camera: MockCamera, service: CameraRPCService, image: ViamImage): assert camera.timeout is None async with ChannelFor([service]) as channel: client = CameraClient("camera", channel) - # Test known mime type - png_img = await client.get_image(timeout=1.82, mime_type=CameraMimeType.PNG) - assert png_img.data == CameraMimeType.PNG.encode_image(image) - assert isinstance(png_img.image, Image.Image) - assert png_img.image.tobytes() == image.tobytes() - assert camera.timeout == loose_approx(1.82) - - # Test raw mime type - rgba_img = await client.get_image(CameraMimeType.VIAM_RGBA) - assert isinstance(rgba_img.image, Image.Image) - rgba_bytes = rgba_img.image.tobytes() - assert rgba_bytes == image.copy().convert("RGBA").tobytes() - - # Test unknown mime type - raw_img = await client.get_image("unknown") - assert isinstance(raw_img, ViamImage) - assert raw_img.image is None - assert raw_img.mime_type == CameraMimeType.UNSUPPORTED + img = await client.get_image(timeout=1.82, mime_type=CameraMimeType.PNG) + assert img.data == image.data + assert img.mime_type == image.mime_type @pytest.mark.asyncio - async def test_get_images(self, camera: MockCamera, service: CameraRPCService, image: Image.Image, metadata: ResponseMetadata): + async def test_get_images(self, camera: MockCamera, service: CameraRPCService, image: ViamImage, metadata: ResponseMetadata): assert camera.timeout is None async with ChannelFor([service]) as channel: client = CameraClient("camera", channel) @@ -272,8 +241,7 @@ async def test_get_images(self, camera: MockCamera, service: CameraRPCService, i imgs, md = await client.get_images(timeout=1.82) assert isinstance(imgs[0], NamedImage) assert imgs[0].name == camera.name - assert imgs[0].image is not None - assert imgs[0].image.tobytes() == image.tobytes() + assert imgs[0].data == image.data assert md == metadata assert camera.timeout == loose_approx(1.82) From 27b669d48f07d7f657182afea02886a1f91a3269 Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Thu, 28 Mar 2024 11:44:24 -0700 Subject: [PATCH 13/36] save width and height in bytes_to_depth_array --- src/viam/media/video.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/viam/media/video.py b/src/viam/media/video.py index 9048612e0..2f50489c1 100644 --- a/src/viam/media/video.py +++ b/src/viam/media/video.py @@ -211,12 +211,14 @@ def bytes_to_depth_array(self) -> List[List[int]]: if self.mime_type != CameraMimeType.VIAM_RAW_DEPTH.value: raise NotSupportedError("Type must be `image/vnd.viam.dep` to use bytes_to_depth_array()") - self.width = int.from_bytes(self.data[8:16], "big") - self.height = int.from_bytes(self.data[16:24], "big") + width = int.from_bytes(self.data[8:16], "big") + height = int.from_bytes(self.data[16:24], "big") + self.width = width + self.height = height depth_arr = array("H", self.data[24:]) depth_arr.byteswap() - depth_arr_2d = [[depth_arr[row * self.width + col] for col in range(self.width)] for row in range(self.height)] + depth_arr_2d = [[depth_arr[row * width + col] for col in range(width)] for row in range(height)] return depth_arr_2d def _get_image_dimensions(self) -> None: From 66168f07d668d74ca93400e71445dae84e541bb8 Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Thu, 28 Mar 2024 12:00:24 -0700 Subject: [PATCH 14/36] remove is_supported --- src/viam/components/camera/service.py | 7 +------ src/viam/media/video.py | 21 --------------------- 2 files changed, 1 insertion(+), 27 deletions(-) diff --git a/src/viam/components/camera/service.py b/src/viam/components/camera/service.py index 78220b372..684480e38 100644 --- a/src/viam/components/camera/service.py +++ b/src/viam/components/camera/service.py @@ -49,12 +49,7 @@ async def GetImage(self, stream: Stream[GetImageRequest, GetImageResponse]) -> N request.mime_type = self._camera_mime_types[camera.name] - mimetype = CameraMimeType.from_string(request.mime_type) - if CameraMimeType.is_supported(mimetype): - response_mime = mimetype - else: - response_mime = request.mime_type - response = GetImageResponse(mime_type=response_mime, image=image.data) + response = GetImageResponse(mime_type=request.mime_type, image=image.data) finally: image.close() await stream.send_message(response) diff --git a/src/viam/media/video.py b/src/viam/media/video.py index 2f50489c1..1944ea660 100644 --- a/src/viam/media/video.py +++ b/src/viam/media/video.py @@ -70,24 +70,8 @@ class CameraMimeType(str, Enum): @classmethod def from_string(cls, value: str) -> Self: - if not cls.is_supported(value): - return cls(CameraMimeType.UNSUPPORTED) return cls(value) - @classmethod - def is_supported(cls, mime_type: str) -> bool: - """Check if the provided mime_type is supported. - - Args: - mime_type (str): The mime_type to check - - Returns: - bool: Whether the mime_type is supported - """ - if mime_type == cls.UNSUPPORTED: - return False - return mime_type in set(item.value for item in cls) - @classmethod def from_proto(cls, format: Format.ValueType) -> "CameraMimeType": """Returns the mimetype from a proto enum. @@ -181,11 +165,6 @@ def height(self, height: int): @property def image(self) -> Optional[Image.Image]: """The PIL.Image representation of the image. If the mime type is not supported, this will be None.""" - if not CameraMimeType.is_supported(self.mime_type): - self._image = None - self._image_decoded = True - return self._image - try: self._image = Image.open(BytesIO(self.data), formats=LIBRARY_SUPPORTED_FORMATS) except UnidentifiedImageError: From b217f71ea7b01803ee3e8670774eb76abba349e5 Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Fri, 29 Mar 2024 08:32:23 -0700 Subject: [PATCH 15/36] remove unsupported mime type --- src/viam/media/video.py | 5 +---- tests/test_media.py | 11 +---------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/src/viam/media/video.py b/src/viam/media/video.py index 1944ea660..4f8b99308 100644 --- a/src/viam/media/video.py +++ b/src/viam/media/video.py @@ -66,7 +66,6 @@ class CameraMimeType(str, Enum): JPEG = "image/jpeg" PNG = "image/png" PCD = "pointcloud/pcd" - UNSUPPORTED = "unsupported" @classmethod def from_string(cls, value: str) -> Self: @@ -87,9 +86,8 @@ def from_proto(cls, format: Format.ValueType) -> "CameraMimeType": Format.FORMAT_RAW_DEPTH: CameraMimeType.VIAM_RAW_DEPTH, Format.FORMAT_JPEG: CameraMimeType.JPEG, Format.FORMAT_PNG: CameraMimeType.PNG, - Format.FORMAT_UNSPECIFIED: CameraMimeType.UNSUPPORTED, } - return mimetypes.get(format, CameraMimeType.UNSUPPORTED) + return mimetypes.get(format, CameraMimeType.JPEG) def to_proto(self) -> Format.ValueType: """Returns the mimetype in a proto enum. @@ -102,7 +100,6 @@ def to_proto(self) -> Format.ValueType: self.VIAM_RAW_DEPTH: Format.FORMAT_RAW_DEPTH, self.JPEG: Format.FORMAT_JPEG, self.PNG: Format.FORMAT_PNG, - self.UNSUPPORTED: Format.FORMAT_UNSPECIFIED, } return formats.get(self, Format.FORMAT_UNSPECIFIED) diff --git a/tests/test_media.py b/tests/test_media.py index ab25a8efb..b4e4afd26 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -8,15 +8,6 @@ class TestViamImage: - def test_unsupported_image(self): - img = ViamImage(b"123", CameraMimeType.UNSUPPORTED) - assert img._image is None - assert img._image_decoded is False - assert img.image is None - assert img._image is None - assert img._image_decoded is True - img.close() - def test_supported_image(self): i = Image.new("RGBA", (100, 100), "#AABBCCDD") b = BytesIO() @@ -70,5 +61,5 @@ def test_bytes_to_depth_array(self): class TestNamedImage: def test_name(self): name = "img" - img = NamedImage(name, b"123", CameraMimeType.UNSUPPORTED) + img = NamedImage(name, b"123", CameraMimeType.JPEG) assert img.name == name From 9eb29f0ba43cf3b77d5611be1da5353b37fa6190 Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Fri, 29 Mar 2024 10:36:12 -0700 Subject: [PATCH 16/36] update server example --- examples/server/v1/components.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/examples/server/v1/components.py b/examples/server/v1/components.py index d6077a445..e845ee400 100644 --- a/examples/server/v1/components.py +++ b/examples/server/v1/components.py @@ -10,6 +10,7 @@ from typing import AsyncIterator from datetime import timedelta +from io import BytesIO from multiprocessing import Lock from pathlib import Path from typing import Any, Dict, List, Mapping, Optional, Tuple @@ -33,7 +34,7 @@ from viam.errors import ResourceNotFoundError from viam.media import MediaStreamWithIterator from viam.media.audio import Audio, AudioStream -from viam.media.video import NamedImage +from viam.media.video import CameraMimeType, NamedImage, ViamImage from viam.operations import run_with_operation from viam.proto.common import ( AnalogStatus, @@ -330,14 +331,17 @@ async def get_geometries(self, extra: Optional[Dict[str, Any]] = None, **kwargs) class ExampleCamera(Camera): def __init__(self, name: str): p = Path(__file__) - self.image = Image.open(p.parent.absolute().joinpath("viam.webp")) + img = Image.open(p.parent.absolute().joinpath("viam.webp")) + buf = BytesIO() + img.copy().save(buf, format="JPEG") + self.image = ViamImage(buf.getvalue(), CameraMimeType.JPEG) super().__init__(name) def __del__(self): self.image.close() - async def get_image(self, mime_type: str = "", extra: Optional[Dict[str, Any]] = None, **kwargs) -> Image.Image: - return self.image.copy() + async def get_image(self, mime_type: str = "", extra: Optional[Dict[str, Any]] = None, **kwargs) -> ViamImage: + return self.image async def get_images(self, timeout: Optional[float] = None, **kwargs) -> Tuple[List[NamedImage], ResponseMetadata]: raise NotImplementedError() From 58e4c2a160464b02a062dcc4b945bd714850562c Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Mon, 1 Apr 2024 14:36:28 -0400 Subject: [PATCH 17/36] remove rawimage --- src/viam/media/video.py | 47 +---------------------------------------- 1 file changed, 1 insertion(+), 46 deletions(-) diff --git a/src/viam/media/video.py b/src/viam/media/video.py index 4f8b99308..4b3d9b55e 100644 --- a/src/viam/media/video.py +++ b/src/viam/media/video.py @@ -1,7 +1,7 @@ from array import array from enum import Enum from io import BytesIO -from typing import List, NamedTuple, Optional, Union +from typing import List, Optional, Union from PIL import Image, UnidentifiedImageError from typing_extensions import Self @@ -15,51 +15,6 @@ LIBRARY_SUPPORTED_FORMATS = ["JPEG", "PNG", RGBA_FORMAT_LABEL] -class RawImage(NamedTuple): - """**DEPRECATED** Use ``ViamImage`` instead - - A raw bytes representation of an image. - - A RawImage should be returned instead of a PIL Image instance under one of - the following conditions - - 1) The requested mime type has the LAZY_SUFFIX string appended to it - 2) The requested mime type is not supported for decoding/encoding by Viam's - Python SDK - """ - - data: bytes - """The raw data of the image""" - - mime_type: str - """The mimetype of the image""" - - def close(self): - """Close the image and release resources. For RawImage, this is a noop.""" - return - - def bytes_to_depth_array(self) -> List[List[int]]: - """Decode the data of an image that has the custom depth MIME type ``image/vnd.viam.dep`` into - a standard representation. - - Raises: - NotSupportedError: Raised if given an image that is not of MIME type `image/vnd.viam.dep`. - - Returns: - List[List[int]]: The standard representation of the image. - """ - if self.mime_type != CameraMimeType.VIAM_RAW_DEPTH.value: - raise NotSupportedError("Type must be `image/vnd.viam.dep` to use bytes_to_depth_array()") - - width = int.from_bytes(self.data[8:16], "big") - height = int.from_bytes(self.data[16:24], "big") - depth_arr = array("H", self.data[24:]) - depth_arr.byteswap() - - depth_arr_2d = [[depth_arr[row * width + col] for col in range(width)] for row in range(height)] - return depth_arr_2d - - class CameraMimeType(str, Enum): VIAM_RGBA = "image/vnd.viam.rgba" VIAM_RAW_DEPTH = "image/vnd.viam.dep" From 7578283cef669748074534ce28f31098b649237d Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Mon, 1 Apr 2024 15:26:40 -0400 Subject: [PATCH 18/36] add and use helper function --- docs/examples/example.ipynb | 4 +++- src/viam/media/utils.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 src/viam/media/utils.py diff --git a/docs/examples/example.ipynb b/docs/examples/example.ipynb index c94726455..0d9afeb91 100644 --- a/docs/examples/example.ipynb +++ b/docs/examples/example.ipynb @@ -161,11 +161,13 @@ "source": [ "from viam.components.camera import Camera\n", "from viam.media.video import CameraMimeType\n", + "from viam.media.utils import viam_to_pil_image\n", "\n", "robot = await connect_with_channel()\n", "camera = Camera.from_robot(robot, \"camera0\")\n", "image = await camera.get_image(CameraMimeType.JPEG)\n", - "image.image.save(\"foo.png\")\n", + "pil = viam_to_pil_image(image)\n", + "pil.save(\"foo.png\")\n", "\n", "# Don't forget to close the robot when you're done!\n", "await robot.close()" diff --git a/src/viam/media/utils.py b/src/viam/media/utils.py new file mode 100644 index 000000000..f87a0bdd5 --- /dev/null +++ b/src/viam/media/utils.py @@ -0,0 +1,14 @@ +from io import BytesIO +from PIL import Image + +from viam.media.video import ViamImage + +from .viam_rgba_plugin import RGBA_FORMAT_LABEL + + +LIBRARY_SUPPORTED_FORMATS = ["JPEG", "PNG", RGBA_FORMAT_LABEL] + + +def viam_to_pil_image(v_img: ViamImage) -> Image.Image: + image = Image.open(BytesIO(v_img.data), formats=LIBRARY_SUPPORTED_FORMATS) + return image From 66650309acf386b59136e5f5c8979d58201c5813 Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Tue, 2 Apr 2024 14:47:26 -0400 Subject: [PATCH 19/36] move helper functions to new util file --- src/viam/components/camera/camera.py | 5 +- src/viam/media/utils.py | 91 ++++++++++++++++++++++++++-- src/viam/media/video.py | 63 +------------------ tests/test_media.py | 61 +++++++++---------- 4 files changed, 119 insertions(+), 101 deletions(-) diff --git a/src/viam/components/camera/camera.py b/src/viam/components/camera/camera.py index 560e1eb20..524eb6531 100644 --- a/src/viam/components/camera/camera.py +++ b/src/viam/components/camera/camera.py @@ -41,10 +41,11 @@ async def get_image( """Get the next image from the camera as a ViamImage. Be sure to close the image when finished. - NOTE: If the mime type is ``image/vnd.viam.dep`` you can use :func:`viam.media.video.ViamImage.bytes_to_depth_array` + NOTE: If the mime type is ``image/vnd.viam.dep`` you can use :func:`viam.media.utils.bytes_to_depth_array` to convert the data to a standard representation. :: + from viam.media.utils import bytes_to_depth_array my_camera = Camera.from_robot(robot=robot, name="my_camera") @@ -53,7 +54,7 @@ async def get_image( # Convert "frame" to a standard 2D image representation. # Remove the 1st 3x8 bytes and reshape the raw bytes to List[List[Int]]. - standard_frame = frame.bytes_to_depth_array() + standard_frame = bytes_to_depth_array(frame) Args: mime_type (str): The desired mime type of the image. This does not guarantee output type diff --git a/src/viam/media/utils.py b/src/viam/media/utils.py index f87a0bdd5..71bdc9de2 100644 --- a/src/viam/media/utils.py +++ b/src/viam/media/utils.py @@ -1,14 +1,95 @@ +import struct +from array import array from io import BytesIO +from typing import List + from PIL import Image -from viam.media.video import ViamImage +from viam.errors import NotSupportedError, ViamError +from viam.media.video import CameraMimeType, ViamImage from .viam_rgba_plugin import RGBA_FORMAT_LABEL - +# Formats that are supported by PIL LIBRARY_SUPPORTED_FORMATS = ["JPEG", "PNG", RGBA_FORMAT_LABEL] -def viam_to_pil_image(v_img: ViamImage) -> Image.Image: - image = Image.open(BytesIO(v_img.data), formats=LIBRARY_SUPPORTED_FORMATS) - return image +def viam_to_pil_image(image: ViamImage) -> Image.Image: + return Image.open(BytesIO(image.data), formats=LIBRARY_SUPPORTED_FORMATS) + + +def bytes_to_depth_array(image: ViamImage) -> List[List[int]]: + """Decode the data of an image that has the custom depth MIME type ``image/vnd.viam.dep`` into + a standard representation. + + Raises: + NotSupportedError: Raised if given an image that is not of MIME type `image/vnd.viam.dep`. + + Returns: + List[List[int]]: The standard representation of the image. + """ + if image.mime_type != CameraMimeType.VIAM_RAW_DEPTH.value: + raise NotSupportedError("Type must be `image/vnd.viam.dep` to use bytes_to_depth_array()") + + width = int.from_bytes(image.data[8:16], "big") + height = int.from_bytes(image.data[16:24], "big") + image.width = width + image.height = height + depth_arr = array("H", image.data[24:]) + depth_arr.byteswap() + + depth_arr_2d = [[depth_arr[row * width + col] for col in range(width)] for row in range(height)] + return depth_arr_2d + + +def get_image_dimensions(image: ViamImage) -> None: + """ + Get image dimensions from the data of an image that has the MIME type ``image/jpeg``, ``image/png``, or ``image/vnd.viam.rgba``. + """ + data = str(image.data) + size = len(image.data) + if image.mime_type not in [CameraMimeType.JPEG, CameraMimeType.PNG, CameraMimeType.VIAM_RGBA]: + NotSupportedError("Type must be `image/jpeg`, `image/png`, or `image/vnd.viam.rgba` to use get_image_dimensions()") + + # See PNG 2. Edition spec (http://www.w3.org/TR/PNG/) + # Bytes 0-7 are below, 4-byte chunk length, then 'IHDR' + # and finally the 4-byte width, height + if (size >= 24) and data.startswith("\211PNG\r\n\032\n") and (data[12:16] == "IHDR"): + w, h = struct.unpack(">LL", image.data[16:24]) + image.width = int(w) + image.height = int(h) + return + # This is for an older PNG version. + elif (size >= 16) and data.startswith("\211PNG\r\n\032\n"): + # Check to see if we have the right content type + w, h = struct.unpack(">LL", image.data[8:16]) + image.width = int(w) + image.height = int(h) + return + # handle JPEGs + elif (size >= 2) and data.startswith("\377\330"): + jpeg = BytesIO(image.data) + jpeg.read(2) + b = jpeg.read(1) + try: + while b and ord(b) != 0xDA: + while ord(b) != 0xFF: + b = jpeg.read(1) + while ord(b) == 0xFF: + b = jpeg.read(1) + if ord(b) >= 0xC0 and ord(b) <= 0xC3: + jpeg.read(3) + h, w = struct.unpack(">HH", jpeg.read(4)) + break + else: + jpeg.read(int(struct.unpack(">H", jpeg.read(2))[0]) - 2) + b = jpeg.read(1) + image.width = int(w) + image.height = int(h) + return + except struct.error: + pass + except ValueError: + pass + + raise ViamError(f"Could not find image dimensions of image {image}") diff --git a/src/viam/media/video.py b/src/viam/media/video.py index 4b3d9b55e..b67be2eec 100644 --- a/src/viam/media/video.py +++ b/src/viam/media/video.py @@ -1,19 +1,10 @@ -from array import array from enum import Enum -from io import BytesIO -from typing import List, Optional, Union +from typing import Optional, Union -from PIL import Image, UnidentifiedImageError from typing_extensions import Self -from viam.errors import NotSupportedError from viam.proto.component.camera import Format -from .viam_rgba_plugin import RGBA_FORMAT_LABEL - -# Formats that are supported by PIL -LIBRARY_SUPPORTED_FORMATS = ["JPEG", "PNG", RGBA_FORMAT_LABEL] - class CameraMimeType(str, Enum): VIAM_RGBA = "image/vnd.viam.rgba" @@ -69,13 +60,10 @@ class ViamImage: _width: Optional[int] _data: bytes _mime_type: CameraMimeType - _image: Optional[Image.Image] = None - _image_decoded = False def __init__(self, data: bytes, mime_type: CameraMimeType) -> None: self._data = data self._mime_type = mime_type - self._get_image_dimensions() @property def data(self) -> bytes: @@ -114,60 +102,11 @@ def height(self) -> Union[int, None]: def height(self, height: int): self._height = height - @property - def image(self) -> Optional[Image.Image]: - """The PIL.Image representation of the image. If the mime type is not supported, this will be None.""" - try: - self._image = Image.open(BytesIO(self.data), formats=LIBRARY_SUPPORTED_FORMATS) - except UnidentifiedImageError: - self._image = None - self._image_decoded = True - return self._image - def close(self): """Close the image and release resources.""" if self._image is not None: self._image.close() - def bytes_to_depth_array(self) -> List[List[int]]: - """Decode the data of an image that has the custom depth MIME type ``image/vnd.viam.dep`` into - a standard representation. - - Raises: - NotSupportedError: Raised if given an image that is not of MIME type `image/vnd.viam.dep`. - - Returns: - List[List[int]]: The standard representation of the image. - """ - if self.mime_type != CameraMimeType.VIAM_RAW_DEPTH.value: - raise NotSupportedError("Type must be `image/vnd.viam.dep` to use bytes_to_depth_array()") - - width = int.from_bytes(self.data[8:16], "big") - height = int.from_bytes(self.data[16:24], "big") - self.width = width - self.height = height - depth_arr = array("H", self.data[24:]) - depth_arr.byteswap() - - depth_arr_2d = [[depth_arr[row * width + col] for col in range(width)] for row in range(height)] - return depth_arr_2d - - def _get_image_dimensions(self) -> None: - """ - Get image dimensions from the data of an image that has the MIME type ``image/jpeg``, ``image/png``, or ``image/vnd.viam.rgba``. - """ - - if self.mime_type not in [CameraMimeType.JPEG, CameraMimeType.PNG, CameraMimeType.VIAM_RGBA]: - return - - try: - image = Image.open(BytesIO(self.data), formats=LIBRARY_SUPPORTED_FORMATS) - except UnidentifiedImageError: - return - - self._width = image.width - self._height = image.height - class NamedImage(ViamImage): """An implementation of ViamImage that contains a name attribute.""" diff --git a/tests/test_media.py b/tests/test_media.py index b4e4afd26..4cb9cccb2 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -4,6 +4,7 @@ from PIL import Image +from viam.media.utils import bytes_to_depth_array, get_image_dimensions, viam_to_pil_image from viam.media.video import CameraMimeType, NamedImage, ViamImage @@ -13,53 +14,49 @@ def test_supported_image(self): b = BytesIO() i.save(b, "PNG") img = ViamImage(b.getvalue(), CameraMimeType.PNG) - assert img._image is None - assert img._image_decoded is False - assert img.image is not None - assert img.image.tobytes() == i.tobytes() - assert img._image_decoded is True - img.close() + assert img._mime_type == CameraMimeType.PNG + pil_img = viam_to_pil_image(img) + assert pil_img.tobytes() == i.tobytes() def test_mime_type_update(self): i = Image.new("RGBA", (100, 100), "#AABBCCDD") b = BytesIO() i.save(b, "PNG") img = ViamImage(b.getvalue(), CameraMimeType.PNG) - assert img._mime_type == CameraMimeType.PNG - assert img._image is None - assert img._image_decoded is False - assert img.image is not None - assert img._image is not None - assert img._image_decoded is True img.mime_type = CameraMimeType.JPEG assert img._mime_type == CameraMimeType.JPEG - assert img._image is None - assert img._image_decoded is False - assert img.image is not None - assert img._image is not None - assert img._image_decoded is True img.close() - def test_bytes_to_depth_array(self): - with open(f"{os.path.dirname(__file__)}/../data/fakeDM.vnd.viam.dep", "rb") as depth_map: - image = ViamImage(depth_map.read(), "image/vnd.viam.dep") - assert isinstance(image, ViamImage) - standard_data = image.bytes_to_depth_array() - assert len(standard_data) == 10 - assert len(standard_data[0]) == 20 - data_arr = array("H", image.data[24:]) - data_arr.byteswap() - assert len(data_arr) == 200 - assert standard_data[0][0] == data_arr[0] - assert standard_data[-1][3] == data_arr[183] - assert standard_data[-1][3] == 9 * 3 - assert standard_data[4][4] == 4 * 4 - class TestNamedImage: def test_name(self): name = "img" img = NamedImage(name, b"123", CameraMimeType.JPEG) assert img.name == name + + +def test_viam_to_pil_image(): + i = Image.new("RGBA", (100, 100), "#AABBCCDD") + b = BytesIO() + i.save(b, "PNG") + img = ViamImage(b.getvalue(), CameraMimeType.JPEG) + pil_img = viam_to_pil_image(img) + assert pil_img.tobytes() == i.tobytes() + + +def test_bytes_to_depth_array(): + with open(f"{os.path.dirname(__file__)}/../data/fakeDM.vnd.viam.dep", "rb") as depth_map: + image = ViamImage(depth_map.read(), CameraMimeType.VIAM_RAW_DEPTH) + assert isinstance(image, ViamImage) + standard_data = bytes_to_depth_array(image) + assert len(standard_data) == 10 + assert len(standard_data[0]) == 20 + data_arr = array("H", image.data[24:]) + data_arr.byteswap() + assert len(data_arr) == 200 + assert standard_data[0][0] == data_arr[0] + assert standard_data[-1][3] == data_arr[183] + assert standard_data[-1][3] == 9 * 3 + assert standard_data[4][4] == 4 * 4 From bd747dbe1e3ea96ceb48b010f6b5732a36ad47f7 Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Tue, 2 Apr 2024 16:21:11 -0400 Subject: [PATCH 20/36] test png image dimensions --- src/viam/components/camera/service.py | 22 ++++------- src/viam/media/utils.py | 57 ++++++++++++++------------- src/viam/media/video.py | 8 ---- tests/test_media.py | 11 +++++- 4 files changed, 48 insertions(+), 50 deletions(-) diff --git a/src/viam/components/camera/service.py b/src/viam/components/camera/service.py index 684480e38..292278b7d 100644 --- a/src/viam/components/camera/service.py +++ b/src/viam/components/camera/service.py @@ -42,16 +42,13 @@ async def GetImage(self, stream: Stream[GetImageRequest, GetImageResponse]) -> N timeout = stream.deadline.time_remaining() if stream.deadline else None image = await camera.get_image(request.mime_type, extra=struct_to_dict(request.extra), timeout=timeout, metadata=stream.metadata) - try: - if not request.mime_type: - if camera.name not in self._camera_mime_types: - self._camera_mime_types[camera.name] = CameraMimeType.JPEG + if not request.mime_type: + if camera.name not in self._camera_mime_types: + self._camera_mime_types[camera.name] = CameraMimeType.JPEG - request.mime_type = self._camera_mime_types[camera.name] + request.mime_type = self._camera_mime_types[camera.name] - response = GetImageResponse(mime_type=request.mime_type, image=image.data) - finally: - image.close() + response = GetImageResponse(mime_type=request.mime_type, image=image.data) await stream.send_message(response) async def GetImages(self, stream: Stream[GetImagesRequest, GetImagesResponse]) -> None: @@ -64,12 +61,9 @@ async def GetImages(self, stream: Stream[GetImagesRequest, GetImagesResponse]) - images, metadata = await camera.get_images(timeout=timeout, metadata=stream.metadata) img_bytes_lst = [] for img in images: - try: - fmt = img.mime_type.to_proto() - img_bytes = img.data - img_bytes_lst.append(Image(source_name=name, format=fmt, image=img_bytes)) - finally: - img.close() + fmt = img.mime_type.to_proto() + img_bytes = img.data + img_bytes_lst.append(Image(source_name=name, format=fmt, image=img_bytes)) response = GetImagesResponse(images=img_bytes_lst, response_metadata=metadata) await stream.send_message(response) diff --git a/src/viam/media/utils.py b/src/viam/media/utils.py index 71bdc9de2..7cb6b2a3e 100644 --- a/src/viam/media/utils.py +++ b/src/viam/media/utils.py @@ -54,42 +54,45 @@ def get_image_dimensions(image: ViamImage) -> None: # See PNG 2. Edition spec (http://www.w3.org/TR/PNG/) # Bytes 0-7 are below, 4-byte chunk length, then 'IHDR' # and finally the 4-byte width, height - if (size >= 24) and data.startswith("\211PNG\r\n\032\n") and (data[12:16] == "IHDR"): + if (size >= 24) and image.data[0:8] == b"\x89PNG\r\n\x1a\n" and (image.data[12:16] == b"IHDR"): + assert image.mime_type == CameraMimeType.PNG w, h = struct.unpack(">LL", image.data[16:24]) image.width = int(w) image.height = int(h) return # This is for an older PNG version. - elif (size >= 16) and data.startswith("\211PNG\r\n\032\n"): - # Check to see if we have the right content type + elif (size >= 16) and image.data[0:8] == b"\x89PNG\r\n\x1a\n": + assert image.mime_type == CameraMimeType.PNG w, h = struct.unpack(">LL", image.data[8:16]) image.width = int(w) image.height = int(h) return # handle JPEGs - elif (size >= 2) and data.startswith("\377\330"): - jpeg = BytesIO(image.data) - jpeg.read(2) - b = jpeg.read(1) - try: - while b and ord(b) != 0xDA: - while ord(b) != 0xFF: - b = jpeg.read(1) - while ord(b) == 0xFF: - b = jpeg.read(1) - if ord(b) >= 0xC0 and ord(b) <= 0xC3: - jpeg.read(3) - h, w = struct.unpack(">HH", jpeg.read(4)) - break - else: - jpeg.read(int(struct.unpack(">H", jpeg.read(2))[0]) - 2) - b = jpeg.read(1) - image.width = int(w) - image.height = int(h) - return - except struct.error: - pass - except ValueError: - pass + # elif (size >= 2) and data.startswith("\377\330"): + # assert image.mime_type == CameraMimeType.JPEG + # jpeg = BytesIO(image.data) + # jpeg.read(2) + # b = jpeg.read(1) + # try: + # while b and ord(b) != 0xDA: + # while ord(b) != 0xFF: + # b = jpeg.read(1) + # while ord(b) == 0xFF: + # b = jpeg.read(1) + # if ord(b) >= 0xC0 and ord(b) <= 0xC3: + # jpeg.read(3) + # h, w = struct.unpack(">HH", jpeg.read(4)) + # break + # else: + # jpeg.read(int(struct.unpack(">H", jpeg.read(2))[0]) - 2) + # b = jpeg.read(1) + # image.width = int(w) + # image.height = int(h) + # return + # except struct.error: + # pass + # except ValueError: + # pass + # elif image.mime_type = CAMERAMIMETYPE.VIAM_RGBA: raise ViamError(f"Could not find image dimensions of image {image}") diff --git a/src/viam/media/video.py b/src/viam/media/video.py index b67be2eec..d50991ff6 100644 --- a/src/viam/media/video.py +++ b/src/viam/media/video.py @@ -80,9 +80,6 @@ def mime_type(self, value: CameraMimeType): if value == self.mime_type: return self._mime_type = value - self.close() - self._image_decoded = False - self._image = None @property def width(self) -> Union[int, None]: @@ -102,11 +99,6 @@ def height(self) -> Union[int, None]: def height(self, height: int): self._height = height - def close(self): - """Close the image and release resources.""" - if self._image is not None: - self._image.close() - class NamedImage(ViamImage): """An implementation of ViamImage that contains a name attribute.""" diff --git a/tests/test_media.py b/tests/test_media.py index 4cb9cccb2..afebb6e05 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -27,7 +27,6 @@ def test_mime_type_update(self): img.mime_type = CameraMimeType.JPEG assert img._mime_type == CameraMimeType.JPEG - img.close() class TestNamedImage: @@ -60,3 +59,13 @@ def test_bytes_to_depth_array(): assert standard_data[-1][3] == data_arr[183] assert standard_data[-1][3] == 9 * 3 assert standard_data[4][4] == 4 * 4 + + +def test_get_image_dimensions(): + i = Image.new("RGBA", (100, 100), "#AABBCCDD") + b = BytesIO() + i.save(b, "PNG") + img = ViamImage(b.getvalue(), CameraMimeType.PNG) + get_image_dimensions(img) + assert img.width == 100 + assert img.height == 100 From ac57979150ce362d5c6b540bdacf0607d78792cd Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Wed, 3 Apr 2024 12:06:13 -0400 Subject: [PATCH 21/36] use pil image to get width and height --- src/viam/media/utils.py | 53 ++++------------------------------------- 1 file changed, 4 insertions(+), 49 deletions(-) diff --git a/src/viam/media/utils.py b/src/viam/media/utils.py index 7cb6b2a3e..2a09eb4da 100644 --- a/src/viam/media/utils.py +++ b/src/viam/media/utils.py @@ -1,11 +1,10 @@ -import struct from array import array from io import BytesIO from typing import List from PIL import Image -from viam.errors import NotSupportedError, ViamError +from viam.errors import NotSupportedError from viam.media.video import CameraMimeType, ViamImage from .viam_rgba_plugin import RGBA_FORMAT_LABEL @@ -46,53 +45,9 @@ def get_image_dimensions(image: ViamImage) -> None: """ Get image dimensions from the data of an image that has the MIME type ``image/jpeg``, ``image/png``, or ``image/vnd.viam.rgba``. """ - data = str(image.data) - size = len(image.data) if image.mime_type not in [CameraMimeType.JPEG, CameraMimeType.PNG, CameraMimeType.VIAM_RGBA]: NotSupportedError("Type must be `image/jpeg`, `image/png`, or `image/vnd.viam.rgba` to use get_image_dimensions()") - # See PNG 2. Edition spec (http://www.w3.org/TR/PNG/) - # Bytes 0-7 are below, 4-byte chunk length, then 'IHDR' - # and finally the 4-byte width, height - if (size >= 24) and image.data[0:8] == b"\x89PNG\r\n\x1a\n" and (image.data[12:16] == b"IHDR"): - assert image.mime_type == CameraMimeType.PNG - w, h = struct.unpack(">LL", image.data[16:24]) - image.width = int(w) - image.height = int(h) - return - # This is for an older PNG version. - elif (size >= 16) and image.data[0:8] == b"\x89PNG\r\n\x1a\n": - assert image.mime_type == CameraMimeType.PNG - w, h = struct.unpack(">LL", image.data[8:16]) - image.width = int(w) - image.height = int(h) - return - # handle JPEGs - # elif (size >= 2) and data.startswith("\377\330"): - # assert image.mime_type == CameraMimeType.JPEG - # jpeg = BytesIO(image.data) - # jpeg.read(2) - # b = jpeg.read(1) - # try: - # while b and ord(b) != 0xDA: - # while ord(b) != 0xFF: - # b = jpeg.read(1) - # while ord(b) == 0xFF: - # b = jpeg.read(1) - # if ord(b) >= 0xC0 and ord(b) <= 0xC3: - # jpeg.read(3) - # h, w = struct.unpack(">HH", jpeg.read(4)) - # break - # else: - # jpeg.read(int(struct.unpack(">H", jpeg.read(2))[0]) - 2) - # b = jpeg.read(1) - # image.width = int(w) - # image.height = int(h) - # return - # except struct.error: - # pass - # except ValueError: - # pass - # elif image.mime_type = CAMERAMIMETYPE.VIAM_RGBA: - - raise ViamError(f"Could not find image dimensions of image {image}") + pil_img = viam_to_pil_image(image) + image.width = pil_img.width + image.height = pil_img.height From d75de48b5bc99f0f649ed91fecbac90169267aa1 Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Wed, 3 Apr 2024 13:00:20 -0400 Subject: [PATCH 22/36] add width and height tests --- tests/test_media.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_media.py b/tests/test_media.py index afebb6e05..64fae91a9 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -1,4 +1,5 @@ import os +import pytest from array import array from io import BytesIO @@ -28,6 +29,17 @@ def test_mime_type_update(self): img.mime_type = CameraMimeType.JPEG assert img._mime_type == CameraMimeType.JPEG + def test_set_image_dimensions(self): + img = ViamImage(b"data", CameraMimeType.JPEG) + with pytest.raises(AttributeError): + img.width + with pytest.raises(AttributeError): + img.height + img.width = 100 + img.height = 100 + assert img.width == 100 + assert img.height == 100 + class TestNamedImage: def test_name(self): @@ -66,6 +78,10 @@ def test_get_image_dimensions(): b = BytesIO() i.save(b, "PNG") img = ViamImage(b.getvalue(), CameraMimeType.PNG) + with pytest.raises(AttributeError): + img.width + with pytest.raises(AttributeError): + img.height get_image_dimensions(img) assert img.width == 100 assert img.height == 100 From 94344ff6fc9d471c6c13379e817407ff8c99a92a Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Wed, 3 Apr 2024 14:32:54 -0400 Subject: [PATCH 23/36] add rgba test --- tests/test_media.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_media.py b/tests/test_media.py index 64fae91a9..41d2d53c7 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -1,8 +1,8 @@ import os -import pytest from array import array from io import BytesIO +import pytest from PIL import Image from viam.media.utils import bytes_to_depth_array, get_image_dimensions, viam_to_pil_image @@ -85,3 +85,15 @@ def test_get_image_dimensions(): get_image_dimensions(img) assert img.width == 100 assert img.height == 100 + + i2 = Image.new("RGBA", (100, 100), "#AABBCCDD") + b2 = BytesIO() + i2.save(b2, "VIAM_RGBA") + img2 = ViamImage(b.getvalue(), CameraMimeType.JPEG) + with pytest.raises(AttributeError): + img2.width + with pytest.raises(AttributeError): + img2.height + get_image_dimensions(img2) + assert img2.width == 100 + assert img2.height == 100 From a035aae2638efad4b20a14647af43c424425f6dc Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Wed, 3 Apr 2024 14:37:18 -0400 Subject: [PATCH 24/36] add jpeg test --- tests/test_media.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_media.py b/tests/test_media.py index 41d2d53c7..1520d0664 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -89,7 +89,7 @@ def test_get_image_dimensions(): i2 = Image.new("RGBA", (100, 100), "#AABBCCDD") b2 = BytesIO() i2.save(b2, "VIAM_RGBA") - img2 = ViamImage(b.getvalue(), CameraMimeType.JPEG) + img2 = ViamImage(b.getvalue(), CameraMimeType.VIAM_RGBA) with pytest.raises(AttributeError): img2.width with pytest.raises(AttributeError): @@ -97,3 +97,15 @@ def test_get_image_dimensions(): get_image_dimensions(img2) assert img2.width == 100 assert img2.height == 100 + + i3 = Image.new("RGB", (100, 100), "#AABBCCDD") + b3 = BytesIO() + i3.save(b3, "JPEG") + img3 = ViamImage(b.getvalue(), CameraMimeType.JPEG) + with pytest.raises(AttributeError): + img3.width + with pytest.raises(AttributeError): + img3.height + get_image_dimensions(img3) + assert img3.width == 100 + assert img3.height == 100 From 060929701e21da279362c99641a227dd2cf5daff Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Wed, 3 Apr 2024 15:05:39 -0400 Subject: [PATCH 25/36] add clarifying comments --- src/viam/media/utils.py | 2 ++ src/viam/services/vision/vision.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/src/viam/media/utils.py b/src/viam/media/utils.py index 2a09eb4da..6cc8d912e 100644 --- a/src/viam/media/utils.py +++ b/src/viam/media/utils.py @@ -44,6 +44,8 @@ def bytes_to_depth_array(image: ViamImage) -> List[List[int]]: def get_image_dimensions(image: ViamImage) -> None: """ Get image dimensions from the data of an image that has the MIME type ``image/jpeg``, ``image/png``, or ``image/vnd.viam.rgba``. + + Alternatively, image dimensions can be set manually as well (i.e.: image.width = 100) """ if image.mime_type not in [CameraMimeType.JPEG, CameraMimeType.PNG, CameraMimeType.VIAM_RGBA]: NotSupportedError("Type must be `image/jpeg`, `image/png`, or `image/vnd.viam.rgba` to use get_image_dimensions()") diff --git a/src/viam/services/vision/vision.py b/src/viam/services/vision/vision.py index 90e832cf2..f5ab90634 100644 --- a/src/viam/services/vision/vision.py +++ b/src/viam/services/vision/vision.py @@ -73,6 +73,9 @@ async def get_detections( # Get an image from the camera img = await cam1.get_image() + # Get image dimensions + get_image_dimensions(img) + # Get detections from that image detections = await my_detector.get_detections(img) @@ -139,6 +142,9 @@ async def get_classifications( # Get an image from the camera img = await cam1.get_image() + # Get image dimensions + get_image_dimensions(img) + # Get the 2 classifications with the highest confidence scores classifications = await my_classifier.get_classifications(img, 2) From 010e0cfccfcdc1eb49327fca1e46c9e9b557784c Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Thu, 4 Apr 2024 14:15:35 -0400 Subject: [PATCH 26/36] add more comments --- src/viam/media/utils.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/src/viam/media/utils.py b/src/viam/media/utils.py index 6cc8d912e..d9a931da1 100644 --- a/src/viam/media/utils.py +++ b/src/viam/media/utils.py @@ -14,12 +14,24 @@ def viam_to_pil_image(image: ViamImage) -> Image.Image: + """ + Convert a ViamImage to a PIL.Image. + + Args: + image (ViamImage): The image to convert. + + Returns: + Image.Image: The resulting PIL.Image + """ return Image.open(BytesIO(image.data), formats=LIBRARY_SUPPORTED_FORMATS) def bytes_to_depth_array(image: ViamImage) -> List[List[int]]: - """Decode the data of an image that has the custom depth MIME type ``image/vnd.viam.dep`` into - a standard representation. + """ + Decode the data of an image that has the custom depth MIME type ``image/vnd.viam.dep`` into a standard representation. + + Args: + image (ViamImage): The image to be decoded. Raises: NotSupportedError: Raised if given an image that is not of MIME type `image/vnd.viam.dep`. @@ -43,9 +55,16 @@ def bytes_to_depth_array(image: ViamImage) -> List[List[int]]: def get_image_dimensions(image: ViamImage) -> None: """ - Get image dimensions from the data of an image that has the MIME type ``image/jpeg``, ``image/png``, or ``image/vnd.viam.rgba``. + Get image dimensions from the data of an image that has the MIME type ``image/jpeg``, ``image/png``, or ``image/vnd.viam.rgba`` and set + the corresponding properties of the image. Alternatively, image dimensions can be set manually as well (i.e.: image.width = 100) + + Args: + image (ViamImage): The image to get dimensions of. + + Raises: + NotSupportedError: Raised if given an image that is not of MIME type `image/vnd.viam.dep`. """ if image.mime_type not in [CameraMimeType.JPEG, CameraMimeType.PNG, CameraMimeType.VIAM_RGBA]: NotSupportedError("Type must be `image/jpeg`, `image/png`, or `image/vnd.viam.rgba` to use get_image_dimensions()") From a8b9473166b344dcfc805a02be1d3ad4ddc08684 Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Thu, 4 Apr 2024 14:44:57 -0400 Subject: [PATCH 27/36] raise viamerror instead of pil error --- src/viam/media/utils.py | 1 + src/viam/services/vision/client.py | 6 +++--- src/viam/services/vision/vision.py | 6 ++++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/viam/media/utils.py b/src/viam/media/utils.py index d9a931da1..a7c26b63c 100644 --- a/src/viam/media/utils.py +++ b/src/viam/media/utils.py @@ -72,3 +72,4 @@ def get_image_dimensions(image: ViamImage) -> None: pil_img = viam_to_pil_image(image) image.width = pil_img.width image.height = pil_img.height + pil_img.close() diff --git a/src/viam/services/vision/client.py b/src/viam/services/vision/client.py index 54eb27595..5ed727559 100644 --- a/src/viam/services/vision/client.py +++ b/src/viam/services/vision/client.py @@ -1,8 +1,8 @@ from typing import Any, List, Mapping, Optional from grpclib.client import Channel -from PIL import UnidentifiedImageError +from viam.errors import ViamError from viam.media.video import CameraMimeType, ViamImage from viam.proto.common import DoCommandRequest, DoCommandResponse, PointCloudObject from viam.proto.service.vision import ( @@ -64,7 +64,7 @@ async def get_detections( mime_type = CameraMimeType.JPEG if image.width is None or image.height is None: - raise UnidentifiedImageError + raise ViamError(f"image {image} needs to have a specified width and height") else: request = GetDetectionsRequest( name=self.name, @@ -104,7 +104,7 @@ async def get_classifications( mime_type = CameraMimeType.JPEG if image.width is None or image.height is None: - raise UnidentifiedImageError + raise ViamError(f"image {image} needs to have a specified width and height") request = GetClassificationsRequest( name=self.name, image=image.data, diff --git a/src/viam/services/vision/vision.py b/src/viam/services/vision/vision.py index f5ab90634..9b451996f 100644 --- a/src/viam/services/vision/vision.py +++ b/src/viam/services/vision/vision.py @@ -45,6 +45,9 @@ async def get_detections_from_camera( Args: camera_name (str): The name of the camera to use for detection + Raises: + ViamError: Raised if given an image without a specified width and height + Returns: List[viam.proto.service.vision.Detection]: A list of 2D bounding boxes, their labels, and the confidence score of the labels, around the found objects in the next 2D image @@ -82,6 +85,9 @@ async def get_detections( Args: image (Image): The image to get detections from + Raises: + ViamError: Raised if given an image without a specified width and height + Returns: List[viam.proto.service.vision.Detection]: A list of 2D bounding boxes, their labels, and the confidence score of the labels, around the found objects in the next 2D image From 75c1698f9a7cc9470d60315a02d0bbce09ae284d Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Fri, 5 Apr 2024 16:18:30 -0400 Subject: [PATCH 28/36] rename image dimension function --- src/viam/media/utils.py | 4 ++-- src/viam/services/vision/vision.py | 4 ++-- tests/test_media.py | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/viam/media/utils.py b/src/viam/media/utils.py index a7c26b63c..77665bb31 100644 --- a/src/viam/media/utils.py +++ b/src/viam/media/utils.py @@ -53,7 +53,7 @@ def bytes_to_depth_array(image: ViamImage) -> List[List[int]]: return depth_arr_2d -def get_image_dimensions(image: ViamImage) -> None: +def determine_image_dimensions(image: ViamImage) -> None: """ Get image dimensions from the data of an image that has the MIME type ``image/jpeg``, ``image/png``, or ``image/vnd.viam.rgba`` and set the corresponding properties of the image. @@ -67,7 +67,7 @@ def get_image_dimensions(image: ViamImage) -> None: NotSupportedError: Raised if given an image that is not of MIME type `image/vnd.viam.dep`. """ if image.mime_type not in [CameraMimeType.JPEG, CameraMimeType.PNG, CameraMimeType.VIAM_RGBA]: - NotSupportedError("Type must be `image/jpeg`, `image/png`, or `image/vnd.viam.rgba` to use get_image_dimensions()") + NotSupportedError("Type must be `image/jpeg`, `image/png`, or `image/vnd.viam.rgba` to use determine_image_dimensions()") pil_img = viam_to_pil_image(image) image.width = pil_img.width diff --git a/src/viam/services/vision/vision.py b/src/viam/services/vision/vision.py index 9b451996f..8c5cbfcab 100644 --- a/src/viam/services/vision/vision.py +++ b/src/viam/services/vision/vision.py @@ -77,7 +77,7 @@ async def get_detections( img = await cam1.get_image() # Get image dimensions - get_image_dimensions(img) + determine_image_dimensions(img) # Get detections from that image detections = await my_detector.get_detections(img) @@ -149,7 +149,7 @@ async def get_classifications( img = await cam1.get_image() # Get image dimensions - get_image_dimensions(img) + determine_image_dimensions(img) # Get the 2 classifications with the highest confidence scores classifications = await my_classifier.get_classifications(img, 2) diff --git a/tests/test_media.py b/tests/test_media.py index 1520d0664..4ee656e55 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -5,7 +5,7 @@ import pytest from PIL import Image -from viam.media.utils import bytes_to_depth_array, get_image_dimensions, viam_to_pil_image +from viam.media.utils import bytes_to_depth_array, determine_image_dimensions, viam_to_pil_image from viam.media.video import CameraMimeType, NamedImage, ViamImage @@ -73,7 +73,7 @@ def test_bytes_to_depth_array(): assert standard_data[4][4] == 4 * 4 -def test_get_image_dimensions(): +def test_determine_image_dimensions(): i = Image.new("RGBA", (100, 100), "#AABBCCDD") b = BytesIO() i.save(b, "PNG") @@ -82,7 +82,7 @@ def test_get_image_dimensions(): img.width with pytest.raises(AttributeError): img.height - get_image_dimensions(img) + determine_image_dimensions(img) assert img.width == 100 assert img.height == 100 @@ -94,7 +94,7 @@ def test_get_image_dimensions(): img2.width with pytest.raises(AttributeError): img2.height - get_image_dimensions(img2) + determine_image_dimensions(img2) assert img2.width == 100 assert img2.height == 100 @@ -106,6 +106,6 @@ def test_get_image_dimensions(): img3.width with pytest.raises(AttributeError): img3.height - get_image_dimensions(img3) + determine_image_dimensions(img3) assert img3.width == 100 assert img3.height == 100 From 0b1f38aad98e857df94a0d82a8cefc81b3a2fc96 Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Fri, 5 Apr 2024 16:39:37 -0400 Subject: [PATCH 29/36] test not supported calls to util functions --- src/viam/media/utils.py | 2 +- tests/test_media.py | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/viam/media/utils.py b/src/viam/media/utils.py index 77665bb31..7774cc099 100644 --- a/src/viam/media/utils.py +++ b/src/viam/media/utils.py @@ -67,7 +67,7 @@ def determine_image_dimensions(image: ViamImage) -> None: NotSupportedError: Raised if given an image that is not of MIME type `image/vnd.viam.dep`. """ if image.mime_type not in [CameraMimeType.JPEG, CameraMimeType.PNG, CameraMimeType.VIAM_RGBA]: - NotSupportedError("Type must be `image/jpeg`, `image/png`, or `image/vnd.viam.rgba` to use determine_image_dimensions()") + raise NotSupportedError("Type must be `image/jpeg`, `image/png`, or `image/vnd.viam.rgba` to use determine_image_dimensions()") pil_img = viam_to_pil_image(image) image.width = pil_img.width diff --git a/tests/test_media.py b/tests/test_media.py index 4ee656e55..eaa20cf92 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -5,6 +5,7 @@ import pytest from PIL import Image +from viam.errors import NotSupportedError from viam.media.utils import bytes_to_depth_array, determine_image_dimensions, viam_to_pil_image from viam.media.video import CameraMimeType, NamedImage, ViamImage @@ -59,12 +60,12 @@ def test_viam_to_pil_image(): def test_bytes_to_depth_array(): with open(f"{os.path.dirname(__file__)}/../data/fakeDM.vnd.viam.dep", "rb") as depth_map: - image = ViamImage(depth_map.read(), CameraMimeType.VIAM_RAW_DEPTH) - assert isinstance(image, ViamImage) - standard_data = bytes_to_depth_array(image) + img = ViamImage(depth_map.read(), CameraMimeType.VIAM_RAW_DEPTH) + assert isinstance(img, ViamImage) + standard_data = bytes_to_depth_array(img) assert len(standard_data) == 10 assert len(standard_data[0]) == 20 - data_arr = array("H", image.data[24:]) + data_arr = array("H", img.data[24:]) data_arr.byteswap() assert len(data_arr) == 200 assert standard_data[0][0] == data_arr[0] @@ -72,6 +73,10 @@ def test_bytes_to_depth_array(): assert standard_data[-1][3] == 9 * 3 assert standard_data[4][4] == 4 * 4 + img2 = ViamImage(b"data", CameraMimeType.PCD) + with pytest.raises(NotSupportedError): + bytes_to_depth_array(img2) + def test_determine_image_dimensions(): i = Image.new("RGBA", (100, 100), "#AABBCCDD") @@ -109,3 +114,7 @@ def test_determine_image_dimensions(): determine_image_dimensions(img3) assert img3.width == 100 assert img3.height == 100 + + img4 = ViamImage(b"data", CameraMimeType.PCD) + with pytest.raises(NotSupportedError): + determine_image_dimensions(img4) From c894f8e90450a1b4100892220b1b61a9ac311872 Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Mon, 15 Apr 2024 13:09:58 -0400 Subject: [PATCH 30/36] add pr feedback --- src/viam/components/camera/client.py | 9 +------ src/viam/components/camera/service.py | 22 +++------------- src/viam/media/utils.py | 32 +++++++++++++++++----- src/viam/media/video.py | 8 +----- src/viam/services/vision/vision.py | 6 ----- tests/test_camera.py | 4 +-- tests/test_media.py | 38 +++++++++++++-------------- 7 files changed, 53 insertions(+), 66 deletions(-) diff --git a/src/viam/components/camera/client.py b/src/viam/components/camera/client.py index 3112293e5..9b2118690 100644 --- a/src/viam/components/camera/client.py +++ b/src/viam/components/camera/client.py @@ -20,13 +20,6 @@ from . import Camera -def get_image_from_response(data: bytes, response_mime_type: str, request_mime_type: str) -> ViamImage: - if not request_mime_type: - request_mime_type = response_mime_type - mime_type = CameraMimeType.from_string(request_mime_type) - return ViamImage(data, mime_type) - - class CameraClient(Camera, ReconfigurableResourceRPCClientBase): """ gRPC client for the Camera component @@ -49,7 +42,7 @@ async def get_image( extra = {} request = GetImageRequest(name=self.name, mime_type=mime_type, extra=dict_to_struct(extra)) response: GetImageResponse = await self.client.GetImage(request, timeout=timeout) - return get_image_from_response(response.image, response_mime_type=response.mime_type, request_mime_type=request.mime_type) + return ViamImage(response.image, CameraMimeType.from_string(response.mime_type)) async def get_images( self, diff --git a/src/viam/components/camera/service.py b/src/viam/components/camera/service.py index 292278b7d..19ccd3c3a 100644 --- a/src/viam/components/camera/service.py +++ b/src/viam/components/camera/service.py @@ -1,11 +1,8 @@ # TODO: Update type checking based with RSDK-4089 # pyright: reportGeneralTypeIssues=false -from typing import Dict - from google.api.httpbody_pb2 import HttpBody from grpclib.server import Stream -from viam.media.video import CameraMimeType from viam.proto.common import DoCommandRequest, DoCommandResponse, GetGeometriesRequest, GetGeometriesResponse from viam.proto.component.camera import ( CameraServiceBase, @@ -23,7 +20,7 @@ from viam.resource.rpc_service_base import ResourceRPCServiceBase from viam.utils import dict_to_struct, struct_to_dict -from . import Camera, ViamImage +from . import Camera class CameraRPCService(CameraServiceBase, ResourceRPCServiceBase[Camera]): @@ -32,7 +29,6 @@ class CameraRPCService(CameraServiceBase, ResourceRPCServiceBase[Camera]): """ RESOURCE_TYPE = Camera - _camera_mime_types: Dict[str, CameraMimeType] = {} async def GetImage(self, stream: Stream[GetImageRequest, GetImageResponse]) -> None: request = await stream.recv_message() @@ -42,13 +38,7 @@ async def GetImage(self, stream: Stream[GetImageRequest, GetImageResponse]) -> N timeout = stream.deadline.time_remaining() if stream.deadline else None image = await camera.get_image(request.mime_type, extra=struct_to_dict(request.extra), timeout=timeout, metadata=stream.metadata) - if not request.mime_type: - if camera.name not in self._camera_mime_types: - self._camera_mime_types[camera.name] = CameraMimeType.JPEG - - request.mime_type = self._camera_mime_types[camera.name] - - response = GetImageResponse(mime_type=request.mime_type, image=image.data) + response = GetImageResponse(mime_type=image.mime_type, image=image.data) await stream.send_message(response) async def GetImages(self, stream: Stream[GetImagesRequest, GetImagesResponse]) -> None: @@ -72,13 +62,9 @@ async def RenderFrame(self, stream: Stream[RenderFrameRequest, HttpBody]) -> Non assert request is not None name = request.name camera = self.get_resource(name) - try: - mimetype = CameraMimeType(request.mime_type) - except ValueError: - mimetype = CameraMimeType.JPEG timeout = stream.deadline.time_remaining() if stream.deadline else None - image = await camera.get_image(mimetype, timeout=timeout, metadata=stream.metadata) - response = HttpBody(data=image.data, content_type=image.mime_type if isinstance(image, ViamImage) else mimetype) # type: ignore + image = await camera.get_image(request.mime_type, timeout=timeout, metadata=stream.metadata) + response = HttpBody(data=image.data, content_type=image.mime_type) await stream.send_message(response) async def GetPointCloud(self, stream: Stream[GetPointCloudRequest, GetPointCloudResponse]) -> None: diff --git a/src/viam/media/utils.py b/src/viam/media/utils.py index 7774cc099..cb8539093 100644 --- a/src/viam/media/utils.py +++ b/src/viam/media/utils.py @@ -26,6 +26,29 @@ def viam_to_pil_image(image: ViamImage) -> Image.Image: return Image.open(BytesIO(image.data), formats=LIBRARY_SUPPORTED_FORMATS) +def pil_to_viam_image(image: Image.Image, mime_type: CameraMimeType) -> ViamImage: + """ + Convert a PIL.Image to a ViamImage. + + Args: + image (Image.Image): The image to convert. + mime_type (CameraMimeType): The mime type to convert the image to. + + Returns: + ViamImage: The resulting ViamImage + """ + if mime_type.name in LIBRARY_SUPPORTED_FORMATS: + buf = BytesIO() + if image.mode == "RGBA" and mime_type == CameraMimeType.JPEG: + image = image.convert("RGB") + image.save(buf, format=mime_type.name) + data = buf.getvalue() + else: + raise ValueError(f"Cannot encode image to {mime_type}") + + return ViamImage(data, mime_type) + + def bytes_to_depth_array(image: ViamImage) -> List[List[int]]: """ Decode the data of an image that has the custom depth MIME type ``image/vnd.viam.dep`` into a standard representation. @@ -39,7 +62,7 @@ def bytes_to_depth_array(image: ViamImage) -> List[List[int]]: Returns: List[List[int]]: The standard representation of the image. """ - if image.mime_type != CameraMimeType.VIAM_RAW_DEPTH.value: + if image.mime_type != CameraMimeType.VIAM_RAW_DEPTH: raise NotSupportedError("Type must be `image/vnd.viam.dep` to use bytes_to_depth_array()") width = int.from_bytes(image.data[8:16], "big") @@ -62,12 +85,9 @@ def determine_image_dimensions(image: ViamImage) -> None: Args: image (ViamImage): The image to get dimensions of. - - Raises: - NotSupportedError: Raised if given an image that is not of MIME type `image/vnd.viam.dep`. """ - if image.mime_type not in [CameraMimeType.JPEG, CameraMimeType.PNG, CameraMimeType.VIAM_RGBA]: - raise NotSupportedError("Type must be `image/jpeg`, `image/png`, or `image/vnd.viam.rgba` to use determine_image_dimensions()") + if image.mime_type.name not in LIBRARY_SUPPORTED_FORMATS: + return pil_img = viam_to_pil_image(image) image.width = pil_img.width diff --git a/src/viam/media/video.py b/src/viam/media/video.py index d50991ff6..a6f2744a5 100644 --- a/src/viam/media/video.py +++ b/src/viam/media/video.py @@ -53,7 +53,7 @@ def to_proto(self) -> Format.ValueType: class ViamImage: """A native implementation of an image. - Provides the raw data and the mime type, as well as lazily loading and caching the PIL.Image representation. + Provides the raw data and the mime type. """ _height: Optional[int] @@ -75,12 +75,6 @@ def mime_type(self) -> CameraMimeType: """The mime type of the image""" return self._mime_type - @mime_type.setter - def mime_type(self, value: CameraMimeType): - if value == self.mime_type: - return - self._mime_type = value - @property def width(self) -> Union[int, None]: """The width of the image""" diff --git a/src/viam/services/vision/vision.py b/src/viam/services/vision/vision.py index 8c5cbfcab..dc2db9d74 100644 --- a/src/viam/services/vision/vision.py +++ b/src/viam/services/vision/vision.py @@ -76,9 +76,6 @@ async def get_detections( # Get an image from the camera img = await cam1.get_image() - # Get image dimensions - determine_image_dimensions(img) - # Get detections from that image detections = await my_detector.get_detections(img) @@ -148,9 +145,6 @@ async def get_classifications( # Get an image from the camera img = await cam1.get_image() - # Get image dimensions - determine_image_dimensions(img) - # Get the 2 classifications with the highest confidence scores classifications = await my_classifier.get_classifications(img, 2) diff --git a/tests/test_camera.py b/tests/test_camera.py index 5f6a9210b..c8adaafc2 100644 --- a/tests/test_camera.py +++ b/tests/test_camera.py @@ -150,11 +150,11 @@ async def test_get_image(self, camera: MockCamera, service: CameraRPCService, im assert response.image == image.data assert camera.timeout == loose_approx(18.2) - # Test empty mime type. Empty mime type should default to JPEG for non-depth cameras + # Test empty mime type. Empty mime type should default to response mime type request = GetImageRequest(name="camera") response: GetImageResponse = await client.GetImage(request) assert response.image == image.data - assert service._camera_mime_types["camera"] == CameraMimeType.JPEG + assert response.mime_type == image.mime_type @pytest.mark.asyncio async def test_get_images(self, camera: MockCamera, service: CameraRPCService, metadata: ResponseMetadata): diff --git a/tests/test_media.py b/tests/test_media.py index eaa20cf92..6aa861fdd 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -6,7 +6,7 @@ from PIL import Image from viam.errors import NotSupportedError -from viam.media.utils import bytes_to_depth_array, determine_image_dimensions, viam_to_pil_image +from viam.media.utils import bytes_to_depth_array, determine_image_dimensions, pil_to_viam_image, viam_to_pil_image from viam.media.video import CameraMimeType, NamedImage, ViamImage @@ -20,16 +20,6 @@ def test_supported_image(self): pil_img = viam_to_pil_image(img) assert pil_img.tobytes() == i.tobytes() - def test_mime_type_update(self): - i = Image.new("RGBA", (100, 100), "#AABBCCDD") - b = BytesIO() - i.save(b, "PNG") - img = ViamImage(b.getvalue(), CameraMimeType.PNG) - assert img._mime_type == CameraMimeType.PNG - - img.mime_type = CameraMimeType.JPEG - assert img._mime_type == CameraMimeType.JPEG - def test_set_image_dimensions(self): img = ViamImage(b"data", CameraMimeType.JPEG) with pytest.raises(AttributeError): @@ -49,13 +39,16 @@ def test_name(self): assert img.name == name -def test_viam_to_pil_image(): +def test_image_conversion(): i = Image.new("RGBA", (100, 100), "#AABBCCDD") - b = BytesIO() - i.save(b, "PNG") - img = ViamImage(b.getvalue(), CameraMimeType.JPEG) - pil_img = viam_to_pil_image(img) - assert pil_img.tobytes() == i.tobytes() + + v_img = pil_to_viam_image(i, CameraMimeType.JPEG) + assert isinstance(v_img, ViamImage) + assert v_img.mime_type == CameraMimeType.JPEG + + pil_img = viam_to_pil_image(v_img) + v_img2 = pil_to_viam_image(pil_img, CameraMimeType.JPEG) + assert v_img2.data == v_img.data def test_bytes_to_depth_array(): @@ -116,5 +109,12 @@ def test_determine_image_dimensions(): assert img3.height == 100 img4 = ViamImage(b"data", CameraMimeType.PCD) - with pytest.raises(NotSupportedError): - determine_image_dimensions(img4) + with pytest.raises(AttributeError): + img4.width + with pytest.raises(AttributeError): + img4.height + determine_image_dimensions(img4) + with pytest.raises(AttributeError): + img4.width + with pytest.raises(AttributeError): + img4.height From 71c2a1c9a7be194fa5508747a6c70caa9a8d443d Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Mon, 15 Apr 2024 14:11:51 -0400 Subject: [PATCH 31/36] determine dimensions internally --- src/viam/media/utils.py | 49 ------------- src/viam/media/video.py | 68 +++++++++++++++--- tests/test_media.py | 134 ++++++++++++++--------------------- tests/test_vision_service.py | 9 +-- 4 files changed, 117 insertions(+), 143 deletions(-) diff --git a/src/viam/media/utils.py b/src/viam/media/utils.py index cb8539093..687179785 100644 --- a/src/viam/media/utils.py +++ b/src/viam/media/utils.py @@ -1,10 +1,7 @@ -from array import array from io import BytesIO -from typing import List from PIL import Image -from viam.errors import NotSupportedError from viam.media.video import CameraMimeType, ViamImage from .viam_rgba_plugin import RGBA_FORMAT_LABEL @@ -47,49 +44,3 @@ def pil_to_viam_image(image: Image.Image, mime_type: CameraMimeType) -> ViamImag raise ValueError(f"Cannot encode image to {mime_type}") return ViamImage(data, mime_type) - - -def bytes_to_depth_array(image: ViamImage) -> List[List[int]]: - """ - Decode the data of an image that has the custom depth MIME type ``image/vnd.viam.dep`` into a standard representation. - - Args: - image (ViamImage): The image to be decoded. - - Raises: - NotSupportedError: Raised if given an image that is not of MIME type `image/vnd.viam.dep`. - - Returns: - List[List[int]]: The standard representation of the image. - """ - if image.mime_type != CameraMimeType.VIAM_RAW_DEPTH: - raise NotSupportedError("Type must be `image/vnd.viam.dep` to use bytes_to_depth_array()") - - width = int.from_bytes(image.data[8:16], "big") - height = int.from_bytes(image.data[16:24], "big") - image.width = width - image.height = height - depth_arr = array("H", image.data[24:]) - depth_arr.byteswap() - - depth_arr_2d = [[depth_arr[row * width + col] for col in range(width)] for row in range(height)] - return depth_arr_2d - - -def determine_image_dimensions(image: ViamImage) -> None: - """ - Get image dimensions from the data of an image that has the MIME type ``image/jpeg``, ``image/png``, or ``image/vnd.viam.rgba`` and set - the corresponding properties of the image. - - Alternatively, image dimensions can be set manually as well (i.e.: image.width = 100) - - Args: - image (ViamImage): The image to get dimensions of. - """ - if image.mime_type.name not in LIBRARY_SUPPORTED_FORMATS: - return - - pil_img = viam_to_pil_image(image) - image.width = pil_img.width - image.height = pil_img.height - pil_img.close() diff --git a/src/viam/media/video.py b/src/viam/media/video.py index a6f2744a5..3870142cd 100644 --- a/src/viam/media/video.py +++ b/src/viam/media/video.py @@ -1,10 +1,19 @@ +from array import array from enum import Enum -from typing import Optional, Union +from io import BytesIO +from typing import List, Optional, Union +from PIL import Image, UnidentifiedImageError from typing_extensions import Self +from viam.errors import NotSupportedError from viam.proto.component.camera import Format +from .viam_rgba_plugin import RGBA_FORMAT_LABEL + +# Formats that are supported by PIL +LIBRARY_SUPPORTED_FORMATS = ["JPEG", "PNG", RGBA_FORMAT_LABEL] + class CameraMimeType(str, Enum): VIAM_RGBA = "image/vnd.viam.rgba" @@ -56,10 +65,12 @@ class ViamImage: Provides the raw data and the mime type. """ - _height: Optional[int] - _width: Optional[int] _data: bytes _mime_type: CameraMimeType + _image: Optional[Image.Image] = None + _image_decoded = False + _height: Optional[int] = None + _width: Optional[int] = None def __init__(self, data: bytes, mime_type: CameraMimeType) -> None: self._data = data @@ -78,20 +89,57 @@ def mime_type(self) -> CameraMimeType: @property def width(self) -> Union[int, None]: """The width of the image""" + if self._width is not None: + return self._width + + if not self._image_decoded: + try: + self._image = Image.open(BytesIO(self.data), formats=LIBRARY_SUPPORTED_FORMATS) + except UnidentifiedImageError: + self._image = None + self._image_decoded = True + # If we have decoded the image and the image is not none, then set the width so we don't have to do this again + if self._image_decoded and self._image is not None: + self._width = self._image.width return self._width - @width.setter - def width(self, width: int): - self._width = width - @property def height(self) -> Union[int, None]: """The height of the image""" + if self._height is not None: + return self._height + + if not self._image_decoded: + try: + self._image = Image.open(BytesIO(self.data), formats=LIBRARY_SUPPORTED_FORMATS) + except UnidentifiedImageError: + self._image = None + self._image_decoded = True + # If we have decoded the image and the image is not none, then set the height so we don't have to do this again + if self._image_decoded and self._image is not None: + self._height = self._image.height return self._height - @height.setter - def height(self, height: int): - self._height = height + def bytes_to_depth_array(self) -> List[List[int]]: + """ + Decode the data of an image that has the custom depth MIME type ``image/vnd.viam.dep`` into a standard representation. + + Raises: + NotSupportedError: Raised if the image is not of MIME type `image/vnd.viam.dep`. + + Returns: + List[List[int]]: The standard representation of the image. + """ + if self.mime_type != CameraMimeType.VIAM_RAW_DEPTH: + raise NotSupportedError("Type must be `image/vnd.viam.dep` to use bytes_to_depth_array()") + + self._width = int.from_bytes(self.data[8:16], "big") + self._height = int.from_bytes(self.data[16:24], "big") + depth_arr = array("H", self.data[24:]) + depth_arr.byteswap() + + depth_arr_2d = [[depth_arr[row * self._width + col] for col in range(self._width)] for row in range(self._height)] + return depth_arr_2d class NamedImage(ViamImage): diff --git a/tests/test_media.py b/tests/test_media.py index 6aa861fdd..6839ff1d0 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -6,7 +6,7 @@ from PIL import Image from viam.errors import NotSupportedError -from viam.media.utils import bytes_to_depth_array, determine_image_dimensions, pil_to_viam_image, viam_to_pil_image +from viam.media.utils import pil_to_viam_image, viam_to_pil_image from viam.media.video import CameraMimeType, NamedImage, ViamImage @@ -20,16 +20,59 @@ def test_supported_image(self): pil_img = viam_to_pil_image(img) assert pil_img.tobytes() == i.tobytes() - def test_set_image_dimensions(self): - img = ViamImage(b"data", CameraMimeType.JPEG) - with pytest.raises(AttributeError): - img.width - with pytest.raises(AttributeError): - img.height - img.width = 100 - img.height = 100 - assert img.width == 100 - assert img.height == 100 + def test_dimensions(self): + i = Image.new("RGBA", (100, 100), "#AABBCCDD") + + img1 = pil_to_viam_image(i, CameraMimeType.JPEG) + assert img1._image_decoded is False + assert img1._image is None + assert img1.width == 100 + assert img1.height == 100 + assert img1._image_decoded is True + assert img1._image is not None + + img2 = pil_to_viam_image(i, CameraMimeType.PNG) + assert img2._image_decoded is False + assert img2._image is None + assert img2.width == 100 + assert img2.height == 100 + assert img2._image_decoded is True + assert img2._image is not None + + img3 = pil_to_viam_image(i, CameraMimeType.VIAM_RGBA) + assert img3._image_decoded is False + assert img3._image is None + assert img3.width == 100 + assert img3.height == 100 + assert img3._image_decoded is True + assert img3._image is not None + + img4 = ViamImage(b"data", CameraMimeType.PCD) + assert img4._image_decoded is False + assert img4._image is None + assert img4.width is None + assert img4.height is None + assert img4._image_decoded is True + assert img4._image is None + + def test_bytes_to_depth_array(self): + with open(f"{os.path.dirname(__file__)}/../data/fakeDM.vnd.viam.dep", "rb") as depth_map: + img = ViamImage(depth_map.read(), CameraMimeType.VIAM_RAW_DEPTH) + assert isinstance(img, ViamImage) + standard_data = img.bytes_to_depth_array() + assert len(standard_data) == 10 + assert len(standard_data[0]) == 20 + data_arr = array("H", img.data[24:]) + data_arr.byteswap() + assert len(data_arr) == 200 + assert standard_data[0][0] == data_arr[0] + assert standard_data[-1][3] == data_arr[183] + assert standard_data[-1][3] == 9 * 3 + assert standard_data[4][4] == 4 * 4 + + img2 = ViamImage(b"data", CameraMimeType.PCD) + with pytest.raises(NotSupportedError): + img2.bytes_to_depth_array() class TestNamedImage: @@ -49,72 +92,3 @@ def test_image_conversion(): pil_img = viam_to_pil_image(v_img) v_img2 = pil_to_viam_image(pil_img, CameraMimeType.JPEG) assert v_img2.data == v_img.data - - -def test_bytes_to_depth_array(): - with open(f"{os.path.dirname(__file__)}/../data/fakeDM.vnd.viam.dep", "rb") as depth_map: - img = ViamImage(depth_map.read(), CameraMimeType.VIAM_RAW_DEPTH) - assert isinstance(img, ViamImage) - standard_data = bytes_to_depth_array(img) - assert len(standard_data) == 10 - assert len(standard_data[0]) == 20 - data_arr = array("H", img.data[24:]) - data_arr.byteswap() - assert len(data_arr) == 200 - assert standard_data[0][0] == data_arr[0] - assert standard_data[-1][3] == data_arr[183] - assert standard_data[-1][3] == 9 * 3 - assert standard_data[4][4] == 4 * 4 - - img2 = ViamImage(b"data", CameraMimeType.PCD) - with pytest.raises(NotSupportedError): - bytes_to_depth_array(img2) - - -def test_determine_image_dimensions(): - i = Image.new("RGBA", (100, 100), "#AABBCCDD") - b = BytesIO() - i.save(b, "PNG") - img = ViamImage(b.getvalue(), CameraMimeType.PNG) - with pytest.raises(AttributeError): - img.width - with pytest.raises(AttributeError): - img.height - determine_image_dimensions(img) - assert img.width == 100 - assert img.height == 100 - - i2 = Image.new("RGBA", (100, 100), "#AABBCCDD") - b2 = BytesIO() - i2.save(b2, "VIAM_RGBA") - img2 = ViamImage(b.getvalue(), CameraMimeType.VIAM_RGBA) - with pytest.raises(AttributeError): - img2.width - with pytest.raises(AttributeError): - img2.height - determine_image_dimensions(img2) - assert img2.width == 100 - assert img2.height == 100 - - i3 = Image.new("RGB", (100, 100), "#AABBCCDD") - b3 = BytesIO() - i3.save(b3, "JPEG") - img3 = ViamImage(b.getvalue(), CameraMimeType.JPEG) - with pytest.raises(AttributeError): - img3.width - with pytest.raises(AttributeError): - img3.height - determine_image_dimensions(img3) - assert img3.width == 100 - assert img3.height == 100 - - img4 = ViamImage(b"data", CameraMimeType.PCD) - with pytest.raises(AttributeError): - img4.width - with pytest.raises(AttributeError): - img4.height - determine_image_dimensions(img4) - with pytest.raises(AttributeError): - img4.width - with pytest.raises(AttributeError): - img4.height diff --git a/tests/test_vision_service.py b/tests/test_vision_service.py index d59e440d6..39c8a790b 100644 --- a/tests/test_vision_service.py +++ b/tests/test_vision_service.py @@ -2,8 +2,10 @@ import pytest from grpclib.testing import ChannelFor +from PIL import Image -from viam.media.video import CameraMimeType, ViamImage +from viam.media.utils import pil_to_viam_image +from viam.media.video import CameraMimeType from viam.proto.common import ( DoCommandRequest, DoCommandResponse, @@ -34,9 +36,8 @@ from .mocks.services import MockVision -IMAGE = ViamImage(b"data", CameraMimeType.JPEG) -IMAGE.width = 2 -IMAGE.height = 4 +i = Image.new("RGBA", (100, 100), "#AABBCCDD") +IMAGE = pil_to_viam_image(i, CameraMimeType.JPEG) DETECTORS = [ "detector-0", "detector-1", From 2f6c5fa238ff41a8205748ee1a87250e6ca0df2b Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Tue, 16 Apr 2024 16:30:11 -0400 Subject: [PATCH 32/36] rename pil utils file to pil.py --- src/viam/media/{utils.py => pil.py} | 0 tests/test_media.py | 2 +- tests/test_vision_service.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename src/viam/media/{utils.py => pil.py} (100%) diff --git a/src/viam/media/utils.py b/src/viam/media/pil.py similarity index 100% rename from src/viam/media/utils.py rename to src/viam/media/pil.py diff --git a/tests/test_media.py b/tests/test_media.py index 6839ff1d0..5b8e5b049 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -6,7 +6,7 @@ from PIL import Image from viam.errors import NotSupportedError -from viam.media.utils import pil_to_viam_image, viam_to_pil_image +from viam.media.pil import pil_to_viam_image, viam_to_pil_image from viam.media.video import CameraMimeType, NamedImage, ViamImage diff --git a/tests/test_vision_service.py b/tests/test_vision_service.py index 39c8a790b..d1635d182 100644 --- a/tests/test_vision_service.py +++ b/tests/test_vision_service.py @@ -4,7 +4,7 @@ from grpclib.testing import ChannelFor from PIL import Image -from viam.media.utils import pil_to_viam_image +from viam.media.pil import pil_to_viam_image from viam.media.video import CameraMimeType from viam.proto.common import ( DoCommandRequest, From 83f844458573ec08af093a8f28933368b125838b Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Wed, 17 Apr 2024 11:27:41 -0400 Subject: [PATCH 33/36] re-add comment --- src/viam/components/camera/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/viam/components/camera/service.py b/src/viam/components/camera/service.py index 19ccd3c3a..e1c617f32 100644 --- a/src/viam/components/camera/service.py +++ b/src/viam/components/camera/service.py @@ -64,7 +64,7 @@ async def RenderFrame(self, stream: Stream[RenderFrameRequest, HttpBody]) -> Non camera = self.get_resource(name) timeout = stream.deadline.time_remaining() if stream.deadline else None image = await camera.get_image(request.mime_type, timeout=timeout, metadata=stream.metadata) - response = HttpBody(data=image.data, content_type=image.mime_type) + response = HttpBody(data=image.data, content_type=image.mime_type) # type: ignore await stream.send_message(response) async def GetPointCloud(self, stream: Stream[GetPointCloudRequest, GetPointCloudResponse]) -> None: From a05f6c6e3efef747c32324a3fd769325facedc9e Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Wed, 17 Apr 2024 11:43:18 -0400 Subject: [PATCH 34/36] move pil to utils --- docs/examples/example.ipynb | 4 ++-- src/viam/components/camera/camera.py | 4 +--- src/viam/media/utils/__init__.py | 0 src/viam/media/{ => utils}/pil.py | 2 +- tests/test_media.py | 2 +- tests/test_vision_service.py | 2 +- 6 files changed, 6 insertions(+), 8 deletions(-) create mode 100644 src/viam/media/utils/__init__.py rename src/viam/media/{ => utils}/pil.py (96%) diff --git a/docs/examples/example.ipynb b/docs/examples/example.ipynb index 0d9afeb91..a396179d2 100644 --- a/docs/examples/example.ipynb +++ b/docs/examples/example.ipynb @@ -161,7 +161,7 @@ "source": [ "from viam.components.camera import Camera\n", "from viam.media.video import CameraMimeType\n", - "from viam.media.utils import viam_to_pil_image\n", + "from viam.media.utils.pil import viam_to_pil_image\n", "\n", "robot = await connect_with_channel()\n", "camera = Camera.from_robot(robot, \"camera0\")\n", @@ -1491,7 +1491,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.6 (main, Oct 2 2023, 20:46:14) [Clang 14.0.3 (clang-1403.0.22.14.1)]" + "version": "3.11.9 (main, Apr 2 2024, 08:25:04) [Clang 15.0.0 (clang-1500.1.0.2.5)]" }, "vscode": { "interpreter": { diff --git a/src/viam/components/camera/camera.py b/src/viam/components/camera/camera.py index 524eb6531..3f66a82c2 100644 --- a/src/viam/components/camera/camera.py +++ b/src/viam/components/camera/camera.py @@ -45,8 +45,6 @@ async def get_image( to convert the data to a standard representation. :: - from viam.media.utils import bytes_to_depth_array - my_camera = Camera.from_robot(robot=robot, name="my_camera") # Assume "frame" has a mime_type of "image/vnd.viam.dep" @@ -54,7 +52,7 @@ async def get_image( # Convert "frame" to a standard 2D image representation. # Remove the 1st 3x8 bytes and reshape the raw bytes to List[List[Int]]. - standard_frame = bytes_to_depth_array(frame) + standard_frame = frame.bytes_to_depth_array() Args: mime_type (str): The desired mime type of the image. This does not guarantee output type diff --git a/src/viam/media/utils/__init__.py b/src/viam/media/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/viam/media/pil.py b/src/viam/media/utils/pil.py similarity index 96% rename from src/viam/media/pil.py rename to src/viam/media/utils/pil.py index 687179785..6f5103df3 100644 --- a/src/viam/media/pil.py +++ b/src/viam/media/utils/pil.py @@ -4,7 +4,7 @@ from viam.media.video import CameraMimeType, ViamImage -from .viam_rgba_plugin import RGBA_FORMAT_LABEL +from ..viam_rgba_plugin import RGBA_FORMAT_LABEL # Formats that are supported by PIL LIBRARY_SUPPORTED_FORMATS = ["JPEG", "PNG", RGBA_FORMAT_LABEL] diff --git a/tests/test_media.py b/tests/test_media.py index 5b8e5b049..ec8d88503 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -6,7 +6,7 @@ from PIL import Image from viam.errors import NotSupportedError -from viam.media.pil import pil_to_viam_image, viam_to_pil_image +from viam.media.utils.pil import pil_to_viam_image, viam_to_pil_image from viam.media.video import CameraMimeType, NamedImage, ViamImage diff --git a/tests/test_vision_service.py b/tests/test_vision_service.py index d1635d182..39973c938 100644 --- a/tests/test_vision_service.py +++ b/tests/test_vision_service.py @@ -4,7 +4,7 @@ from grpclib.testing import ChannelFor from PIL import Image -from viam.media.pil import pil_to_viam_image +from viam.media.utils.pil import pil_to_viam_image from viam.media.video import CameraMimeType from viam.proto.common import ( DoCommandRequest, From 580b67f471d55d1bca32ffa76b91daba93744312 Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Wed, 17 Apr 2024 12:14:16 -0400 Subject: [PATCH 35/36] revert comment --- src/viam/components/camera/camera.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/viam/components/camera/camera.py b/src/viam/components/camera/camera.py index 3f66a82c2..2cb165779 100644 --- a/src/viam/components/camera/camera.py +++ b/src/viam/components/camera/camera.py @@ -41,7 +41,7 @@ async def get_image( """Get the next image from the camera as a ViamImage. Be sure to close the image when finished. - NOTE: If the mime type is ``image/vnd.viam.dep`` you can use :func:`viam.media.utils.bytes_to_depth_array` + NOTE: If the mime type is ``image/vnd.viam.dep`` you can use :func:`viam.media.video.ViamImage.bytes_to_depth_array` to convert the data to a standard representation. :: From 96f1fcb4d1dec19fd6b99be9e2c66b36d8dbb09a Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Wed, 17 Apr 2024 12:41:10 -0400 Subject: [PATCH 36/36] undo change --- src/viam/components/camera/camera.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/viam/components/camera/camera.py b/src/viam/components/camera/camera.py index 2cb165779..560e1eb20 100644 --- a/src/viam/components/camera/camera.py +++ b/src/viam/components/camera/camera.py @@ -45,6 +45,7 @@ async def get_image( to convert the data to a standard representation. :: + my_camera = Camera.from_robot(robot=robot, name="my_camera") # Assume "frame" has a mime_type of "image/vnd.viam.dep"