From 1de6fce883b5105ba3ea3bb2bd0579f658f807f4 Mon Sep 17 00:00:00 2001 From: Naveed Jooma Date: Thu, 1 Feb 2024 04:59:19 -0500 Subject: [PATCH] [RSDK-6497] Vision Service (#236) --- package-lock.json | 26 ++--- src/main.ts | 26 ++--- src/services/vision.ts | 7 ++ src/services/vision/client.spec.ts | 173 +++++++++++++++++++++++++++++ src/services/vision/client.ts | 149 +++++++++++++++++++++++++ src/services/vision/types.ts | 6 + src/services/vision/vision.ts | 78 +++++++++++++ 7 files changed, 433 insertions(+), 32 deletions(-) create mode 100644 src/services/vision.ts create mode 100644 src/services/vision/client.spec.ts create mode 100644 src/services/vision/client.ts create mode 100644 src/services/vision/types.ts create mode 100644 src/services/vision/vision.ts diff --git a/package-lock.json b/package-lock.json index c825dbd99..3f0430849 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1419,9 +1419,9 @@ "dev": true }, "node_modules/@types/node": { - "version": "20.11.10", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.10.tgz", - "integrity": "sha512-rZEfe/hJSGYmdfX9tvcPMYeYPW2sNl50nsw4jZmRcaG0HIAb0WYEpsB05GOb53vjqpyE9GUhlDQ4jLSoB5q9kg==", + "version": "20.11.14", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.14.tgz", + "integrity": "sha512-w3yWCcwULefjP9DmDDsgUskrMoOy5Z8MiwKHr1FvqGPtx7CvJzQvxD7eKpxNtklQxLruxSXWddyeRtyud0RcXQ==", "dev": true, "dependencies": { "undici-types": "~5.26.4" @@ -3730,9 +3730,9 @@ "dev": true }, "node_modules/electron-to-chromium": { - "version": "1.4.649", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.649.tgz", - "integrity": "sha512-dq/owIaALxZGqWm5RXpKQ4baX6aDC19e2Z16c8SXYN+I71PyEKjbVqQUgm7kcuk8CRqljTKXbolo0XXDjxnh2w==", + "version": "1.4.652", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.652.tgz", + "integrity": "sha512-XvQaa8hVUAuEJtLw6VKQqvdOxTOfBLWfI10t2xWpezx4XXD3k8bdLweEKeItqaa0+OkJX5l0mP1W+JWobyIDrg==", "dev": true, "peer": true }, @@ -3965,12 +3965,12 @@ } }, "node_modules/eslint-plugin-vitest": { - "version": "0.3.20", - "resolved": "https://registry.npmjs.org/eslint-plugin-vitest/-/eslint-plugin-vitest-0.3.20.tgz", - "integrity": "sha512-O05k4j9TGMOkkghj9dRgpeLDyOSiVIxQWgNDPfhYPm5ioJsehcYV/zkRLekQs+c8+RBCVXucSED3fYOyy2EoWA==", + "version": "0.3.21", + "resolved": "https://registry.npmjs.org/eslint-plugin-vitest/-/eslint-plugin-vitest-0.3.21.tgz", + "integrity": "sha512-oYwR1MrwaBw/OG6CKU+SJYleAc442w6CWL1RTQl5WLwy8X3sh0bgHIQk5iEtmTak3Q+XAvZglr0bIoDOjFdkcw==", "dev": true, "dependencies": { - "@typescript-eslint/utils": "^6.15.0" + "@typescript-eslint/utils": "^6.20.0" }, "engines": { "node": "^18.0.0 || >= 20.0.0" @@ -5698,9 +5698,9 @@ "dev": true }, "node_modules/magic-string": { - "version": "0.30.5", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.5.tgz", - "integrity": "sha512-7xlpfBaQaP/T6Vh8MO/EqXSW5En6INHEvEXQiuff7Gku0PWjU3uf6w/j9o7O+SpB5fOAkrI5HeoNgwjEO0pFsA==", + "version": "0.30.6", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.6.tgz", + "integrity": "sha512-n62qCLbPjNjyo+owKtveQxZFZTBm+Ms6YoGD23Wew6Vw337PElFNifQpknPruVRQV57kVShPnLGo9vWxVhpPvA==", "dev": true, "dependencies": { "@jridgewell/sourcemap-codec": "^1.4.15" diff --git a/src/main.ts b/src/main.ts index a70ef11b0..cd363e02a 100644 --- a/src/main.ts +++ b/src/main.ts @@ -379,29 +379,17 @@ export { * * Generated with https://github.com/improbable-eng/grpc-web * - * @example - * - * ```ts - * import { grpc } from '@improbable-eng/grpc-web'; - * - * const client = {}; // replace with a connected robot client - * - * const request = new visionApi.GetDetectorNamesRequest(); - * request.setName('myvision'); - * - * client.visionService.getDetectorNames( - * request, - * new grpc.Metadata(), - * (error, response) => { - * // do something with error or response - * } - * ); - * ``` - * + * @deprecated Use {@link VisionClient} instead. * @alpha * @group Raw Protobufs */ export { default as visionApi } from './gen/service/vision/v1/vision_pb'; +export { + type Detection, + type Classification, + type PointCloudObject, + VisionClient, +} from './services/vision'; /** * Raw Protobuf interfaces that are shared across multiple components and diff --git a/src/services/vision.ts b/src/services/vision.ts new file mode 100644 index 000000000..e01b2e49b --- /dev/null +++ b/src/services/vision.ts @@ -0,0 +1,7 @@ +export type { Vision } from './vision/vision'; +export type { + Detection, + Classification, + PointCloudObject, +} from './vision/types'; +export { VisionClient } from './vision/client'; diff --git a/src/services/vision/client.spec.ts b/src/services/vision/client.spec.ts new file mode 100644 index 000000000..15adc30bc --- /dev/null +++ b/src/services/vision/client.spec.ts @@ -0,0 +1,173 @@ +// @vitest-environment happy-dom + +import { beforeEach, describe, expect, it, vi, type Mock } from 'vitest'; +import { + Classification as PBClassification, + Detection as PBDetection, +} from '../../gen/service/vision/v1/vision_pb'; +import { PointCloudObject as PBPointCloudObject } from '../../gen/common/v1/common_pb'; +import { VisionServiceClient } from '../../gen/service/vision/v1/vision_pb_service'; +import { RobotClient } from '../../robot'; +import { VisionClient } from './client'; +import type { Classification, Detection, PointCloudObject } from './types'; +vi.mock('../../robot'); +vi.mock('../../gen/service/vision/v1/vision_pb_service'); + +const visionClientName = 'test-vision'; + +let vision: VisionClient; + +describe('VisionClient Tests', () => { + beforeEach(() => { + RobotClient.prototype.createServiceClient = vi + .fn() + .mockImplementation(() => new VisionServiceClient(visionClientName)); + + vision = new VisionClient(new RobotClient('host'), visionClientName); + }); + + describe('Detection Tests', () => { + const testDetection: Detection = { + xMin: 251, + yMin: 225, + xMax: 416, + yMax: 451, + confidence: 0.995_482_683_181_762_7, + className: 'face', + }; + let detection: Mock<[], PBDetection>; + const encodeDetection = (det: Detection) => { + const pbDetection = new PBDetection(); + pbDetection.setClassName(det.className); + pbDetection.setConfidence(det.confidence); + pbDetection.setXMin(det.xMin); + pbDetection.setXMax(det.xMax); + pbDetection.setYMin(det.yMin); + pbDetection.setYMax(det.yMax); + return pbDetection; + }; + + beforeEach(() => { + VisionServiceClient.prototype.getDetections = vi + .fn() + .mockImplementation((_req, _md, cb) => { + cb(null, { + getDetectionsList: () => [detection()], + }); + }); + + VisionServiceClient.prototype.getDetectionsFromCamera = vi + .fn() + .mockImplementation((_req, _md, cb) => { + cb(null, { + getDetectionsList: () => [detection()], + }); + }); + }); + + it('returns detections from a camera', async () => { + detection = vi.fn(() => encodeDetection(testDetection)); + + const expected = [testDetection]; + + await expect( + vision.getDetectionsFromCamera('camera') + ).resolves.toStrictEqual(expected); + }); + + it('returns detections from an image', async () => { + detection = vi.fn(() => encodeDetection(testDetection)); + + const expected = [testDetection]; + + await expect( + vision.getDetections(new Uint8Array(), 1, 1, 'image/jpeg') + ).resolves.toStrictEqual(expected); + }); + }); + + describe('Classification Tests', () => { + const testClassification: Classification = { + className: 'face', + confidence: 0.995_482_683_181_762_7, + }; + let classification: Mock<[], PBClassification>; + const encodeClassification = (cls: Classification) => { + const pbClassification = new PBClassification(); + pbClassification.setClassName(cls.className); + pbClassification.setConfidence(cls.confidence); + return pbClassification; + }; + + beforeEach(() => { + VisionServiceClient.prototype.getClassifications = vi + .fn() + .mockImplementation((_req, _md, cb) => { + cb(null, { + getClassificationsList: () => [classification()], + }); + }); + + VisionServiceClient.prototype.getClassificationsFromCamera = vi + .fn() + .mockImplementation((_req, _md, cb) => { + cb(null, { + getClassificationsList: () => [classification()], + }); + }); + }); + + it('returns classifications from a camera', async () => { + classification = vi.fn(() => encodeClassification(testClassification)); + + const expected = [testClassification]; + + await expect( + vision.getClassificationsFromCamera('camera', 1) + ).resolves.toStrictEqual(expected); + }); + + it('returns classifications from an image', async () => { + classification = vi.fn(() => encodeClassification(testClassification)); + + const expected = [testClassification]; + + await expect( + vision.getClassifications(new Uint8Array(), 1, 1, 'image/jpeg', 1) + ).resolves.toStrictEqual(expected); + }); + }); + + describe('Object Point Cloud Tests', () => { + const testPCO: PointCloudObject = { + pointCloud: 'This is a PointCloudObject', + geometries: undefined, + }; + let pointCloudObject: Mock<[], PBPointCloudObject>; + const encodePCO = (pco: PointCloudObject) => { + const pbPCO = new PBPointCloudObject(); + pbPCO.setPointCloud(pco.pointCloud); + return pbPCO; + }; + + beforeEach(() => { + VisionServiceClient.prototype.getObjectPointClouds = vi + .fn() + .mockImplementation((_req, _md, cb) => { + cb(null, { + getObjectsList: () => [pointCloudObject()], + }); + }); + }); + + it('returns a PointCloudObject from a camera', async () => { + pointCloudObject = vi.fn(() => encodePCO(testPCO)); + + const expected = [testPCO]; + + await expect( + vision.getObjectPointClouds('camera') + ).resolves.toStrictEqual(expected); + }); + }); +}); diff --git a/src/services/vision/client.ts b/src/services/vision/client.ts new file mode 100644 index 000000000..5ad11f598 --- /dev/null +++ b/src/services/vision/client.ts @@ -0,0 +1,149 @@ +import { Struct } from 'google-protobuf/google/protobuf/struct_pb'; +import pb from '../../gen/service/vision/v1/vision_pb'; +import { VisionServiceClient } from '../../gen/service/vision/v1/vision_pb_service'; +import type { MimeType } from '../../main'; +import type { RobotClient } from '../../robot'; +import type { Options, StructType } from '../../types'; +import { doCommandFromClient, promisify } from '../../utils'; +import type { Vision } from './vision'; + +/** + * A gRPC-web client for a Vision service. + * + * @group Clients + */ +export class VisionClient implements Vision { + private client: VisionServiceClient; + private readonly name: string; + private readonly options: Options; + + constructor(client: RobotClient, name: string, options: Options = {}) { + this.client = client.createServiceClient(VisionServiceClient); + this.name = name; + this.options = options; + } + + private get service() { + return this.client; + } + + async getDetectionsFromCamera(cameraName: string, extra: StructType = {}) { + const { service } = this; + + const request = new pb.GetDetectionsFromCameraRequest(); + request.setName(this.name); + request.setCameraName(cameraName); + request.setExtra(Struct.fromJavaScript(extra)); + + this.options.requestLogger?.(request); + + const response = await promisify< + pb.GetDetectionsFromCameraRequest, + pb.GetDetectionsFromCameraResponse + >(service.getDetectionsFromCamera.bind(service), request); + + return response.getDetectionsList().map((x) => x.toObject()); + } + + async getDetections( + image: Uint8Array, + width: number, + height: number, + mimeType: MimeType, + extra: StructType = {} + ) { + const { service } = this; + + const request = new pb.GetDetectionsRequest(); + request.setName(this.name); + request.setImage(image); + request.setWidth(width); + request.setHeight(height); + request.setMimeType(mimeType); + request.setExtra(Struct.fromJavaScript(extra)); + + this.options.requestLogger?.(request); + + const response = await promisify< + pb.GetDetectionsRequest, + pb.GetDetectionsResponse + >(service.getDetections.bind(service), request); + + return response.getDetectionsList().map((x) => x.toObject()); + } + + async getClassificationsFromCamera( + cameraName: string, + count: number, + extra: StructType = {} + ) { + const { service } = this; + + const request = new pb.GetClassificationsFromCameraRequest(); + request.setName(this.name); + request.setCameraName(cameraName); + request.setN(count); + request.setExtra(Struct.fromJavaScript(extra)); + + this.options.requestLogger?.(request); + + const response = await promisify< + pb.GetClassificationsFromCameraRequest, + pb.GetClassificationsFromCameraResponse + >(service.getClassificationsFromCamera.bind(service), request); + + return response.getClassificationsList().map((x) => x.toObject()); + } + + async getClassifications( + image: Uint8Array, + width: number, + height: number, + mimeType: MimeType, + count: number, + extra: StructType = {} + ) { + const { service } = this; + + const request = new pb.GetClassificationsRequest(); + request.setName(this.name); + request.setImage(image); + request.setWidth(width); + request.setHeight(height); + request.setMimeType(mimeType); + request.setN(count); + request.setExtra(Struct.fromJavaScript(extra)); + + this.options.requestLogger?.(request); + + const response = await promisify< + pb.GetClassificationsRequest, + pb.GetClassificationsResponse + >(service.getClassifications.bind(service), request); + + return response.getClassificationsList().map((x) => x.toObject()); + } + + async getObjectPointClouds(cameraName: string, extra: StructType = {}) { + const { service } = this; + + const request = new pb.GetObjectPointCloudsRequest(); + request.setName(this.name); + request.setCameraName(cameraName); + request.setExtra(Struct.fromJavaScript(extra)); + + this.options.requestLogger?.(request); + + const response = await promisify< + pb.GetObjectPointCloudsRequest, + pb.GetObjectPointCloudsResponse + >(service.getObjectPointClouds.bind(service), request); + + return response.getObjectsList().map((x) => x.toObject()); + } + + async doCommand(command: StructType): Promise { + const { service } = this; + return doCommandFromClient(service, this.name, command, this.options); + } +} diff --git a/src/services/vision/types.ts b/src/services/vision/types.ts new file mode 100644 index 000000000..e28e2ff06 --- /dev/null +++ b/src/services/vision/types.ts @@ -0,0 +1,6 @@ +import pb from '../../gen/service/vision/v1/vision_pb'; +import commonPB from '../../gen/common/v1/common_pb'; + +export type Detection = pb.Detection.AsObject; +export type Classification = pb.Classification.AsObject; +export type PointCloudObject = commonPB.PointCloudObject.AsObject; diff --git a/src/services/vision/vision.ts b/src/services/vision/vision.ts new file mode 100644 index 000000000..53ecc211c --- /dev/null +++ b/src/services/vision/vision.ts @@ -0,0 +1,78 @@ +import type { MimeType } from '../../main'; +import type { Resource, StructType } from '../../types'; +import type { Classification, Detection, PointCloudObject } from './types'; + +/** A service that enables various computer vision algorithms */ +export interface Vision extends Resource { + /** + * Get a list of detections in the next image given a camera. + * + * @param cameraName - The name of the camera to use for detection. + * @returns - The list of Detections. + */ + getDetectionsFromCamera: ( + cameraName: string, + extra?: StructType + ) => Promise; + + /** + * Get a list of detections in the given image. + * + * @param image - The image from which to get detections. + * @param width - The width of the image. + * @param height - The height of the image. + * @param mimeType - The MimeType of the image. + * @returns - The list of Detections. + */ + getDetections: ( + image: Uint8Array, + width: number, + height: number, + mimeType: MimeType, + extra?: StructType + ) => Promise; + + /** + * Get a list of classifications in the next image given a camera. + * + * @param cameraName - The name of the camera to use for classification. + * @param count - The number of Classifications requested. + * @returns - The list of Classifications. + */ + getClassificationsFromCamera: ( + cameraName: string, + count: number, + extra?: StructType + ) => Promise; + + /** + * Get a list of classifications in the given image. + * + * @param image - The image from which to get classifications. + * @param width - The width of the image. + * @param height - The height of the image. + * @param mimeType - The MimeType of the image. + * @param count - The number of Classifications requested. + * @returns - The list of Classifications. + */ + getClassifications: ( + image: Uint8Array, + width: number, + height: number, + mimeType: MimeType, + count: number, + extra?: StructType + ) => Promise; + + /** + * Returns a list of the 3D point cloud objects and associated metadata in the + * latest picture obtained from the specified 3D camera. + * + * @param cameraName - The name of the camera. + * @returns - The list of PointCloudObjects + */ + getObjectPointClouds: ( + cameraName: string, + extra?: StructType + ) => Promise; +}