Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add image data api #165

Draft
wants to merge 6 commits into
base: main
Choose a base branch
from
+272 −64
Draft

Add image data api #165

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
926f81d
add imageData api and tests
clintpurser Jan 3, 2024
679feb6
update example to use new imageData api
clintpurser Jan 3, 2024
b945c41
flyby typo fix
clintpurser Jan 3, 2024
db3340c
remove flutter dependency
clintpurser Jan 4, 2024
2235b9f
add comment
clintpurser Jan 4, 2024
5824e64
add camera image widgets
clintpurser Jan 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 7 additions & 39 deletions example/viam_robot_example_app/lib/screens/stream.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
import 'dart:typed_data';
import 'dart:ui' as ui;

import 'package:flutter/material.dart';
import 'package:image/image.dart' as img;
import 'package:viam_sdk/viam_sdk.dart';
import 'package:viam_sdk/widgets.dart';

Expand All @@ -21,50 +19,20 @@ class StreamScreen extends StatefulWidget {

class _StreamScreenState extends State<StreamScreen> {
// Single frame
ByteData? imageBytes;
Uint8List? _imageData;
bool _imgLoaded = false;

void _getImage() {
Future<void> _getImage() async {
setState(() {
_imgLoaded = false;
});
final imageFut = widget.camera.image();
imageFut.then((value) {
final convertFut = convertImageToFlutterUi(value.image ?? img.Image.empty());
convertFut.then((value) {
final pngFut = value.toByteData(format: ui.ImageByteFormat.png);
pngFut.then((value) => setState(() {
imageBytes = value;
_imgLoaded = true;
}));
});
final imageDataResponse = await widget.camera.imageData();
setState(() {
_imageData = imageDataResponse;
_imgLoaded = true;
});
}

Future<ui.Image> convertImageToFlutterUi(img.Image image) async {
if (image.format != img.Format.uint8 || image.numChannels != 4) {
final cmd = img.Command()
..image(image)
..convert(format: img.Format.uint8, numChannels: 4);
final rgba8 = await cmd.getImageThread();
if (rgba8 != null) {
image = rgba8;
}
}

final ui.ImmutableBuffer buffer = await ui.ImmutableBuffer.fromUint8List(image.toUint8List());

final ui.ImageDescriptor id =
ui.ImageDescriptor.raw(buffer, height: image.height, width: image.width, pixelFormat: ui.PixelFormat.rgba8888);

final ui.Codec codec = await id.instantiateCodec(targetHeight: image.height, targetWidth: image.width);

final ui.FrameInfo fi = await codec.getNextFrame();
final ui.Image uiImage = fi.image;

return uiImage;
}

@override
Widget build(BuildContext context) {
return Scaffold(
Expand All @@ -82,7 +50,7 @@ class _StreamScreenState extends State<StreamScreen> {
const SizedBox(height: 16),
ViamCameraStreamView(camera: widget.camera, streamClient: widget.client),
const SizedBox(height: 16),
if (_imgLoaded) Image.memory(Uint8List.view(imageBytes!.buffer), scale: 3),
if (_imgLoaded) Image.memory(_imageData!, scale: 3),
const SizedBox(height: 16),
ElevatedButton(
child: const Text('Get image'),
Expand Down
10 changes: 10 additions & 0 deletions lib/src/components/camera/camera.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import 'dart:typed_data';

import '../../gen/common/v1/common.pb.dart';
import '../../gen/component/camera/v1/camera.pb.dart';
import '../../media/image.dart';
Expand All @@ -20,6 +22,14 @@ abstract class Camera extends Resource {
/// Get the camera's intrinsic parameters and the camera's distortion parameters.
Future<CameraProperties> properties();

/// Get the next image from the camera.
///
/// This can then be wrapped in an Image widget such as:
/// ```dart
/// Image.memory(myImageData);
/// ```
Future<Uint8List> imageData({MimeType? mimeType});

/// Get the [ResourceName] for this [Camera] with the given [name]
static ResourceName getResourceName(String name) {
return Camera.subtype.getResourceName(name);
Expand Down
37 changes: 37 additions & 0 deletions lib/src/components/camera/client.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
import 'dart:typed_data';
import 'dart:ui' as ui;

import 'package:flutter/foundation.dart';
import 'package:grpc/grpc_connection_interface.dart';
import 'package:image/image.dart' as img;

import '../../gen/common/v1/common.pb.dart';
import '../../gen/component/camera/v1/camera.pbgrpc.dart';
Expand Down Expand Up @@ -54,4 +59,36 @@ class CameraClient extends Camera implements ResourceRPCClient {
final response = await client.doCommand(request);
return response.result.toMap();
}

@override
Future<Uint8List> imageData({MimeType? mimeType}) async {
final imageFromCamera = await image(mimeType: mimeType);
final convertedImage = await _convertImageToFlutterUi(imageFromCamera.image!);
final png = await convertedImage.toByteData(format: ui.ImageByteFormat.png);
return Uint8List.view(png!.buffer);
}

Future<ui.Image> _convertImageToFlutterUi(img.Image image) async {
if (image.format != img.Format.uint8 || image.numChannels != 4) {
final cmd = img.Command()
..image(image)
..convert(format: img.Format.uint8, numChannels: 4);
final rgba8 = await cmd.getImageThread();
if (rgba8 != null) {
image = rgba8;
}
}

final ui.ImmutableBuffer buffer = await ui.ImmutableBuffer.fromUint8List(image.toUint8List());

final ui.ImageDescriptor id =
ui.ImageDescriptor.raw(buffer, height: image.height, width: image.width, pixelFormat: ui.PixelFormat.rgba8888);

final ui.Codec codec = await id.instantiateCodec(targetHeight: image.height, targetWidth: image.width);

final ui.FrameInfo fi = await codec.getNextFrame();
final ui.Image uiImage = fi.image;

return uiImage;
}
}
2 changes: 1 addition & 1 deletion lib/src/media/image.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ class MimeType {
}
}

/// A custom image type that contains the [MimeTYpe], raw image data, and lazily loads and caches an [img.Image].
/// A custom image type that contains the [MimeType], raw image data, and lazily loads and caches an [img.Image].
clintpurser marked this conversation as resolved.
Show resolved Hide resolved
class ViamImage {
/// The mimetype of the image
final MimeType mimeType;
Expand Down
86 changes: 78 additions & 8 deletions test/unit_test/components/camera_test.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import 'dart:typed_data';

import 'package:flutter_test/flutter_test.dart';
import 'package:grpc/grpc.dart';
import 'package:viam_sdk/src/components/camera/service.dart';
Expand All @@ -8,6 +10,49 @@ import 'package:viam_sdk/viam_sdk.dart';

import '../../test_utils.dart';

const validJpeg = [
255, 216, 255, 224, 0, 16, 74, 70, 73, 70, 0, 1, 2, 1, 0, 72, 0,
72, 0, 0, 255, 219, 0, 67, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 255, 219, 0, 67, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 255, 192, 0, 17, 8, 0, 1, 0, 1, 3, 1, 17, 0, 2, 17, 1, 3, 17, 1, 255, 196, 0, 31,
0, 0, 1, 5, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
255, 196, 0, 181, 16, 0, 2, 1, 3, 3, 2, 4, 3, 5, 5, 4, 4, 0, 0, 1, 125, 1, 2, 3, 0, 4,
17, 5, 18, 33, 49, 65, 6, 19, 81, 97, 7, 34, 113, 20, 50, 129, 145, 161, 8, 35, 66, 177,
193, 21, 82, 209, 240, 36, 51, 98, 114, 130, 9, 10, 22, 23, 24, 25, 26, 37, 38, 39, 40,
41, 42, 52, 53, 54, 55, 56, 57, 58, 67, 68, 69, 70, 71, 72, 73, 74, 83, 84, 85, 86, 87,
88, 89, 90, 99, 100, 101, 102, 103, 104, 105, 106, 115, 116, 117, 118, 119, 120, 121,
122, 131, 132, 133, 134, 135, 136, 137, 138, 146, 147, 148, 149, 150, 151, 152, 153,
154, 162, 163, 164, 165, 166, 167, 168, 169, 170, 178, 179, 180, 181, 182, 183, 184,
185, 186, 194, 195, 196, 197, 198, 199, 200, 201, 202, 210, 211, 212, 213, 214, 215,
216, 217, 218, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 241, 242, 243, 244,
245, 246, 247, 248, 249, 250, 255, 196, 0, 31, 1, 0, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0,
0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 255, 196, 0, 181, 17, 0, 2, 1, 2, 4,
4, 3, 4, 7, 5, 4, 4, 0, 1, 2, 119, 0, 1, 2, 3, 17, 4, 5, 33, 49, 6, 18, 65, 81, 7, 97,
113, 19, 34, 50, 129, 8, 20, 66, 145, 161, 177, 193, 9, 35, 51, 82, 240, 21, 98, 114,
209, 10, 22, 36, 52, 225, 37, 241, 23, 24, 25, 26, 38, 39, 40, 41, 42, 53, 54, 55, 56,
57, 58, 67, 68, 69, 70, 71, 72, 73, 74, 83, 84, 85, 86, 87, 88, 89, 90, 99, 100, 101,
102, 103, 104, 105, 106, 115, 116, 117, 118, 119, 120, 121, 122, 130, 131, 132, 133,
134, 135, 136, 137, 138, 146, 147, 148, 149, 150, 151, 152, 153, 154, 162, 163, 164,
165, 166, 167, 168, 169, 170, 178, 179, 180, 181, 182, 183, 184, 185, 186, 194, 195,
196, 197, 198, 199, 200, 201, 202, 210, 211, 212, 213, 214, 215, 216, 217, 218, 226,
227, 228, 229, 230, 231, 232, 233, 234, 242, 243, 244, 245, 246, 247, 248, 249, 250,
255, 218, 0, 12, 3, 1, 0, 2, 17, 3, 17, 0, 63, 0, 254, 178, 40, 0, 255, 217, 1, 1, 1,
1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 255, 196, 0, 181,
17, 0, 2, 1, 2, 4, 4, 3, 4, 7, 5, 4, 4, 0, 1, 2, 119, 0, 1, 2, 3, 17, 4, 5, 33, 49, 6, 18,
65, 81, 7, 97, 113, 19, 34, 50, 129, 8, 20, 66, 145, 161, 177, 193, 9, 35, 51, 82, 240, 21,
98, 114, 209, 10, 22, 36, 52, 225, 37, 241, 23, 24, 25, 26, 38, 39, 40, 41, 42, 53, 54, 55,
56, 57, 58, 67, 68, 69, 70, 71, 72, 73, 74, 83, 84, 85, 86, 87, 88, 89, 90, 99, 100, 101, 102,
103, 104, 105, 106, 115, 116, 117, 118, 119, 120, 121, 122, 130, 131, 132, 133, 134, 135, 136,
137, 138, 146, 147, 148, 149, 150, 151, 152, 153, 154, 162, 163, 164, 165, 166, 167, 168, 169,
170, 178, 179, 180, 181, 182, 183, 184, 185, 186, 194, 195, 196, 197, 198, 199, 200, 201, 202,
210, 211, 212, 213, 214, 215, 216, 217, 218, 226, 227, 228, 229, 230, 231, 232, 233, 234, 242,
243, 244, 245, 246, 247, 248, 249, 250, 255, 218, 0, 12, 3, 1, 0, 2, 17, 3, 17, 0, 63, 0, 254,
178, 40, 0, 255, 217 // prevent dartfmt
];

clintpurser marked this conversation as resolved.
Show resolved Hide resolved
class FakeCamera extends Camera {
Map<String, dynamic>? extra;

Expand All @@ -26,7 +71,8 @@ class FakeCamera extends Camera {
if (mimeType == null) {
throw const GrpcError.invalidArgument('invalid mimetype');
}
return ViamImage([0, 0, 0], mimeType);

return ViamImage(validJpeg, mimeType);
}

@override
Expand All @@ -41,10 +87,15 @@ class FakeCamera extends Camera {
..intrinsicParameters = (IntrinsicParameters()..widthPx = 10)
..distortionParameters = (DistortionParameters()..model = 'test');
}

@override
Future<Uint8List> imageData({MimeType? mimeType}) async {
return Uint8List(0);
}
}

void main() {
group('Camera Tests', () {
group('FakeCamera Tests', () {
const String name = 'camera';
late FakeCamera camera;

Expand All @@ -55,11 +106,11 @@ void main() {
test('image', () async {
final actualJpeg = await camera.image(mimeType: MimeType.jpeg);
expect(actualJpeg.mimeType, MimeType.jpeg);
expect(actualJpeg.raw, [0, 0, 0]);
expect(actualJpeg.raw, validJpeg);

final actualPng = await camera.image(mimeType: MimeType.png);
expect(actualPng.mimeType, MimeType.png);
expect(actualPng.raw, [0, 0, 0]);
expect(actualPng.raw, validJpeg);
});

test('pointCloud', () async {
Expand All @@ -74,6 +125,11 @@ void main() {
expect(actual.intrinsicParameters.widthPx, 10);
});

test('imageData', () async {
final actual = await camera.imageData();
expect(actual, Uint8List(0));
});

test('doCommand', () async {
final cmd = {'foo': 'bar'};
final resp = await camera.doCommand(cmd);
Expand Down Expand Up @@ -113,15 +169,15 @@ void main() {

final actualJpeg = await client.getImage(jpegRequest);
expect(actualJpeg.mimeType, 'jpeg');
expect(actualJpeg.image, [0, 0, 0]);
expect(actualJpeg.image, validJpeg);

final pngRequest = GetImageRequest()
..name = name
..mimeType = 'png';

final actualPng = await client.getImage(pngRequest);
expect(actualPng.mimeType, 'png');
expect(actualPng.image, [0, 0, 0]);
expect(actualPng.image, validJpeg);
});

test('pointCloud', () async {
Expand Down Expand Up @@ -152,11 +208,11 @@ void main() {
final client = CameraClient(name, channel);
final actualJpeg = await client.image(mimeType: MimeType.jpeg);
expect(actualJpeg.mimeType, MimeType.jpeg);
expect(actualJpeg.raw, [0, 0, 0]);
expect(actualJpeg.raw, validJpeg);

final actualPng = await client.image(mimeType: MimeType.png);
expect(actualPng.mimeType, MimeType.png);
expect(actualPng.raw, [0, 0, 0]);
expect(actualPng.raw, validJpeg);
});

test('pointCloud', () async {
Expand All @@ -173,6 +229,20 @@ void main() {
expect(actual.intrinsicParameters.widthPx, 10);
});

test('imageData', () async {
final client = CameraClient(name, channel);
final actual = await client.imageData(mimeType: MimeType.jpeg);
final validImageData = [
137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0,
13, 73, 72, 68, 82, 0, 0, 0, 1, 0, 0, 0, 1, 8, 6, 0, 0, 0,
31, 21, 196, 137, 0, 0, 0, 4, 115, 66, 73, 84, 8, 8, 8, 8,
124, 8, 100, 136, 0, 0, 0, 13, 73, 68, 65, 84, 8, 153, 99,
184, 121, 243, 230, 127, 0, 8, 165, 3, 139, 65, 53, 234,
255, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130 // prevent dartFmt
];
expect(actual, validImageData);
});

test('doCommand', () async {
final client = CameraClient(name, channel);
final cmd = {'foo': 'bar'};
Expand Down