diff --git a/example/viam_robot_example_app/lib/screens/stream.dart b/example/viam_robot_example_app/lib/screens/stream.dart index 2bf317e64e..2eac30e313 100644 --- a/example/viam_robot_example_app/lib/screens/stream.dart +++ b/example/viam_robot_example_app/lib/screens/stream.dart @@ -1,8 +1,6 @@ import 'dart:typed_data'; -import 'dart:ui' as ui; import 'package:flutter/material.dart'; -import 'package:image/image.dart' as img; import 'package:viam_sdk/viam_sdk.dart'; import 'package:viam_sdk/widgets.dart'; @@ -21,50 +19,26 @@ class StreamScreen extends StatefulWidget { class _StreamScreenState extends State { // Single frame - ByteData? imageBytes; + Uint8List? _imageData; bool _imgLoaded = false; - void _getImage() { + @override + void initState() { + _getImage(); + super.initState(); + } + + Future _getImage() async { setState(() { _imgLoaded = false; }); - final imageFut = widget.camera.image(); - imageFut.then((value) { - final convertFut = convertImageToFlutterUi(value.image ?? img.Image.empty()); - convertFut.then((value) { - final pngFut = value.toByteData(format: ui.ImageByteFormat.png); - pngFut.then((value) => setState(() { - imageBytes = value; - _imgLoaded = true; - })); - }); + final imageDataResponse = await widget.camera.imageData(); + setState(() { + _imageData = imageDataResponse; + _imgLoaded = true; }); } - Future convertImageToFlutterUi(img.Image image) async { - if (image.format != img.Format.uint8 || image.numChannels != 4) { - final cmd = img.Command() - ..image(image) - ..convert(format: img.Format.uint8, numChannels: 4); - final rgba8 = await cmd.getImageThread(); - if (rgba8 != null) { - image = rgba8; - } - } - - final ui.ImmutableBuffer buffer = await ui.ImmutableBuffer.fromUint8List(image.toUint8List()); - - final ui.ImageDescriptor id = - ui.ImageDescriptor.raw(buffer, height: image.height, width: image.width, pixelFormat: ui.PixelFormat.rgba8888); - - final ui.Codec codec = await id.instantiateCodec(targetHeight: image.height, targetWidth: image.width); - - final ui.FrameInfo fi = await codec.getNextFrame(); - final ui.Image uiImage = fi.image; - - return uiImage; - } - @override Widget build(BuildContext context) { return Scaffold( @@ -72,23 +46,33 @@ class _StreamScreenState extends State { title: Text(widget.resourceName.name.toUpperCase()), ), body: Center( - child: Column( - children: [ - const SizedBox(height: 16), - Text( - '${widget.resourceName.namespace}:${widget.resourceName.type}:${widget.resourceName.subtype}/${widget.resourceName.name}', - style: const TextStyle(fontWeight: FontWeight.w300), - ), - const SizedBox(height: 16), - ViamCameraStreamView(camera: widget.camera, streamClient: widget.client), - const SizedBox(height: 16), - if (_imgLoaded) Image.memory(Uint8List.view(imageBytes!.buffer), scale: 3), - const SizedBox(height: 16), - ElevatedButton( - child: const Text('Get image'), - onPressed: () => _getImage(), - ) - ], + child: SingleChildScrollView( + child: Column( + children: [ + const SizedBox(height: 16), + Text( + '${widget.resourceName.namespace}:${widget.resourceName.type}:${widget.resourceName.subtype}/${widget.resourceName.name}', + style: const TextStyle(fontWeight: FontWeight.w300), + ), + const SizedBox(height: 16), + const Text('Live Camera Feed'), + ViamCameraStreamView(camera: widget.camera, streamClient: widget.client), + const SizedBox(height: 16), + const Text('Refresh every 1 second'), + Center(child: ViamCameraStreamVariableRefresh(camera: widget.camera, frequency: 1)), + const SizedBox(height: 16), + const Text('Static image'), + ViamCameraImage(camera: widget.camera), + const SizedBox(height: 16), + ElevatedButton( + child: const Text('Manually refresh image'), + onPressed: () => _getImage(), + ), + const SizedBox(height: 16), + if (_imgLoaded) Image.memory(_imageData!), + const SizedBox(height: 36), + ], + ), ), ), ); diff --git a/lib/src/components/camera/camera.dart b/lib/src/components/camera/camera.dart index c4e1499a75..c7fdb92d53 100644 --- a/lib/src/components/camera/camera.dart +++ b/lib/src/components/camera/camera.dart @@ -1,3 +1,5 @@ +import 'dart:typed_data'; + import '../../gen/common/v1/common.pb.dart'; import '../../gen/component/camera/v1/camera.pb.dart'; import '../../media/image.dart'; @@ -20,6 +22,14 @@ abstract class Camera extends Resource { /// Get the camera's intrinsic parameters and the camera's distortion parameters. Future properties(); + /// Get the next image from the camera. + /// + /// This can then be wrapped in an Image widget such as: + /// ```dart + /// Image.memory(myImageData); + /// ``` + Future imageData({MimeType? mimeType}); + /// Get the [ResourceName] for this [Camera] with the given [name] static ResourceName getResourceName(String name) { return Camera.subtype.getResourceName(name); diff --git a/lib/src/components/camera/client.dart b/lib/src/components/camera/client.dart index eb9d8bb51d..7670f4deb0 100644 --- a/lib/src/components/camera/client.dart +++ b/lib/src/components/camera/client.dart @@ -1,4 +1,8 @@ +import 'dart:typed_data'; +import 'dart:ui' as ui; + import 'package:grpc/grpc_connection_interface.dart'; +import 'package:image/image.dart' as img; import '../../gen/common/v1/common.pb.dart'; import '../../gen/component/camera/v1/camera.pbgrpc.dart'; @@ -54,4 +58,36 @@ class CameraClient extends Camera implements ResourceRPCClient { final response = await client.doCommand(request); return response.result.toMap(); } + + @override + Future imageData({MimeType? mimeType}) async { + final imageFromCamera = await image(mimeType: mimeType); + final convertedImage = await _convertImageToFlutterUi(imageFromCamera.image!); + final png = await convertedImage.toByteData(format: ui.ImageByteFormat.png); + return Uint8List.view(png!.buffer); + } + + Future _convertImageToFlutterUi(img.Image image) async { + if (image.format != img.Format.uint8 || image.numChannels != 4) { + final cmd = img.Command() + ..image(image) + ..convert(format: img.Format.uint8, numChannels: 4); + final rgba8 = await cmd.getImageThread(); + if (rgba8 != null) { + image = rgba8; + } + } + + final ui.ImmutableBuffer buffer = await ui.ImmutableBuffer.fromUint8List(image.toUint8List()); + + final ui.ImageDescriptor id = + ui.ImageDescriptor.raw(buffer, height: image.height, width: image.width, pixelFormat: ui.PixelFormat.rgba8888); + + final ui.Codec codec = await id.instantiateCodec(targetHeight: image.height, targetWidth: image.width); + + final ui.FrameInfo fi = await codec.getNextFrame(); + final ui.Image uiImage = fi.image; + + return uiImage; + } } diff --git a/lib/src/media/image.dart b/lib/src/media/image.dart index 8aad08a19f..799740a1dc 100644 --- a/lib/src/media/image.dart +++ b/lib/src/media/image.dart @@ -89,7 +89,7 @@ class MimeType { } } -/// A custom image type that contains the [MimeTYpe], raw image data, and lazily loads and caches an [img.Image]. +/// A custom image type that contains the [MimeType], raw image data, and lazily loads and caches an [img.Image]. class ViamImage { /// The mimetype of the image final MimeType mimeType; diff --git a/lib/widgets/camera_stream.dart b/lib/widgets/camera_stream.dart index d61d04d78b..6ff4421d3b 100644 --- a/lib/widgets/camera_stream.dart +++ b/lib/widgets/camera_stream.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:typed_data'; import 'package:flutter/material.dart'; import 'package:flutter_webrtc/flutter_webrtc.dart'; @@ -93,3 +94,105 @@ class _ViamCameraStreamViewState extends State { })); } } + +/// a widget for rendering images from a [Camera] at a set [frequency] +class ViamCameraStreamVariableRefresh extends StatefulWidget { + const ViamCameraStreamVariableRefresh({super.key, required this.camera, required this.frequency}); + + /// the Viam [Camera] from which to capture images from + final Camera camera; + + /// frequency in seconds for the image to be refreshed + final int frequency; + + @override + State createState() => _ViamCameraStreamVariableRefreshState(); +} + +class _ViamCameraStreamVariableRefreshState extends State { + late Uint8List _imageData = Uint8List(0); + late Uint8List _oldImageData = Uint8List(0); + bool _loading = true; + bool _error = false; + + @override + void initState() { + _refreshLoop(); + super.initState(); + } + + Future _refreshLoop() async { + while (mounted) { + try { + setState(() { + _oldImageData = _imageData; + }); + + _imageData = await widget.camera.imageData(); + if (!mounted) break; + setState(() { + _loading = false; + _error = false; + }); + } catch (e) { + setState(() { + _error = true; + }); + } + + await Future.delayed(Duration(seconds: widget.frequency)); + } + } + + @override + Widget build(BuildContext context) { + return Container( + child: _loading + ? Stack( + children: [ + const Align(child: CircularProgressIndicator.adaptive()), + if (_error) const Icon(Icons.error, color: Colors.red), + ], + ) + : Stack( + children: [ + // stack two Images like this to avoid a flicker when one image is updating + if (_oldImageData.isNotEmpty) Image.memory(_oldImageData), + Image.memory(_imageData), + if (_error) const Icon(Icons.error, color: Colors.red), + ], + ), + ); + } +} + +/// a widget that fetches and shows a single image from a [Camera]. +class ViamCameraImage extends StatefulWidget { + final Camera camera; + + const ViamCameraImage({super.key, required this.camera}); + + @override + State createState() => _ViamCameraImageState(); +} + +class _ViamCameraImageState extends State { + late Uint8List _imageData = Uint8List(0); + + @override + void initState() { + _fetchImage(); + super.initState(); + } + + Future _fetchImage() async { + setState(() async { + _imageData = await widget.camera.imageData(); + }); + } + + @override + Widget build(BuildContext context) { + return (_imageData.isEmpty) ? Container() : Image.memory(_imageData); + } +} diff --git a/test/unit_test/components/camera_test.dart b/test/unit_test/components/camera_test.dart index 9411573c90..59e8147b26 100644 --- a/test/unit_test/components/camera_test.dart +++ b/test/unit_test/components/camera_test.dart @@ -1,3 +1,5 @@ +import 'dart:typed_data'; + import 'package:flutter_test/flutter_test.dart'; import 'package:grpc/grpc.dart'; import 'package:viam_sdk/src/components/camera/service.dart'; @@ -8,6 +10,54 @@ import 'package:viam_sdk/viam_sdk.dart'; import '../../test_utils.dart'; +// originally we were testing with the [0,0,0] but that started breaking the converters +// inside the imageData() method and throwing exceptions from the Image package. +// So in order to test imageData() we have to use bytes for a valid image, +// This list of ints is from a valid 1x1 pixel jpeg which then I ran a script to convert +// into bytes and then saved here. +const validJpeg = [ + 255, 216, 255, 224, 0, 16, 74, 70, 73, 70, 0, 1, 2, 1, 0, 72, 0, + 72, 0, 0, 255, 219, 0, 67, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 255, 219, 0, 67, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 255, 192, 0, 17, 8, 0, 1, 0, 1, 3, 1, 17, 0, 2, 17, 1, 3, 17, 1, 255, 196, 0, 31, + 0, 0, 1, 5, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 255, 196, 0, 181, 16, 0, 2, 1, 3, 3, 2, 4, 3, 5, 5, 4, 4, 0, 0, 1, 125, 1, 2, 3, 0, 4, + 17, 5, 18, 33, 49, 65, 6, 19, 81, 97, 7, 34, 113, 20, 50, 129, 145, 161, 8, 35, 66, 177, + 193, 21, 82, 209, 240, 36, 51, 98, 114, 130, 9, 10, 22, 23, 24, 25, 26, 37, 38, 39, 40, + 41, 42, 52, 53, 54, 55, 56, 57, 58, 67, 68, 69, 70, 71, 72, 73, 74, 83, 84, 85, 86, 87, + 88, 89, 90, 99, 100, 101, 102, 103, 104, 105, 106, 115, 116, 117, 118, 119, 120, 121, + 122, 131, 132, 133, 134, 135, 136, 137, 138, 146, 147, 148, 149, 150, 151, 152, 153, + 154, 162, 163, 164, 165, 166, 167, 168, 169, 170, 178, 179, 180, 181, 182, 183, 184, + 185, 186, 194, 195, 196, 197, 198, 199, 200, 201, 202, 210, 211, 212, 213, 214, 215, + 216, 217, 218, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 241, 242, 243, 244, + 245, 246, 247, 248, 249, 250, 255, 196, 0, 31, 1, 0, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, + 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 255, 196, 0, 181, 17, 0, 2, 1, 2, 4, + 4, 3, 4, 7, 5, 4, 4, 0, 1, 2, 119, 0, 1, 2, 3, 17, 4, 5, 33, 49, 6, 18, 65, 81, 7, 97, + 113, 19, 34, 50, 129, 8, 20, 66, 145, 161, 177, 193, 9, 35, 51, 82, 240, 21, 98, 114, + 209, 10, 22, 36, 52, 225, 37, 241, 23, 24, 25, 26, 38, 39, 40, 41, 42, 53, 54, 55, 56, + 57, 58, 67, 68, 69, 70, 71, 72, 73, 74, 83, 84, 85, 86, 87, 88, 89, 90, 99, 100, 101, + 102, 103, 104, 105, 106, 115, 116, 117, 118, 119, 120, 121, 122, 130, 131, 132, 133, + 134, 135, 136, 137, 138, 146, 147, 148, 149, 150, 151, 152, 153, 154, 162, 163, 164, + 165, 166, 167, 168, 169, 170, 178, 179, 180, 181, 182, 183, 184, 185, 186, 194, 195, + 196, 197, 198, 199, 200, 201, 202, 210, 211, 212, 213, 214, 215, 216, 217, 218, 226, + 227, 228, 229, 230, 231, 232, 233, 234, 242, 243, 244, 245, 246, 247, 248, 249, 250, + 255, 218, 0, 12, 3, 1, 0, 2, 17, 3, 17, 0, 63, 0, 254, 178, 40, 0, 255, 217, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 255, 196, 0, 181, + 17, 0, 2, 1, 2, 4, 4, 3, 4, 7, 5, 4, 4, 0, 1, 2, 119, 0, 1, 2, 3, 17, 4, 5, 33, 49, 6, 18, + 65, 81, 7, 97, 113, 19, 34, 50, 129, 8, 20, 66, 145, 161, 177, 193, 9, 35, 51, 82, 240, 21, + 98, 114, 209, 10, 22, 36, 52, 225, 37, 241, 23, 24, 25, 26, 38, 39, 40, 41, 42, 53, 54, 55, + 56, 57, 58, 67, 68, 69, 70, 71, 72, 73, 74, 83, 84, 85, 86, 87, 88, 89, 90, 99, 100, 101, 102, + 103, 104, 105, 106, 115, 116, 117, 118, 119, 120, 121, 122, 130, 131, 132, 133, 134, 135, 136, + 137, 138, 146, 147, 148, 149, 150, 151, 152, 153, 154, 162, 163, 164, 165, 166, 167, 168, 169, + 170, 178, 179, 180, 181, 182, 183, 184, 185, 186, 194, 195, 196, 197, 198, 199, 200, 201, 202, + 210, 211, 212, 213, 214, 215, 216, 217, 218, 226, 227, 228, 229, 230, 231, 232, 233, 234, 242, + 243, 244, 245, 246, 247, 248, 249, 250, 255, 218, 0, 12, 3, 1, 0, 2, 17, 3, 17, 0, 63, 0, 254, + 178, 40, 0, 255, 217 // prevent dartfmt +]; + class FakeCamera extends Camera { Map? extra; @@ -26,7 +76,8 @@ class FakeCamera extends Camera { if (mimeType == null) { throw const GrpcError.invalidArgument('invalid mimetype'); } - return ViamImage([0, 0, 0], mimeType); + + return ViamImage(validJpeg, mimeType); } @override @@ -41,10 +92,15 @@ class FakeCamera extends Camera { ..intrinsicParameters = (IntrinsicParameters()..widthPx = 10) ..distortionParameters = (DistortionParameters()..model = 'test'); } + + @override + Future imageData({MimeType? mimeType}) async { + return Uint8List(0); + } } void main() { - group('Camera Tests', () { + group('FakeCamera Tests', () { const String name = 'camera'; late FakeCamera camera; @@ -55,11 +111,11 @@ void main() { test('image', () async { final actualJpeg = await camera.image(mimeType: MimeType.jpeg); expect(actualJpeg.mimeType, MimeType.jpeg); - expect(actualJpeg.raw, [0, 0, 0]); + expect(actualJpeg.raw, validJpeg); final actualPng = await camera.image(mimeType: MimeType.png); expect(actualPng.mimeType, MimeType.png); - expect(actualPng.raw, [0, 0, 0]); + expect(actualPng.raw, validJpeg); }); test('pointCloud', () async { @@ -74,6 +130,11 @@ void main() { expect(actual.intrinsicParameters.widthPx, 10); }); + test('imageData', () async { + final actual = await camera.imageData(); + expect(actual, Uint8List(0)); + }); + test('doCommand', () async { final cmd = {'foo': 'bar'}; final resp = await camera.doCommand(cmd); @@ -113,7 +174,7 @@ void main() { final actualJpeg = await client.getImage(jpegRequest); expect(actualJpeg.mimeType, 'jpeg'); - expect(actualJpeg.image, [0, 0, 0]); + expect(actualJpeg.image, validJpeg); final pngRequest = GetImageRequest() ..name = name @@ -121,7 +182,7 @@ void main() { final actualPng = await client.getImage(pngRequest); expect(actualPng.mimeType, 'png'); - expect(actualPng.image, [0, 0, 0]); + expect(actualPng.image, validJpeg); }); test('pointCloud', () async { @@ -152,11 +213,11 @@ void main() { final client = CameraClient(name, channel); final actualJpeg = await client.image(mimeType: MimeType.jpeg); expect(actualJpeg.mimeType, MimeType.jpeg); - expect(actualJpeg.raw, [0, 0, 0]); + expect(actualJpeg.raw, validJpeg); final actualPng = await client.image(mimeType: MimeType.png); expect(actualPng.mimeType, MimeType.png); - expect(actualPng.raw, [0, 0, 0]); + expect(actualPng.raw, validJpeg); }); test('pointCloud', () async { @@ -173,6 +234,20 @@ void main() { expect(actual.intrinsicParameters.widthPx, 10); }); + test('imageData', () async { + final client = CameraClient(name, channel); + final actual = await client.imageData(mimeType: MimeType.jpeg); + final validImageData = [ + 137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, + 13, 73, 72, 68, 82, 0, 0, 0, 1, 0, 0, 0, 1, 8, 6, 0, 0, 0, + 31, 21, 196, 137, 0, 0, 0, 4, 115, 66, 73, 84, 8, 8, 8, 8, + 124, 8, 100, 136, 0, 0, 0, 13, 73, 68, 65, 84, 8, 153, 99, + 184, 121, 243, 230, 127, 0, 8, 165, 3, 139, 65, 53, 234, + 255, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130 // prevent dartFmt + ]; + expect(actual, validImageData); + }); + test('doCommand', () async { final client = CameraClient(name, channel); final cmd = {'foo': 'bar'};