Add image data api

example/viam_robot_example_app/lib/screens/stream.dart

@@ -1,8 +1,6 @@
 import 'dart:typed_data';
-import 'dart:ui' as ui;
 
 import 'package:flutter/material.dart';
-import 'package:image/image.dart' as img;
 import 'package:viam_sdk/viam_sdk.dart';
 import 'package:viam_sdk/widgets.dart';
 
@@ -21,74 +19,60 @@ class StreamScreen extends StatefulWidget {
 
 class _StreamScreenState extends State<StreamScreen> {
   // Single frame
-  ByteData? imageBytes;
+  Uint8List? _imageData;
   bool _imgLoaded = false;
 
-  void _getImage() {
+  @override
+  void initState() {
+    _getImage();
+    super.initState();
+  }
+
+  Future<void> _getImage() async {
     setState(() {
       _imgLoaded = false;
     });
-    final imageFut = widget.camera.image();
-    imageFut.then((value) {
-      final convertFut = convertImageToFlutterUi(value.image ?? img.Image.empty());
-      convertFut.then((value) {
-        final pngFut = value.toByteData(format: ui.ImageByteFormat.png);
-        pngFut.then((value) => setState(() {
-              imageBytes = value;
-              _imgLoaded = true;
-            }));
-      });
+    final imageDataResponse = await widget.camera.imageData();
+    setState(() {
+      _imageData = imageDataResponse;
+      _imgLoaded = true;
     });
   }
 
-  Future<ui.Image> convertImageToFlutterUi(img.Image image) async {
-    if (image.format != img.Format.uint8 || image.numChannels != 4) {
-      final cmd = img.Command()
-        ..image(image)
-        ..convert(format: img.Format.uint8, numChannels: 4);
-      final rgba8 = await cmd.getImageThread();
-      if (rgba8 != null) {
-        image = rgba8;
-      }
-    }
-
-    final ui.ImmutableBuffer buffer = await ui.ImmutableBuffer.fromUint8List(image.toUint8List());
-
-    final ui.ImageDescriptor id =
-        ui.ImageDescriptor.raw(buffer, height: image.height, width: image.width, pixelFormat: ui.PixelFormat.rgba8888);
-
-    final ui.Codec codec = await id.instantiateCodec(targetHeight: image.height, targetWidth: image.width);
-
-    final ui.FrameInfo fi = await codec.getNextFrame();
-    final ui.Image uiImage = fi.image;
-
-    return uiImage;
-  }
-
   @override
   Widget build(BuildContext context) {
     return Scaffold(
       appBar: AppBar(
         title: Text(widget.resourceName.name.toUpperCase()),
       ),
       body: Center(
-        child: Column(
-          children: [
-            const SizedBox(height: 16),
-            Text(
-              '${widget.resourceName.namespace}:${widget.resourceName.type}:${widget.resourceName.subtype}/${widget.resourceName.name}',
-              style: const TextStyle(fontWeight: FontWeight.w300),
-            ),
-            const SizedBox(height: 16),
-            ViamCameraStreamView(camera: widget.camera, streamClient: widget.client),
-            const SizedBox(height: 16),
-            if (_imgLoaded) Image.memory(Uint8List.view(imageBytes!.buffer), scale: 3),
-            const SizedBox(height: 16),
-            ElevatedButton(
-              child: const Text('Get image'),
-              onPressed: () => _getImage(),
-            )
-          ],
+        child: SingleChildScrollView(
+          child: Column(
+            children: [
+              const SizedBox(height: 16),
+              Text(
+                '${widget.resourceName.namespace}:${widget.resourceName.type}:${widget.resourceName.subtype}/${widget.resourceName.name}',
+                style: const TextStyle(fontWeight: FontWeight.w300),
+              ),
+              const SizedBox(height: 16),
+              const Text('Live Camera Feed'),
+              ViamCameraStreamView(camera: widget.camera, streamClient: widget.client),
+              const SizedBox(height: 16),
+              const Text('Refresh every 1 second'),
+              Center(child: ViamCameraStreamVariableRefresh(camera: widget.camera, frequency: 1)),
+              const SizedBox(height: 16),
+              const Text('Static image'),
+              ViamCameraImage(camera: widget.camera),
+              const SizedBox(height: 16),
+              ElevatedButton(
+                child: const Text('Manually refresh image'),
+                onPressed: () => _getImage(),
+              ),
+              const SizedBox(height: 16),
+              if (_imgLoaded) Image.memory(_imageData!),
+              const SizedBox(height: 36),
+            ],
+          ),
         ),
       ),
     );

lib/src/components/camera/camera.dart

@@ -1,3 +1,5 @@
+import 'dart:typed_data';
+
 import '../../gen/common/v1/common.pb.dart';
 import '../../gen/component/camera/v1/camera.pb.dart';
 import '../../media/image.dart';
@@ -20,6 +22,14 @@ abstract class Camera extends Resource {
   /// Get the camera's intrinsic parameters and the camera's distortion parameters.
   Future<CameraProperties> properties();
 
+  /// Get the next image from the camera.
+  ///
+  /// This can then be wrapped in an Image widget such as:
+  /// ```dart
+  /// Image.memory(myImageData);
+  /// ```
+  Future<Uint8List> imageData({MimeType? mimeType});
+
   /// Get the [ResourceName] for this [Camera] with the given [name]
   static ResourceName getResourceName(String name) {
     return Camera.subtype.getResourceName(name);

lib/src/components/camera/client.dart

@@ -1,4 +1,8 @@
+import 'dart:typed_data';
+import 'dart:ui' as ui;
+
 import 'package:grpc/grpc_connection_interface.dart';
+import 'package:image/image.dart' as img;
 
 import '../../gen/common/v1/common.pb.dart';
 import '../../gen/component/camera/v1/camera.pbgrpc.dart';
@@ -54,4 +58,36 @@ class CameraClient extends Camera implements ResourceRPCClient {
     final response = await client.doCommand(request);
     return response.result.toMap();
   }
+
+  @override
+  Future<Uint8List> imageData({MimeType? mimeType}) async {
+    final imageFromCamera = await image(mimeType: mimeType);
+    final convertedImage = await _convertImageToFlutterUi(imageFromCamera.image!);
+    final png = await convertedImage.toByteData(format: ui.ImageByteFormat.png);
+    return Uint8List.view(png!.buffer);
+  }
+
+  Future<ui.Image> _convertImageToFlutterUi(img.Image image) async {
+    if (image.format != img.Format.uint8 || image.numChannels != 4) {
+      final cmd = img.Command()
+        ..image(image)
+        ..convert(format: img.Format.uint8, numChannels: 4);
+      final rgba8 = await cmd.getImageThread();
+      if (rgba8 != null) {
+        image = rgba8;
+      }
+    }
+
+    final ui.ImmutableBuffer buffer = await ui.ImmutableBuffer.fromUint8List(image.toUint8List());
+
+    final ui.ImageDescriptor id =
+        ui.ImageDescriptor.raw(buffer, height: image.height, width: image.width, pixelFormat: ui.PixelFormat.rgba8888);
+
+    final ui.Codec codec = await id.instantiateCodec(targetHeight: image.height, targetWidth: image.width);
+
+    final ui.FrameInfo fi = await codec.getNextFrame();
+    final ui.Image uiImage = fi.image;
+
+    return uiImage;
+  }
 }

lib/src/media/image.dart

@@ -89,7 +89,7 @@ class MimeType {
   }
 }
 
-/// A custom image type that contains the [MimeTYpe], raw image data, and lazily loads and caches an [img.Image].
+/// A custom image type that contains the [MimeType], raw image data, and lazily loads and caches an [img.Image].
 class ViamImage {
   /// The mimetype of the image
   final MimeType mimeType;

lib/widgets/camera_stream.dart

@@ -1,4 +1,5 @@
 import 'dart:async';
+import 'dart:typed_data';
 
 import 'package:flutter/material.dart';
 import 'package:flutter_webrtc/flutter_webrtc.dart';
@@ -93,3 +94,105 @@ class _ViamCameraStreamViewState extends State<ViamCameraStreamView> {
           }));
   }
 }
+
+/// a widget for rendering images from a [Camera] at a set [frequency]
+class ViamCameraStreamVariableRefresh extends StatefulWidget {
+  const ViamCameraStreamVariableRefresh({super.key, required this.camera, required this.frequency});
+
+  /// the Viam [Camera] from which to capture images from
+  final Camera camera;
+
+  /// frequency in seconds for the image to be refreshed
+  final int frequency;
+
+  @override
+  State<ViamCameraStreamVariableRefresh> createState() => _ViamCameraStreamVariableRefreshState();
+}
+
+class _ViamCameraStreamVariableRefreshState extends State<ViamCameraStreamVariableRefresh> {
+  late Uint8List _imageData = Uint8List(0);
+  late Uint8List _oldImageData = Uint8List(0);
+  bool _loading = true;
+  bool _error = false;
+
+  @override
+  void initState() {
+    _refreshLoop();
+    super.initState();
+  }
+
+  Future<void> _refreshLoop() async {
+    while (mounted) {
+      try {
+        setState(() {
+          _oldImageData = _imageData;
+        });
+
+        _imageData = await widget.camera.imageData();
+        if (!mounted) break;
+        setState(() {
+          _loading = false;
+          _error = false;
+        });
+      } catch (e) {
+        setState(() {
+          _error = true;
+        });
+      }
+
+      await Future.delayed(Duration(seconds: widget.frequency));
+    }
+  }
+
+  @override
+  Widget build(BuildContext context) {
+    return Container(
+      child: _loading
+          ? Stack(
+              children: [
+                const Align(child: CircularProgressIndicator.adaptive()),
+                if (_error) const Icon(Icons.error, color: Colors.red),
+              ],
+            )
+          : Stack(
+              children: [
+                // stack two Images like this to avoid a flicker when one image is updating
+                if (_oldImageData.isNotEmpty) Image.memory(_oldImageData),
+                Image.memory(_imageData),
+                if (_error) const Icon(Icons.error, color: Colors.red),
+              ],
+            ),
+    );
+  }
+}
+
+/// a widget that fetches and shows a single image from a [Camera].
+class ViamCameraImage extends StatefulWidget {
+  final Camera camera;
+
+  const ViamCameraImage({super.key, required this.camera});
+
+  @override
+  State<ViamCameraImage> createState() => _ViamCameraImageState();
+}
+
+class _ViamCameraImageState extends State<ViamCameraImage> {
+  late Uint8List _imageData = Uint8List(0);
+
+  @override
+  void initState() {
+    _fetchImage();
+    super.initState();
+  }
+
+  Future<void> _fetchImage() async {
+    setState(() async {
+      _imageData = await widget.camera.imageData();
+    });
+  }
+
+  @override
+  Widget build(BuildContext context) {
+    return (_imageData.isEmpty) ? Container() : Image.memory(_imageData);
+  }
+}