From 692bc919aa2c9e48a8cf8b172290eb3f3f827886 Mon Sep 17 00:00:00 2001 From: Naveed Jooma Date: Wed, 17 Jan 2024 18:11:57 -0500 Subject: [PATCH] Enable ViamImage image provider --- example/viam_robot_example_app/lib/main.dart | 10 +- .../lib/screens/stream.dart | 57 +++------- example/viam_robot_example_app/macos/Podfile | 2 +- .../macos/Runner.xcodeproj/project.pbxproj | 10 +- .../xcshareddata/xcschemes/Runner.xcscheme | 8 +- lib/src/media/image.dart | 2 +- lib/widgets.dart | 1 + lib/widgets/image_view.dart | 103 ++++++++++++++++++ 8 files changed, 135 insertions(+), 58 deletions(-) create mode 100644 lib/widgets/image_view.dart diff --git a/example/viam_robot_example_app/lib/main.dart b/example/viam_robot_example_app/lib/main.dart index 33dc24f8e6..05687cee9c 100644 --- a/example/viam_robot_example_app/lib/main.dart +++ b/example/viam_robot_example_app/lib/main.dart @@ -55,12 +55,14 @@ class _MyHomePageState extends State { }); // Be sure to create a .env file with these fields + final opts = RobotClientOptions.withApiKey( + dotenv.env['API_KEY_ID']!, + dotenv.env['API_KEY']!, + ); + opts.dialOptions.attemptMdns = false; _robot = await RobotClient.atAddress( dotenv.env['ROBOT_LOCATION']!, - RobotClientOptions.withApiKey( - dotenv.env['API_KEY_ID']!, - dotenv.env['API_KEY']!, - ), + opts, ); final services = _robot.resourceNames.where((element) => element.type == resourceTypeService); diff --git a/example/viam_robot_example_app/lib/screens/stream.dart b/example/viam_robot_example_app/lib/screens/stream.dart index 2bf317e64e..8aff036711 100644 --- a/example/viam_robot_example_app/lib/screens/stream.dart +++ b/example/viam_robot_example_app/lib/screens/stream.dart @@ -1,8 +1,4 @@ -import 'dart:typed_data'; -import 'dart:ui' as ui; - import 'package:flutter/material.dart'; -import 'package:image/image.dart' as img; import 'package:viam_sdk/viam_sdk.dart'; import 'package:viam_sdk/widgets.dart'; @@ -21,57 +17,27 @@ class StreamScreen extends StatefulWidget { class _StreamScreenState extends State { // Single frame - ByteData? imageBytes; - bool _imgLoaded = false; + late Future _viamImage; + + @override + void initState() { + super.initState(); + _getImage(); + } void _getImage() { setState(() { - _imgLoaded = false; - }); - final imageFut = widget.camera.image(); - imageFut.then((value) { - final convertFut = convertImageToFlutterUi(value.image ?? img.Image.empty()); - convertFut.then((value) { - final pngFut = value.toByteData(format: ui.ImageByteFormat.png); - pngFut.then((value) => setState(() { - imageBytes = value; - _imgLoaded = true; - })); - }); + _viamImage = widget.camera.image(); }); } - Future convertImageToFlutterUi(img.Image image) async { - if (image.format != img.Format.uint8 || image.numChannels != 4) { - final cmd = img.Command() - ..image(image) - ..convert(format: img.Format.uint8, numChannels: 4); - final rgba8 = await cmd.getImageThread(); - if (rgba8 != null) { - image = rgba8; - } - } - - final ui.ImmutableBuffer buffer = await ui.ImmutableBuffer.fromUint8List(image.toUint8List()); - - final ui.ImageDescriptor id = - ui.ImageDescriptor.raw(buffer, height: image.height, width: image.width, pixelFormat: ui.PixelFormat.rgba8888); - - final ui.Codec codec = await id.instantiateCodec(targetHeight: image.height, targetWidth: image.width); - - final ui.FrameInfo fi = await codec.getNextFrame(); - final ui.Image uiImage = fi.image; - - return uiImage; - } - @override Widget build(BuildContext context) { return Scaffold( appBar: AppBar( title: Text(widget.resourceName.name.toUpperCase()), ), - body: Center( + body: SingleChildScrollView( child: Column( children: [ const SizedBox(height: 16), @@ -82,7 +48,10 @@ class _StreamScreenState extends State { const SizedBox(height: 16), ViamCameraStreamView(camera: widget.camera, streamClient: widget.client), const SizedBox(height: 16), - if (_imgLoaded) Image.memory(Uint8List.view(imageBytes!.buffer), scale: 3), + FutureBuilder( + future: _viamImage, + builder: ((context, snapshot) => + snapshot.hasData ? Image(image: snapshot.data!.imageProvider) : const CircularProgressIndicator.adaptive())), const SizedBox(height: 16), ElevatedButton( child: const Text('Get image'), diff --git a/example/viam_robot_example_app/macos/Podfile b/example/viam_robot_example_app/macos/Podfile index 049abe2954..c637ce5c19 100644 --- a/example/viam_robot_example_app/macos/Podfile +++ b/example/viam_robot_example_app/macos/Podfile @@ -1,4 +1,4 @@ -platform :osx, '10.14' +platform :osx, '11' # CocoaPods analytics sends network stats synchronously affecting flutter build latency. ENV['COCOAPODS_DISABLE_STATS'] = 'true' diff --git a/example/viam_robot_example_app/macos/Runner.xcodeproj/project.pbxproj b/example/viam_robot_example_app/macos/Runner.xcodeproj/project.pbxproj index 1a636d755b..3ec8af27fa 100644 --- a/example/viam_robot_example_app/macos/Runner.xcodeproj/project.pbxproj +++ b/example/viam_robot_example_app/macos/Runner.xcodeproj/project.pbxproj @@ -56,7 +56,7 @@ 15242523DB1FCDEE32F203C2 /* Pods_Runner.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_Runner.framework; sourceTree = BUILT_PRODUCTS_DIR; }; 333000ED22D3DE5D00554162 /* Warnings.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Warnings.xcconfig; sourceTree = ""; }; 335BBD1A22A9A15E00E9071D /* GeneratedPluginRegistrant.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = GeneratedPluginRegistrant.swift; sourceTree = ""; }; - 33CC10ED2044A3C60003C045 /* viam_example_app.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = viam_example_app.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 33CC10ED2044A3C60003C045 /* Viam Example App.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "Viam Example App.app"; sourceTree = BUILT_PRODUCTS_DIR; }; 33CC10F02044A3C60003C045 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; 33CC10F22044A3C60003C045 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; name = Assets.xcassets; path = Runner/Assets.xcassets; sourceTree = ""; }; 33CC10F52044A3C60003C045 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.xib; name = Base; path = Base.lproj/MainMenu.xib; sourceTree = ""; }; @@ -112,7 +112,7 @@ 33CC10EE2044A3C60003C045 /* Products */ = { isa = PBXGroup; children = ( - 33CC10ED2044A3C60003C045 /* viam_example_app.app */, + 33CC10ED2044A3C60003C045 /* Viam Example App.app */, ); name = Products; sourceTree = ""; @@ -159,7 +159,6 @@ 670B361EC221A983BEC9B00C /* Pods-Runner.release.xcconfig */, CD7EFC0D75004BBCC6F28023 /* Pods-Runner.profile.xcconfig */, ); - name = Pods; path = Pods; sourceTree = ""; }; @@ -193,7 +192,7 @@ ); name = Runner; productName = Runner; - productReference = 33CC10ED2044A3C60003C045 /* viam_example_app.app */; + productReference = 33CC10ED2044A3C60003C045 /* Viam Example App.app */; productType = "com.apple.product-type.application"; }; /* End PBXNativeTarget section */ @@ -427,6 +426,7 @@ "$(inherited)", "@executable_path/../Frameworks", ); + MACOSX_DEPLOYMENT_TARGET = 11.0; PROVISIONING_PROFILE_SPECIFIER = ""; SWIFT_VERSION = 5.0; }; @@ -553,6 +553,7 @@ "$(inherited)", "@executable_path/../Frameworks", ); + MACOSX_DEPLOYMENT_TARGET = 11.0; PROVISIONING_PROFILE_SPECIFIER = ""; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; SWIFT_VERSION = 5.0; @@ -573,6 +574,7 @@ "$(inherited)", "@executable_path/../Frameworks", ); + MACOSX_DEPLOYMENT_TARGET = 11.0; PROVISIONING_PROFILE_SPECIFIER = ""; SWIFT_VERSION = 5.0; }; diff --git a/example/viam_robot_example_app/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme b/example/viam_robot_example_app/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme index 880af5120f..b765e76d68 100644 --- a/example/viam_robot_example_app/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme +++ b/example/viam_robot_example_app/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme @@ -15,7 +15,7 @@ @@ -31,7 +31,7 @@ @@ -54,7 +54,7 @@ @@ -71,7 +71,7 @@ diff --git a/lib/src/media/image.dart b/lib/src/media/image.dart index 8aad08a19f..799740a1dc 100644 --- a/lib/src/media/image.dart +++ b/lib/src/media/image.dart @@ -89,7 +89,7 @@ class MimeType { } } -/// A custom image type that contains the [MimeTYpe], raw image data, and lazily loads and caches an [img.Image]. +/// A custom image type that contains the [MimeType], raw image data, and lazily loads and caches an [img.Image]. class ViamImage { /// The mimetype of the image final MimeType mimeType; diff --git a/lib/widgets.dart b/lib/widgets.dart index cd0dc79233..e86b161912 100644 --- a/lib/widgets.dart +++ b/lib/widgets.dart @@ -1,5 +1,6 @@ export 'widgets/button.dart'; export 'widgets/camera_stream.dart'; +export 'widgets/image_view.dart'; export 'widgets/joystick.dart'; export 'widgets/multi_camera_stream.dart'; export 'widgets/refreshable_data_table.dart'; diff --git a/lib/widgets/image_view.dart b/lib/widgets/image_view.dart new file mode 100644 index 0000000000..7ef5c69b0d --- /dev/null +++ b/lib/widgets/image_view.dart @@ -0,0 +1,103 @@ +import 'dart:async'; +import 'dart:ui' as ui; + +import 'package:flutter/foundation.dart'; +import 'package:flutter/material.dart'; +import 'package:image/image.dart' as img; +import 'package:viam_sdk/viam_sdk.dart'; + +/// Display a ViamImage inside Flutter's [Image] widget using this provider +/// as the image value. +/// +/// ```dart +/// Image(image: viamImage.imageProvider); +/// ``` +class ViamImageProvider extends ImageProvider { + const ViamImageProvider(this.image); + + final ViamImage image; + + @override + Future obtainKey(ImageConfiguration configuration) { + return SynchronousFuture(image); + } + + @override + ImageStreamCompleter loadImage(ViamImage key, ImageDecoderCallback decode) { + return MultiFrameImageStreamCompleter( + codec: key.uiData + .then((value) { + if (value == null) { + throw Exception('Unable to convert image to displayable format'); + } + return value; + }) + .catchError((Object e, StackTrace stack) { + scheduleMicrotask(() { + PaintingBinding.instance.imageCache.evict(key); + }); + return Future.error(e, stack); + }) + .then(ui.ImmutableBuffer.fromUint8List) + .then(decode), + scale: 1.0); + } +} + +extension UI on img.Image { + /// Data that allows [img.Image] to be compatible with dart:ui or Flutter. + /// + /// ```dart + /// img.Image image = ...; + /// final imgData = await image.uiData; + /// Image.memory(imgData); + /// ``` + Future get uiData async { + img.Image image = this; + if (image.format != img.Format.uint8 || image.numChannels != 4) { + final cmd = img.Command() + ..image(this) + ..convert(format: img.Format.uint8, numChannels: 4); + final rgba8 = await cmd.getImageThread(); + if (rgba8 != null) { + image = rgba8; + } + } + + final ui.ImmutableBuffer buffer = await ui.ImmutableBuffer.fromUint8List(image.toUint8List()); + + final ui.ImageDescriptor id = + ui.ImageDescriptor.raw(buffer, height: image.height, width: image.width, pixelFormat: ui.PixelFormat.rgba8888); + + final ui.Codec codec = await id.instantiateCodec(targetHeight: image.height, targetWidth: image.width); + + final ui.FrameInfo fi = await codec.getNextFrame(); + final ui.Image uiImage = fi.image; + + final png = await uiImage.toByteData(format: ui.ImageByteFormat.png); + return Uint8List.view(png!.buffer); + } +} + +extension Flutter on ViamImage { + /// Data that allows [ViamImage] to be compatible with dart:ui or Flutter. + /// + /// ```dart + /// ViamImage image = ...; + /// final imgData = await image.uiData; + /// Image.memory(imgData); + /// ``` + Future get uiData async { + return await image?.uiData; + } + + /// Display a ViamImage inside Flutter's [Image] widget using this provider + /// as the image value. + /// + /// ```dart + /// Image(image: viamImage.imageProvider); + /// ``` + ImageProvider get imageProvider { + return ViamImageProvider(this); + } +}