diff --git a/app/lib/backend/schema/bt_device.dart b/app/lib/backend/schema/bt_device.dart index b14fb7f7e..999518f0a 100644 --- a/app/lib/backend/schema/bt_device.dart +++ b/app/lib/backend/schema/bt_device.dart @@ -3,7 +3,17 @@ import 'package:friend_private/services/device_connections.dart'; import 'package:friend_private/services/frame_connection.dart'; import 'package:friend_private/utils/ble/gatt_utils.dart'; -enum BleAudioCodec { pcm16, pcm8, mulaw16, mulaw8, opus, unknown } +enum BleAudioCodec { + pcm16, + pcm8, + mulaw16, + mulaw8, + opus, + unknown; + + @override + String toString() => mapCodecToName(this); +} String mapCodecToName(BleAudioCodec codec) { switch (codec) { diff --git a/app/lib/main.dart b/app/lib/main.dart index 47daaf1a5..663084819 100644 --- a/app/lib/main.dart +++ b/app/lib/main.dart @@ -164,16 +164,15 @@ class _MyAppState extends State with WidgetsBindingObserver { update: (BuildContext context, value, MessageProvider? previous) => (previous?..updatePluginProvider(value)) ?? MessageProvider(), ), - ChangeNotifierProvider(create: (context) => WebSocketProvider()), - ChangeNotifierProxyProvider3( + ChangeNotifierProxyProvider2( create: (context) => CaptureProvider(), - update: (BuildContext context, memory, message, wsProvider, CaptureProvider? previous) => - (previous?..updateProviderInstances(memory, message, wsProvider)) ?? CaptureProvider(), + update: (BuildContext context, memory, message, CaptureProvider? previous) => + (previous?..updateProviderInstances(memory, message)) ?? CaptureProvider(), ), - ChangeNotifierProxyProvider2( + ChangeNotifierProxyProvider( create: (context) => DeviceProvider(), - update: (BuildContext context, captureProvider, wsProvider, DeviceProvider? previous) => - (previous?..setProviders(captureProvider, wsProvider)) ?? DeviceProvider(), + update: (BuildContext context, captureProvider, DeviceProvider? previous) => + (previous?..setProviders(captureProvider)) ?? DeviceProvider(), ), ChangeNotifierProxyProvider( create: (context) => OnboardingProvider(), @@ -181,10 +180,10 @@ class _MyAppState extends State with WidgetsBindingObserver { (previous?..setDeviceProvider(value)) ?? OnboardingProvider(), ), ListenableProvider(create: (context) => HomeProvider()), - ChangeNotifierProxyProvider3( + ChangeNotifierProxyProvider( create: (context) => SpeechProfileProvider(), - update: (BuildContext context, device, capture, wsProvider, SpeechProfileProvider? previous) => - (previous?..setProviders(device, capture, wsProvider)) ?? SpeechProfileProvider(), + update: (BuildContext context, device, SpeechProfileProvider? previous) => + (previous?..setProviders(device)) ?? SpeechProfileProvider(), ), ChangeNotifierProxyProvider2( create: (context) => MemoryDetailProvider(), diff --git a/app/lib/pages/capture/_page.dart b/app/lib/pages/capture/_page.dart index b392425d0..75d10b227 100644 --- a/app/lib/pages/capture/_page.dart +++ b/app/lib/pages/capture/_page.dart @@ -1,22 +1,6 @@ import 'package:flutter/material.dart'; -import 'package:flutter/scheduler.dart'; -import 'package:flutter_foreground_task/flutter_foreground_task.dart'; -import 'package:flutter_provider_utilities/flutter_provider_utilities.dart'; -import 'package:friend_private/backend/schema/bt_device.dart'; -import 'package:friend_private/backend/schema/geolocation.dart'; -import 'package:friend_private/pages/capture/widgets/widgets.dart'; -import 'package:friend_private/providers/capture_provider.dart'; -import 'package:friend_private/providers/connectivity_provider.dart'; -import 'package:friend_private/providers/device_provider.dart'; -import 'package:friend_private/providers/onboarding_provider.dart'; -import 'package:friend_private/utils/audio/wav_bytes.dart'; -import 'package:friend_private/utils/ble/communication.dart'; -import 'package:friend_private/utils/enums.dart'; -import 'package:friend_private/widgets/dialog.dart'; -import 'package:provider/provider.dart'; - -import '../../providers/websocket_provider.dart'; +@Deprecated("Capture page is deprecated, use @pages > memories > widgets > capture instead.") class CapturePage extends StatefulWidget { const CapturePage({ super.key, @@ -26,252 +10,9 @@ class CapturePage extends StatefulWidget { State createState() => CapturePageState(); } -class CapturePageState extends State with AutomaticKeepAliveClientMixin, WidgetsBindingObserver { - @override - bool get wantKeepAlive => true; - - /// ---- - - // List segments = List.filled(100, '') - // .mapIndexed((i, e) => TranscriptSegment( - // text: - // '''[00:00:00 - 00:02:23] Speaker 0: The tech giants already know these techniques. - // My goal is to unlock their secrets for the benefit of businesses who to design and help users develop healthy habits. - // To that end, there's so much I wanted to put in this book that just didn't fit. Before you reading, please take a moment to download these - // supplementary materials included free with the purchase of this audiobook. Please go to nirandfar.com forward slash hooked. - // Near is spelled like my first name, speck, n I r. Andfar.com/hooked. There you will find the hooked model workbook, an ebook of case studies, - // and a free email course about product psychology. Also, if you'd like to connect with me, you can reach me through my blog at nirafar.com. - // You can schedule office hours to discuss your questions. Look forward to hearing from you as you build habits for good. - // - // Introduction. 79% of smartphone owners check their device within 15 minutes of waking up every morning. Perhaps most startling, - // fully 1 third of Americans say they would rather give up sex than lose their cell phones. A 2011 university study suggested people check their - // phones 34 times per day. However, industry insiders believe that number is closer to an astounding 150 daily sessions. We are hooked. - // It's the poll to visit YouTube, Facebook, or Twitter for just a few minutes only to find yourself still capping and scrolling an hour later. - // It's the urge you likely feel throughout your day but hardly notice. Cognitive psychologists define habits as, quote, automatic behaviors triggered - // by situational cues. Things we do with little or no conscious thought. The products and services we use habitually alter our everyday behavior. - // Just as their designers intended. Our actions have been engineered. How do companies producing little more than bits of code displayed on a screen - // seemingly control users' minds? What makes some products so habit forming? Forming habit is imperative for the survival of many products. - // - // As infinite distractions compete for our attention, companies are learning to master novel tactics that stay relevant in users' minds. - // Amassing millions of users is no longer good enough. Companies increasingly find that their economic value is a function of the strength of the habits they create. - // - // In order to win the loyalty of their users and create a product that's regularly used, companies must learn not only what compels users to click, - // but also what makes them click. Although some companies are just waking up to this new reality, others are already cashing in. By mastering habit - // forming product design, companies profiles in this book make their goods indispensable. First to mind wins. Companies that form strong user habits enjoy - // several benefits to their bottom line. These companies attach their product to internal triggers. A result, users show up without any external prompting. - // Instead of relying on expensive marketing, how did forming companies link their services to users' daily routines and emotions. - // A habit is at work when users feel a tad bored and instantly open Twitter. Feel a hang of loneliness, and before rational thought occurs, - // they're scrolling through their Facebook feeds.''', - // speaker: 'SPEAKER_0${i % 2}', - // isUser: false, - // start: 0, - // end: 10, - // )) - // .toList(); - - setHasTranscripts(bool hasTranscripts) { - context.read().setHasTranscripts(hasTranscripts); - } - - void _onReceiveTaskData(dynamic data) { - if (data is Map) { - if (data.containsKey('latitude') && data.containsKey('longitude')) { - context.read().setGeolocation(Geolocation( - latitude: data['latitude'], - longitude: data['longitude'], - accuracy: data['accuracy'], - altitude: data['altitude'], - time: DateTime.parse(data['time']), - )); - } else { - if (mounted) { - context.read().setGeolocation(null); - } - } - } - } - - @override - void initState() { - WavBytesUtil.clearTempWavFiles(); - - FlutterForegroundTask.addTaskDataCallback(_onReceiveTaskData); - WidgetsBinding.instance.addObserver(this); - SchedulerBinding.instance.addPostFrameCallback((_) async { - // await context.read().processCachedTranscript(); - if (context.read().connectedDevice != null) { - context.read().stopFindDeviceTimer(); - } - // if (await LocationService().displayPermissionsDialog()) { - // await showDialog( - // context: context, - // builder: (c) => getDialog( - // context, - // () => Navigator.of(context).pop(), - // () async { - // await requestLocationPermission(); - // await LocationService().requestBackgroundPermission(); - // if (mounted) Navigator.of(context).pop(); - // }, - // 'Enable Location? 🌍', - // 'Allow location access to tag your memories. Set to "Always Allow" in Settings', - // singleButton: false, - // okButtonText: 'Continue', - // ), - // ); - // } - final connectivityProvider = Provider.of(context, listen: false); - if (!connectivityProvider.isConnected) { - context.read().cancelMemoryCreationTimer(); - } - }); - - super.initState(); - } - - @override - void dispose() { - WidgetsBinding.instance.removeObserver(this); - // context.read().closeWebSocket(); - super.dispose(); - } - - // Future requestLocationPermission() async { - // LocationService locationService = LocationService(); - // bool serviceEnabled = await locationService.enableService(); - // if (!serviceEnabled) { - // debugPrint('Location service not enabled'); - // if (mounted) { - // ScaffoldMessenger.of(context).showSnackBar( - // const SnackBar( - // content: Text( - // 'Location services are disabled. Enable them for a better experience.', - // style: TextStyle(color: Colors.white, fontSize: 14), - // ), - // ), - // ); - // } - // } else { - // PermissionStatus permissionGranted = await locationService.requestPermission(); - // SharedPreferencesUtil().locationEnabled = permissionGranted == PermissionStatus.granted; - // MixpanelManager().setUserProperty('Location Enabled', SharedPreferencesUtil().locationEnabled); - // if (permissionGranted == PermissionStatus.denied) { - // debugPrint('Location permission not granted'); - // } else if (permissionGranted == PermissionStatus.deniedForever) { - // debugPrint('Location permission denied forever'); - // if (mounted) { - // ScaffoldMessenger.of(context).showSnackBar( - // const SnackBar( - // content: Text( - // 'If you change your mind, you can enable location services in your device settings.', - // style: TextStyle(color: Colors.white, fontSize: 14), - // ), - // ), - // ); - // } - // } - // } - // } - +class CapturePageState extends State { @override Widget build(BuildContext context) { - super.build(context); - return Consumer2(builder: (context, provider, deviceProvider, child) { - return MessageListener( - showInfo: (info) { - // This probably will never be called because this has been handled even before we start the audio stream. But it's here just in case. - if (info == 'FIM_CHANGE') { - showDialog( - context: context, - barrierDismissible: false, - builder: (c) => getDialog( - context, - () async { - context.read().closeWebSocketWithoutReconnect('Firmware change detected'); - var connectedDevice = deviceProvider.connectedDevice; - var codec = await getAudioCodec(connectedDevice!.id); - context.read().resetState(restartBytesProcessing: true); - context.read().initiateWebsocket(codec); - if (Navigator.canPop(context)) { - Navigator.pop(context); - } - }, - () => {}, - 'Firmware change detected!', - 'You are currently using a different firmware version than the one you were using before. Please restart the app to apply the changes.', - singleButton: true, - okButtonText: 'Restart', - ), - ); - } - }, - showError: (error) { - ScaffoldMessenger.of(context).showSnackBar( - SnackBar( - content: Text( - error, - style: const TextStyle(color: Colors.white, fontSize: 14), - ), - ), - ); - }, - child: Stack( - children: [ - ListView(children: [ - SpeechProfileCardWidget(), - ...getConnectionStateWidgets( - context, - provider.hasTranscripts, - deviceProvider.connectedDevice, - context.read().wsConnectionState, - ), - getTranscriptWidget( - provider.memoryCreating, - provider.segments, - provider.photos, - deviceProvider.connectedDevice, - ), - ...connectionStatusWidgets( - context, - provider.segments, - context.read().wsConnectionState, - ), - const SizedBox(height: 16) - ]), - getPhoneMicRecordingButton(() => _recordingToggled(provider), provider.recordingState), - ], - ), - ); - }); - } - - _recordingToggled(CaptureProvider provider) async { - var recordingState = provider.recordingState; - if (recordingState == RecordingState.record) { - provider.stopStreamRecording(); - provider.updateRecordingState(RecordingState.stop); - context.read().cancelMemoryCreationTimer(); - // await context.read().tryCreateMemoryManually(); - } else if (recordingState == RecordingState.initialising) { - debugPrint('initialising, have to wait'); - } else { - showDialog( - context: context, - builder: (c) => getDialog( - context, - () => Navigator.pop(context), - () async { - provider.updateRecordingState(RecordingState.initialising); - context.read().closeWebSocketWithoutReconnect('Recording with phone mic'); - await provider.initiateWebsocket(BleAudioCodec.pcm16, 16000); - await provider.streamRecording(); - Navigator.pop(context); - }, - 'Limited Capabilities', - 'Recording with your phone microphone has a few limitations, including but not limited to: speaker profiles, background reliability.', - okButtonText: 'Ok, I understand', - ), - ); - } + return const Text("Depreacted"); } } diff --git a/app/lib/pages/capture/widgets/widgets.dart b/app/lib/pages/capture/widgets/widgets.dart index 0cdf0da9f..d131f5ae7 100644 --- a/app/lib/pages/capture/widgets/widgets.dart +++ b/app/lib/pages/capture/widgets/widgets.dart @@ -4,6 +4,7 @@ import 'package:friend_private/backend/schema/bt_device.dart'; import 'package:friend_private/backend/schema/transcript_segment.dart'; import 'package:friend_private/pages/capture/connect.dart'; import 'package:friend_private/pages/speech_profile/page.dart'; +import 'package:friend_private/providers/capture_provider.dart'; import 'package:friend_private/providers/connectivity_provider.dart'; import 'package:friend_private/providers/device_provider.dart'; import 'package:friend_private/providers/home_provider.dart'; @@ -227,8 +228,7 @@ class SpeechProfileCardWidget extends StatelessWidget { await routeToPage(context, const SpeechProfilePage()); if (hasSpeakerProfile != SharedPreferencesUtil().hasSpeakerProfile) { if (context.mounted) { - // TODO: is the websocket restarting once the user comes back? - context.read().restartWebSocket(); + context.read().onRecordProfileSettingChanged(); } } }, diff --git a/app/lib/pages/home/page.dart b/app/lib/pages/home/page.dart index 81655b6d1..dde45dcb6 100644 --- a/app/lib/pages/home/page.dart +++ b/app/lib/pages/home/page.dart @@ -13,6 +13,7 @@ import 'package:friend_private/pages/home/device.dart'; import 'package:friend_private/pages/memories/page.dart'; import 'package:friend_private/pages/plugins/page.dart'; import 'package:friend_private/pages/settings/page.dart'; +import 'package:friend_private/providers/capture_provider.dart'; import 'package:friend_private/providers/connectivity_provider.dart'; import 'package:friend_private/providers/device_provider.dart'; import 'package:friend_private/providers/home_provider.dart'; @@ -21,6 +22,7 @@ import 'package:friend_private/providers/memory_provider.dart'; import 'package:friend_private/providers/message_provider.dart'; import 'package:friend_private/providers/plugin_provider.dart'; import 'package:friend_private/services/notification_service.dart'; +import 'package:friend_private/services/services.dart'; import 'package:friend_private/utils/analytics/mixpanel.dart'; import 'package:friend_private/utils/audio/foreground.dart'; import 'package:friend_private/utils/other/temp.dart'; @@ -132,6 +134,10 @@ class _HomePageState extends State with WidgetsBindingObserver, Ticker ForegroundUtil.startForegroundTask(); if (mounted) { await context.read().setUserPeople(); + + // Start stream recording + await Provider.of(context, listen: false) + .streamDeviceRecording(device: context.read().connectedDevice); } }); @@ -544,7 +550,9 @@ class _HomePageState extends State with WidgetsBindingObserver, Ticker if (language != SharedPreferencesUtil().recordingsLanguage || hasSpeech != SharedPreferencesUtil().hasSpeakerProfile || transcriptModel != SharedPreferencesUtil().transcriptionModel) { - context.read().restartWebSocket(); + if (context.mounted) { + context.read().onRecordProfileSettingChanged(); + } } }, ), diff --git a/app/lib/pages/memories/widgets/capture.dart b/app/lib/pages/memories/widgets/capture.dart index d94468dbc..76196a8ef 100644 --- a/app/lib/pages/memories/widgets/capture.dart +++ b/app/lib/pages/memories/widgets/capture.dart @@ -34,13 +34,15 @@ class LiteCaptureWidgetState extends State void _onReceiveTaskData(dynamic data) { if (data is Map) { if (data.containsKey('latitude') && data.containsKey('longitude')) { - context.read().setGeolocation(Geolocation( - latitude: data['latitude'], - longitude: data['longitude'], - accuracy: data['accuracy'], - altitude: data['altitude'], - time: DateTime.parse(data['time']), - )); + if (mounted) { + context.read().setGeolocation(Geolocation( + latitude: data['latitude'], + longitude: data['longitude'], + accuracy: data['accuracy'], + altitude: data['altitude'], + time: DateTime.parse(data['time']), + )); + } } else { if (mounted) { context.read().setGeolocation(null); @@ -73,7 +75,6 @@ class LiteCaptureWidgetState extends State @override void dispose() { WidgetsBinding.instance.removeObserver(this); - // context.read().closeWebSocket(); super.dispose(); } @@ -136,11 +137,9 @@ class LiteCaptureWidgetState extends State builder: (c) => getDialog( context, () async { - context.read().closeWebSocketWithoutReconnect('Firmware change detected'); var connectedDevice = deviceProvider.connectedDevice; var codec = await _getAudioCodec(connectedDevice!.id); - context.read().resetState(restartBytesProcessing: true); - context.read().initiateWebsocket(codec); + await context.read().changeAudioRecordProfile(codec); if (Navigator.canPop(context)) { Navigator.pop(context); } diff --git a/app/lib/pages/memories/widgets/processing_capture.dart b/app/lib/pages/memories/widgets/processing_capture.dart index 2654fad4b..68a9a9a83 100644 --- a/app/lib/pages/memories/widgets/processing_capture.dart +++ b/app/lib/pages/memories/widgets/processing_capture.dart @@ -7,7 +7,6 @@ import 'package:friend_private/pages/memory_capturing/page.dart'; import 'package:friend_private/providers/capture_provider.dart'; import 'package:friend_private/providers/connectivity_provider.dart'; import 'package:friend_private/providers/device_provider.dart'; -import 'package:friend_private/providers/websocket_provider.dart'; import 'package:friend_private/utils/analytics/mixpanel.dart'; import 'package:friend_private/utils/enums.dart'; import 'package:friend_private/utils/other/temp.dart'; @@ -80,7 +79,7 @@ class _MemoryCaptureWidgetState extends State { _toggleRecording(BuildContext context, CaptureProvider provider) async { var recordingState = provider.recordingState; if (recordingState == RecordingState.record) { - provider.stopStreamRecording(); + await provider.stopStreamRecording(); context.read().cancelMemoryCreationTimer(); await context.read().createMemory(); MixpanelManager().phoneMicRecordingStopped(); @@ -95,8 +94,7 @@ class _MemoryCaptureWidgetState extends State { () async { Navigator.pop(context); provider.updateRecordingState(RecordingState.initialising); - context.read().closeWebSocketWithoutReconnect('Recording with phone mic'); - await provider.initiateWebsocket(BleAudioCodec.pcm16, 16000); + await provider.changeAudioRecordProfile(BleAudioCodec.pcm16, 16000); await provider.streamRecording(); MixpanelManager().phoneMicRecordingStarted(); }, diff --git a/app/lib/pages/onboarding/speech_profile_widget.dart b/app/lib/pages/onboarding/speech_profile_widget.dart index acbb51f98..e05a4f8b8 100644 --- a/app/lib/pages/onboarding/speech_profile_widget.dart +++ b/app/lib/pages/onboarding/speech_profile_widget.dart @@ -3,7 +3,10 @@ import 'dart:async'; import 'package:flutter/material.dart'; import 'package:flutter_provider_utilities/flutter_provider_utilities.dart'; import 'package:friend_private/backend/preferences.dart'; +import 'package:friend_private/backend/schema/bt_device.dart'; +import 'package:friend_private/providers/capture_provider.dart'; import 'package:friend_private/providers/speech_profile_provider.dart'; +import 'package:friend_private/services/services.dart'; import 'package:friend_private/widgets/dialog.dart'; import 'package:gradient_borders/box_borders/gradient_box_border.dart'; import 'package:provider/provider.dart'; @@ -49,13 +52,35 @@ class _SpeechProfileWidgetState extends State with TickerPr @override Widget build(BuildContext context) { + Future restartDeviceRecording() async { + debugPrint("restartDeviceRecording $mounted"); + + // Restart device recording, clear transcripts + if (mounted) { + Provider.of(context, listen: false).clearTranscripts(); + Provider.of(context, listen: false).streamDeviceRecording( + device: Provider.of(context, listen: false).deviceProvider?.connectedDevice, + ); + } + } + + Future stopDeviceRecording() async { + debugPrint("stopDeviceRecording $mounted"); + + // Restart device recording, clear transcripts + if (mounted) { + await Provider.of(context, listen: false).stopStreamDeviceRecording(); + } + } + return PopScope( canPop: true, - onPopInvoked: (didPop) { + onPopInvoked: (didPop) async { context.read().close(); + restartDeviceRecording(); }, - child: Consumer( - builder: (context, provider, child) { + child: Consumer2( + builder: (context, provider, _, child) { return MessageListener( showInfo: (info) { if (info == 'SCROLL_DOWN') { @@ -204,10 +229,11 @@ class _SpeechProfileWidgetState extends State with TickerPr ), child: TextButton( onPressed: () async { - await provider.initialise(true); + await stopDeviceRecording(); + await provider.initialise(true, finalizedCallback: restartDeviceRecording); provider.forceCompletionTimer = Timer(Duration(seconds: provider.maxDuration), () async { - provider.finalize(true); + provider.finalize(); }); provider.updateStartedRecording(true); }, diff --git a/app/lib/pages/speech_profile/page.dart b/app/lib/pages/speech_profile/page.dart index 99a66ef9a..198c27234 100644 --- a/app/lib/pages/speech_profile/page.dart +++ b/app/lib/pages/speech_profile/page.dart @@ -6,6 +6,7 @@ import 'package:friend_private/backend/preferences.dart'; import 'package:friend_private/backend/schema/bt_device.dart'; import 'package:friend_private/pages/home/page.dart'; import 'package:friend_private/pages/speech_profile/user_speech_samples.dart'; +import 'package:friend_private/providers/capture_provider.dart'; import 'package:friend_private/providers/speech_profile_provider.dart'; import 'package:friend_private/services/services.dart'; import 'package:friend_private/utils/other/temp.dart'; @@ -30,6 +31,7 @@ class _SpeechProfilePageState extends State with TickerProvid WidgetsBinding.instance.addPostFrameCallback((timeStamp) async { await context.read().updateDevice(); }); + super.initState(); } @@ -65,16 +67,36 @@ class _SpeechProfilePageState extends State with TickerProvid @override Widget build(BuildContext context) { + Future restartDeviceRecording() async { + debugPrint("restartDeviceRecording $mounted"); + if (mounted) { + Provider.of(context, listen: false).clearTranscripts(); + Provider.of(context, listen: false).streamDeviceRecording( + device: Provider.of(context, listen: false).deviceProvider?.connectedDevice, + ); + } + } + + Future stopDeviceRecording() async { + debugPrint("stopDeviceRecording $mounted"); + if (mounted) { + await Provider.of(context, listen: false).stopStreamDeviceRecording(); + } + } + return PopScope( canPop: true, onPopInvoked: (didPop) { if (context.read().isInitialised) { WidgetsBinding.instance.addPostFrameCallback((timeStamp) async { await context.read().close(); + + // Restart device recording + restartDeviceRecording(); }); } }, - child: Consumer(builder: (context, provider, child) { + child: Consumer2(builder: (context, provider, _, child) { return MessageListener( showInfo: (info) { if (info == 'SCROLL_DOWN') { @@ -320,12 +342,13 @@ class _SpeechProfilePageState extends State with TickerProvid ); return; } - await provider.initialise(false); - // provider.initiateWebsocket(false); + + await stopDeviceRecording(); + await provider.initialise(false, finalizedCallback: restartDeviceRecording); // 1.5 minutes seems reasonable provider.forceCompletionTimer = Timer(Duration(seconds: provider.maxDuration), () { - provider.finalize(false); + provider.finalize(); }); provider.updateStartedRecording(true); }, diff --git a/app/lib/providers/capture_provider.dart b/app/lib/providers/capture_provider.dart index 8d286d29f..1724f08ff 100644 --- a/app/lib/providers/capture_provider.dart +++ b/app/lib/providers/capture_provider.dart @@ -20,7 +20,6 @@ import 'package:friend_private/backend/schema/transcript_segment.dart'; import 'package:friend_private/pages/capture/logic/openglass_mixin.dart'; import 'package:friend_private/providers/memory_provider.dart'; import 'package:friend_private/providers/message_provider.dart'; -import 'package:friend_private/providers/websocket_provider.dart'; import 'package:friend_private/services/services.dart'; import 'package:friend_private/utils/analytics/growthbook.dart'; import 'package:friend_private/utils/analytics/mixpanel.dart'; @@ -31,23 +30,24 @@ import 'package:friend_private/utils/features/calendar.dart'; import 'package:friend_private/utils/logger.dart'; import 'package:friend_private/utils/memories/integrations.dart'; import 'package:friend_private/utils/memories/process.dart'; -import 'package:friend_private/utils/websockets.dart'; +import 'package:friend_private/utils/pure_socket.dart'; import 'package:permission_handler/permission_handler.dart'; import 'package:uuid/uuid.dart'; -class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifierMixin { +class CaptureProvider extends ChangeNotifier + with OpenGlassMixin, MessageNotifierMixin + implements ITransctipSegmentSocketServiceListener { MemoryProvider? memoryProvider; MessageProvider? messageProvider; - WebSocketProvider? webSocketProvider; + TranscripSegmentSocketService? _socket; - void updateProviderInstances(MemoryProvider? mp, MessageProvider? p, WebSocketProvider? wsProvider) { + void updateProviderInstances(MemoryProvider? mp, MessageProvider? p) { memoryProvider = mp; messageProvider = p; - webSocketProvider = wsProvider; notifyListeners(); } - BTDeviceStruct? connectedDevice; + BTDeviceStruct? _recordingDevice; bool isGlasses = false; List segments = []; @@ -85,21 +85,15 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie String? processingMemoryId; - bool resetStateAlreadyCalled = false; String dateTimeStorageString = ""; - void setResetStateAlreadyCalled(bool value) { - resetStateAlreadyCalled = value; - notifyListeners(); - } - void setHasTranscripts(bool value) { hasTranscripts = value; notifyListeners(); } void setMemoryCreating(bool value) { - print('set memory creating ${value}'); + debugPrint('set memory creating ${value}'); memoryCreating = value; notifyListeners(); } @@ -120,9 +114,9 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie notifyListeners(); } - void updateConnectedDevice(BTDeviceStruct? device) { - debugPrint('connected device changed from ${connectedDevice?.id} to ${device?.id}'); - connectedDevice = device; + void _updateRecordingDevice(BTDeviceStruct? device) { + debugPrint('connected device changed from ${_recordingDevice?.id} to ${device?.id}'); + _recordingDevice = device; notifyListeners(); } @@ -143,7 +137,7 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie emotionalFeedback: GrowthbookUtil().isOmiFeedbackEnabled(), ); if (result?.result == null) { - print("Can not update processing memory, result null"); + debugPrint("Can not update processing memory, result null"); } } @@ -154,7 +148,7 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie Future _onMemoryCreated(ServerMessageEvent event) async { if (event.memory == null) { - print("Memory is not found, processing memory ${event.processingMemoryId}"); + debugPrint("Memory is not found, processing memory ${event.processingMemoryId}"); return; } _processOnMemoryCreated(event.memory, event.messages ?? []); @@ -167,7 +161,7 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie Future _onMemoryPostProcessSuccess(String memoryId) async { var memory = await getMemoryById(memoryId); if (memory == null) { - print("Memory is not found $memoryId"); + debugPrint("Memory is not found $memoryId"); return; } @@ -177,7 +171,7 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie Future _onMemoryPostProcessFailed(String memoryId) async { var memory = await getMemoryById(memoryId); if (memory == null) { - print("Memory is not found $memoryId"); + debugPrint("Memory is not found $memoryId"); return; } @@ -294,7 +288,7 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie return true; } - void _cleanNew() async { + Future _clean() async { segments = []; audioStorage?.clearAudioBytes(); @@ -307,11 +301,15 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie photos = []; conversationId = const Uuid().v4(); processingMemoryId = null; + } + + Future _cleanNew() async { + _clean(); // Create new socket session // Warn: should have a better solution to keep the socket alived - await webSocketProvider?.closeWebSocketWithoutReconnect('reset new memory session'); - await initiateWebsocket(); + debugPrint("_cleanNew"); + await _initiateWebsocket(force: true); } _handleCalendarCreation(ServerMemory memory) { @@ -334,128 +332,47 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie } } - Future initiateWebsocket([ + Future onRecordProfileSettingChanged() async { + await _resetState(restartBytesProcessing: true); + } + + Future changeAudioRecordProfile([ BleAudioCodec? audioCodec, int? sampleRate, ]) async { - // setWebSocketConnecting(true); - print('initiateWebsocket in capture_provider'); - BleAudioCodec codec = audioCodec ?? SharedPreferencesUtil().deviceCodec; - sampleRate ??= (codec == BleAudioCodec.opus ? 16000 : 8000); - print('is ws null: ${webSocketProvider == null}'); - await webSocketProvider?.initWebSocket( - codec: codec, - sampleRate: sampleRate, - includeSpeechProfile: true, - newMemoryWatch: true, - // Warn: need clarify about initiateWebsocket - onConnectionSuccess: () { - print('inside onConnectionSuccess'); - if (segments.isNotEmpty) { - // means that it was a reconnection, so we need to reset - streamStartedAtSecond = null; - secondsMissedOnReconnect = (DateTime.now().difference(firstStreamReceivedAt!).inSeconds); - } - print('bottom in onConnectionSuccess'); - notifyListeners(); - }, - onConnectionFailed: (err) { - print('inside onConnectionFailed'); - print('err: $err'); - notifyListeners(); - }, - onConnectionClosed: (int? closeCode, String? closeReason) { - print('inside onConnectionClosed'); - print('closeCode: $closeCode'); - // connection was closed, either on resetState, or by backend, or by some other reason. - // setState(() {}); - }, - onConnectionError: (err) { - print('inside onConnectionError'); - print('err: $err'); - // connection was okay, but then failed. - notifyListeners(); - }, - onMessageEventReceived: (ServerMessageEvent event) { - if (event.type == MessageEventType.newMemoryCreating) { - _onMemoryCreating(); - return; - } + debugPrint("changeAudioRecordProfile"); + await _resetState(restartBytesProcessing: true); + await _initiateWebsocket(audioCodec: audioCodec, sampleRate: sampleRate); + } - if (event.type == MessageEventType.newMemoryCreated) { - _onMemoryCreated(event); - return; - } + Future _initiateWebsocket({ + BleAudioCodec? audioCodec, + int? sampleRate, + bool force = false, + }) async { + debugPrint('initiateWebsocket in capture_provider'); - if (event.type == MessageEventType.newMemoryCreateFailed) { - _onMemoryCreateFailed(); - return; - } + BleAudioCodec codec = audioCodec ?? SharedPreferencesUtil().deviceCodec; + sampleRate ??= (codec == BleAudioCodec.opus ? 16000 : 8000); - if (event.type == MessageEventType.newProcessingMemoryCreated) { - if (event.processingMemoryId == null) { - print("New processing memory created message event is invalid"); - return; - } - _onProcessingMemoryCreated(event.processingMemoryId!); - return; - } + debugPrint('is ws null: ${_socket == null}'); - if (event.type == MessageEventType.memoryPostProcessingSuccess) { - if (event.memoryId == null) { - print("Post proccess message event is invalid"); - return; - } - _onMemoryPostProcessSuccess(event.memoryId!); - return; - } - - if (event.type == MessageEventType.memoryPostProcessingFailed) { - if (event.memoryId == null) { - print("Post proccess message event is invalid"); - return; - } - _onMemoryPostProcessFailed(event.memoryId!); - return; - } - }, - onMessageReceived: (List newSegments) { - if (newSegments.isEmpty) return; - - if (segments.isEmpty) { - debugPrint('newSegments: ${newSegments.last}'); - // TODO: small bug -> when memory A creates, and memory B starts, memory B will clean a lot more seconds than available, - // losing from the audio the first part of the recording. All other parts are fine. - FlutterForegroundTask.sendDataToTask(jsonEncode({'location': true})); - var currentSeconds = (audioStorage?.frames.length ?? 0) ~/ 100; - var removeUpToSecond = newSegments[0].start.toInt(); - audioStorage?.removeFramesRange(fromSecond: 0, toSecond: min(max(currentSeconds - 5, 0), removeUpToSecond)); - firstStreamReceivedAt = DateTime.now(); - } + // Get memory socket + _socket = await ServiceManager.instance().socket.memory(codec: codec, sampleRate: sampleRate, force: force); + if (_socket == null) { + throw Exception("Can not create new memory socket"); + } + _socket?.subscribe(this, this); - streamStartedAtSecond ??= newSegments[0].start; - TranscriptSegment.combineSegments( - segments, - newSegments, - toRemoveSeconds: streamStartedAtSecond ?? 0, - toAddSeconds: secondsMissedOnReconnect ?? 0, - ); - triggerTranscriptSegmentReceivedEvents(newSegments, conversationId, sendMessageToChat: (v) { - messageProvider?.addMessage(v); - }); - - debugPrint('Memory creation timer restarted'); - _memoryCreationTimer?.cancel(); - _memoryCreationTimer = - Timer(const Duration(seconds: quietSecondsForMemoryCreation), () => _createPhotoCharacteristicMemory()); - setHasTranscripts(true); - notifyListeners(); - }, - ); + if (segments.isNotEmpty) { + // means that it was a reconnection, so we need to reset + streamStartedAtSecond = null; + secondsMissedOnReconnect = (DateTime.now().difference(firstStreamReceivedAt!).inSeconds); + } } Future streamAudioToWs(String id, BleAudioCodec codec) async { - print('streamAudioToWs in capture_provider'); + debugPrint('streamAudioToWs in capture_provider'); audioStorage = WavBytesUtil(codec: codec); if (_bleBytesStream != null) { _bleBytesStream?.cancel(); @@ -468,8 +385,8 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie final trimmedValue = value.sublist(3); // TODO: if this (0,3) is not removed, deepgram can't seem to be able to detect the audio. // https://developers.deepgram.com/docs/determining-your-audio-format-for-live-streaming-audio - if (webSocketProvider?.wsConnectionState == WebsocketConnectionStatus.connected) { - webSocketProvider?.websocketChannel?.sink.add(trimmedValue); + if (_socket?.state == SocketServiceState.connected) { + _socket?.send(trimmedValue); } }, ); @@ -527,13 +444,13 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie Future getFileFromDevice(int fileNum) async { storageUtil.fileNum = fileNum; int command = 0; - writeToStorage(connectedDevice!.id, storageUtil.fileNum, command); + writeToStorage(_recordingDevice!.id, storageUtil.fileNum, command); } Future clearFileFromDevice(int fileNum) async { storageUtil.fileNum = fileNum; int command = 1; - writeToStorage(connectedDevice!.id, storageUtil.fileNum, command); + writeToStorage(_recordingDevice!.id, storageUtil.fileNum, command); } void clearTranscripts() { @@ -544,40 +461,25 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie Future resetForSpeechProfile() async { closeBleStream(); - await webSocketProvider?.closeWebSocketWithoutReconnect('reset for speech profile'); + await _socket?.stop(reason: 'reset for speech profile'); setAudioBytesConnected(false); notifyListeners(); } - Future resetState({ + Future _resetState({ bool restartBytesProcessing = true, - bool isFromSpeechProfile = false, - BTDeviceStruct? btDevice, }) async { - if (resetStateAlreadyCalled) { - debugPrint('resetState already called'); - return; - } - setResetStateAlreadyCalled(true); - debugPrint('resetState: restartBytesProcessing=$restartBytesProcessing, isFromSpeechProfile=$isFromSpeechProfile'); + debugPrint('resetState: restartBytesProcessing=$restartBytesProcessing'); _cleanupCurrentState(); - await startOpenGlass(); - if (!isFromSpeechProfile) { - await _handleMemoryCreation(restartBytesProcessing); - } - - bool codecChanged = await _checkCodecChange(); + await _handleMemoryCreation(restartBytesProcessing); - if (restartBytesProcessing || codecChanged) { - await _manageWebSocketConnection(codecChanged, isFromSpeechProfile); - } + await _ensureSocketConnection(force: true); - await initiateFriendAudioStreaming(isFromSpeechProfile); + await startOpenGlass(); + await _initiateFriendAudioStreaming(); // TODO: Commenting this for now as DevKit 2 is not yet used in production // await initiateStorageBytesStreaming(); - - setResetStateAlreadyCalled(false); notifyListeners(); } @@ -655,8 +557,8 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie } Future _checkCodecChange() async { - if (connectedDevice != null) { - BleAudioCodec newCodec = await _getAudioCodec(connectedDevice!.id); + if (_recordingDevice != null) { + BleAudioCodec newCodec = await _getAudioCodec(_recordingDevice!.id); if (SharedPreferencesUtil().deviceCodec != newCodec) { debugPrint('Device codec changed from ${SharedPreferencesUtil().deviceCodec} to $newCodec'); SharedPreferencesUtil().deviceCodec = newCodec; @@ -666,31 +568,31 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie return false; } - Future _manageWebSocketConnection(bool codecChanged, bool isFromSpeechProfile) async { - if (codecChanged || webSocketProvider?.wsConnectionState != WebsocketConnectionStatus.connected) { - await webSocketProvider?.closeWebSocketWithoutReconnect('reset state $isFromSpeechProfile'); - // if (!isFromSpeechProfile) { - await initiateWebsocket(); - // } + Future _ensureSocketConnection({bool force = false}) async { + debugPrint("_ensureSocketConnection"); + var codec = SharedPreferencesUtil().deviceCodec; + if (force || (codec != _socket?.codec || _socket?.state != SocketServiceState.connected)) { + await _socket?.stop(reason: 'reset state, force $force'); + await _initiateWebsocket(force: force); } } - Future initiateFriendAudioStreaming(bool isFromSpeechProfile) async { - print('connectedDevice: $connectedDevice in initiateFriendAudioStreaming'); - if (connectedDevice == null) return; + Future _initiateFriendAudioStreaming() async { + debugPrint('_recordingDevice: $_recordingDevice in initiateFriendAudioStreaming'); + if (_recordingDevice == null) return; - BleAudioCodec codec = await _getAudioCodec(connectedDevice!.id); + BleAudioCodec codec = await _getAudioCodec(_recordingDevice!.id); if (SharedPreferencesUtil().deviceCodec != codec) { debugPrint('Device codec changed from ${SharedPreferencesUtil().deviceCodec} to $codec'); SharedPreferencesUtil().deviceCodec = codec; notifyInfo('FIM_CHANGE'); - await _manageWebSocketConnection(true, isFromSpeechProfile); + await _ensureSocketConnection(); } - // Why is the connectedDevice null at this point? + // Why is the _recordingDevice null at this point? if (!audioBytesConnected) { - if (connectedDevice != null) { - await streamAudioToWs(connectedDevice!.id, codec); + if (_recordingDevice != null) { + await streamAudioToWs(_recordingDevice!.id, codec); } else { // Is the app in foreground when this happens? Logger.handle(Exception('Device Not Connected'), StackTrace.current, @@ -703,19 +605,19 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie Future initiateStorageBytesStreaming() async { debugPrint('initiateStorageBytesStreaming'); - if (connectedDevice == null) return; - currentStorageFiles = await _getStorageList(connectedDevice!.id); + if (_recordingDevice == null) return; + currentStorageFiles = await _getStorageList(_recordingDevice!.id); debugPrint('Storage files: $currentStorageFiles'); - await sendStorage(connectedDevice!.id); + await sendStorage(_recordingDevice!.id); notifyListeners(); } Future startOpenGlass() async { - if (connectedDevice == null) return; - isGlasses = await _hasPhotoStreamingCharacteristic(connectedDevice!.id); + if (_recordingDevice == null) return; + isGlasses = await _hasPhotoStreamingCharacteristic(_recordingDevice!.id); if (!isGlasses) return; - await openGlassProcessing(connectedDevice!, (p) {}, setHasTranscripts); - webSocketProvider?.closeWebSocketWithoutReconnect('reset state open glass'); + await openGlassProcessing(_recordingDevice!, (p) {}, setHasTranscripts); + _socket?.stop(reason: 'reset state open glass'); notifyListeners(); } @@ -733,6 +635,7 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie void dispose() { _bleBytesStream?.cancel(); _memoryCreationTimer?.cancel(); + _socket?.unsubscribe(this); super.dispose(); } @@ -746,8 +649,8 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie // record await ServiceManager.instance().mic.start(onByteReceived: (bytes) { - if (webSocketProvider?.wsConnectionState == WebsocketConnectionStatus.connected) { - webSocketProvider?.websocketChannel?.sink.add(bytes); + if (_socket?.state == SocketServiceState.connected) { + _socket?.send(bytes); } }, onRecording: () { updateRecordingState(RecordingState.record); @@ -761,4 +664,126 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie stopStreamRecording() { ServiceManager.instance().mic.stop(); } + + Future streamDeviceRecording({ + BTDeviceStruct? device, + bool restartBytesProcessing = true, + }) async { + debugPrint("streamDeviceRecording ${device} ${restartBytesProcessing}"); + if (device != null) { + _updateRecordingDevice(device); + } + + await _resetState( + restartBytesProcessing: restartBytesProcessing, + ); + } + + Future stopStreamDeviceRecording({bool cleanDevice = false}) async { + if (cleanDevice) { + _updateRecordingDevice(null); + } + _cleanupCurrentState(); + await _socket?.stop(reason: 'stop stream device recording'); + await _handleMemoryCreation(false); + } + + // Socket handling + + @override + void onClosed() { + debugPrint('[Provider] Socket is closed'); + + _clean(); + + // Notify + setMemoryCreating(false); + setHasTranscripts(false); + notifyListeners(); + } + + @override + void onError(Object err) { + debugPrint('err: $err'); + notifyListeners(); + } + + @override + void onMessageEventReceived(ServerMessageEvent event) { + if (event.type == MessageEventType.newMemoryCreating) { + _onMemoryCreating(); + return; + } + + if (event.type == MessageEventType.newMemoryCreated) { + _onMemoryCreated(event); + return; + } + + if (event.type == MessageEventType.newMemoryCreateFailed) { + _onMemoryCreateFailed(); + return; + } + + if (event.type == MessageEventType.newProcessingMemoryCreated) { + if (event.processingMemoryId == null) { + debugPrint("New processing memory created message event is invalid"); + return; + } + _onProcessingMemoryCreated(event.processingMemoryId!); + return; + } + + if (event.type == MessageEventType.memoryPostProcessingSuccess) { + if (event.memoryId == null) { + debugPrint("Post proccess message event is invalid"); + return; + } + _onMemoryPostProcessSuccess(event.memoryId!); + return; + } + + if (event.type == MessageEventType.memoryPostProcessingFailed) { + if (event.memoryId == null) { + debugPrint("Post proccess message event is invalid"); + return; + } + _onMemoryPostProcessFailed(event.memoryId!); + return; + } + } + + @override + void onSegmentReceived(List newSegments) { + if (newSegments.isEmpty) return; + + if (segments.isEmpty) { + debugPrint('newSegments: ${newSegments.last}'); + // TODO: small bug -> when memory A creates, and memory B starts, memory B will clean a lot more seconds than available, + // losing from the audio the first part of the recording. All other parts are fine. + FlutterForegroundTask.sendDataToTask(jsonEncode({'location': true})); + var currentSeconds = (audioStorage?.frames.length ?? 0) ~/ 100; + var removeUpToSecond = newSegments[0].start.toInt(); + audioStorage?.removeFramesRange(fromSecond: 0, toSecond: min(max(currentSeconds - 5, 0), removeUpToSecond)); + firstStreamReceivedAt = DateTime.now(); + } + + streamStartedAtSecond ??= newSegments[0].start; + TranscriptSegment.combineSegments( + segments, + newSegments, + toRemoveSeconds: streamStartedAtSecond ?? 0, + toAddSeconds: secondsMissedOnReconnect ?? 0, + ); + triggerTranscriptSegmentReceivedEvents(newSegments, conversationId, sendMessageToChat: (v) { + messageProvider?.addMessage(v); + }); + + debugPrint('Memory creation timer restarted'); + _memoryCreationTimer?.cancel(); + _memoryCreationTimer = + Timer(const Duration(seconds: quietSecondsForMemoryCreation), () => _createPhotoCharacteristicMemory()); + setHasTranscripts(true); + notifyListeners(); + } } diff --git a/app/lib/providers/device_provider.dart b/app/lib/providers/device_provider.dart index 0494f8d59..936594f8b 100644 --- a/app/lib/providers/device_provider.dart +++ b/app/lib/providers/device_provider.dart @@ -3,7 +3,6 @@ import 'dart:async'; import 'package:flutter/material.dart'; import 'package:friend_private/backend/preferences.dart'; import 'package:friend_private/backend/schema/bt_device.dart'; -import 'package:friend_private/providers/websocket_provider.dart'; import 'package:friend_private/providers/capture_provider.dart'; import 'package:friend_private/services/devices.dart'; import 'package:friend_private/services/notification_service.dart'; @@ -13,7 +12,6 @@ import 'package:instabug_flutter/instabug_flutter.dart'; class DeviceProvider extends ChangeNotifier implements IDeviceServiceSubsciption { CaptureProvider? captureProvider; - WebSocketProvider? webSocketProvider; bool isConnecting = false; bool isConnected = false; @@ -25,16 +23,14 @@ class DeviceProvider extends ChangeNotifier implements IDeviceServiceSubsciption Timer? _disconnectNotificationTimer; - void setProviders(CaptureProvider provider, WebSocketProvider wsProvider) { + void setProviders(CaptureProvider provider) { captureProvider = provider; - webSocketProvider = wsProvider; notifyListeners(); } void setConnectedDevice(BTDeviceStruct? device) { connectedDevice = device; print('setConnectedDevice: $device'); - captureProvider?.updateConnectedDevice(device); notifyListeners(); } @@ -149,25 +145,10 @@ class DeviceProvider extends ChangeNotifier implements IDeviceServiceSubsciption if (isConnected) { await initiateBleBatteryListener(); } - await captureProvider?.resetState(restartBytesProcessing: true, btDevice: connectedDevice); - // if (captureProvider?.webSocketConnected == false) { - // restartWebSocket(); - // } notifyListeners(); } - Future restartWebSocket() async { - debugPrint('restartWebSocket'); - - await webSocketProvider?.closeWebSocketWithoutReconnect('Restarting WebSocket'); - if (connectedDevice == null) { - return; - } - await captureProvider?.resetState(restartBytesProcessing: true); - notifyListeners(); - } - void updateConnectingStatus(bool value) { isConnecting = value; notifyListeners(); @@ -197,7 +178,7 @@ class DeviceProvider extends ChangeNotifier implements IDeviceServiceSubsciption setConnectedDevice(null); setIsConnected(false); updateConnectingStatus(false); - await captureProvider?.resetState(restartBytesProcessing: false); + await captureProvider?.stopStreamDeviceRecording(cleanDevice: true); captureProvider?.setAudioBytesConnected(false); print('after resetState inside initiateConnectionListener'); @@ -224,7 +205,7 @@ class DeviceProvider extends ChangeNotifier implements IDeviceServiceSubsciption setConnectedDevice(device); setIsConnected(true); updateConnectingStatus(false); - await captureProvider?.resetState(restartBytesProcessing: true, btDevice: connectedDevice); + await captureProvider?.streamDeviceRecording(restartBytesProcessing: true, device: connectedDevice); // initiateBleBatteryListener(); // The device is still disconnected for some reason if (connectedDevice != null) { diff --git a/app/lib/providers/onboarding_provider.dart b/app/lib/providers/onboarding_provider.dart index 70d7a183d..d7b90723e 100644 --- a/app/lib/providers/onboarding_provider.dart +++ b/app/lib/providers/onboarding_provider.dart @@ -179,6 +179,7 @@ class OnboardingProvider extends BaseProvider with MessageNotifierMixin implemen deviceProvider!.setConnectedDevice(cDevice); SharedPreferencesUtil().btDeviceStruct = cDevice; SharedPreferencesUtil().deviceName = cDevice.name; + SharedPreferencesUtil().deviceCodec = await _getAudioCodec(device.id); deviceProvider!.setIsConnected(true); } //TODO: should'nt update codec here, becaause then the prev connection codec and the current codec will @@ -196,6 +197,7 @@ class OnboardingProvider extends BaseProvider with MessageNotifierMixin implemen await Future.delayed(const Duration(seconds: 2)); SharedPreferencesUtil().btDeviceStruct = connectedDevice!; SharedPreferencesUtil().deviceName = connectedDevice.name; + SharedPreferencesUtil().deviceCodec = await _getAudioCodec(device.id); foundDevicesMap.clear(); deviceList.clear(); if (isFromOnboarding) { @@ -250,6 +252,14 @@ class OnboardingProvider extends BaseProvider with MessageNotifierMixin implemen return connection?.device; } + Future _getAudioCodec(String deviceId) async { + var connection = await ServiceManager.instance().device.ensureConnection(deviceId); + if (connection == null) { + return BleAudioCodec.pcm8; + } + return connection.getAudioCodec(); + } + @override void dispose() { //TODO: This does not get called when the page is popped diff --git a/app/lib/providers/speech_profile_provider.dart b/app/lib/providers/speech_profile_provider.dart index 7e2acaf5c..8efec243b 100644 --- a/app/lib/providers/speech_profile_provider.dart +++ b/app/lib/providers/speech_profile_provider.dart @@ -10,22 +10,22 @@ import 'package:friend_private/backend/http/cloud_storage.dart'; import 'package:friend_private/backend/preferences.dart'; import 'package:friend_private/backend/schema/bt_device.dart'; import 'package:friend_private/backend/schema/memory.dart'; +import 'package:friend_private/backend/schema/message_event.dart'; import 'package:friend_private/backend/schema/structured.dart'; import 'package:friend_private/backend/schema/transcript_segment.dart'; import 'package:friend_private/providers/capture_provider.dart'; import 'package:friend_private/providers/device_provider.dart'; -import 'package:friend_private/providers/websocket_provider.dart'; import 'package:friend_private/services/devices.dart'; import 'package:friend_private/services/services.dart'; import 'package:friend_private/utils/audio/wav_bytes.dart'; import 'package:friend_private/utils/memories/process.dart'; -import 'package:friend_private/utils/websockets.dart'; +import 'package:friend_private/utils/pure_socket.dart'; import 'package:uuid/uuid.dart'; -class SpeechProfileProvider extends ChangeNotifier with MessageNotifierMixin implements IDeviceServiceSubsciption { +class SpeechProfileProvider extends ChangeNotifier + with MessageNotifierMixin + implements IDeviceServiceSubsciption, ITransctipSegmentSocketServiceListener { DeviceProvider? deviceProvider; - CaptureProvider? captureProvider; - WebSocketProvider? webSocketProvider; bool? permissionEnabled; bool loading = false; BTDeviceStruct? device; @@ -38,6 +38,8 @@ class SpeechProfileProvider extends ChangeNotifier with MessageNotifierMixin imp WavBytesUtil audioStorage = WavBytesUtil(codec: BleAudioCodec.opus); StreamSubscription? _bleBytesStream; + TranscripSegmentSocketService? _socket; + bool startedRecording = false; double percentageCompleted = 0; bool uploadingProfile = false; @@ -50,6 +52,9 @@ class SpeechProfileProvider extends ChangeNotifier with MessageNotifierMixin imp String text = ''; String message = ''; + late bool _isFromOnboarding; + late Function? _finalizedCallback; + /// only used during onboarding ///// String loadingText = 'Uploading your voice profile....'; ServerMemory? memory; @@ -71,10 +76,8 @@ class SpeechProfileProvider extends ChangeNotifier with MessageNotifierMixin imp notifyListeners(); } - void setProviders(DeviceProvider provider, CaptureProvider captureProvider, WebSocketProvider wsProvider) { + void setProviders(DeviceProvider provider) { deviceProvider = provider; - this.captureProvider = captureProvider; - webSocketProvider = wsProvider; notifyListeners(); } @@ -86,15 +89,15 @@ class SpeechProfileProvider extends ChangeNotifier with MessageNotifierMixin imp notifyListeners(); } - Future initialise(bool isFromOnboarding) async { + Future initialise(bool isFromOnboarding, {Function? finalizedCallback}) async { + _isFromOnboarding = isFromOnboarding; + _finalizedCallback = finalizedCallback; setInitialising(true); device = deviceProvider?.connectedDevice; - await captureProvider!.resetForSpeechProfile(); - await initiateWebsocket(isFromOnboarding); + await _initiateWebsocket(force: true); - // _bleBytesStream = captureProvider?.bleBytesStream; if (device != null) await initiateFriendAudioStreaming(); - if (webSocketProvider?.wsConnectionState != WebsocketConnectionStatus.connected) { + if (_socket?.state != SocketServiceState.connected) { // wait for websocket to connect await Future.delayed(Duration(seconds: 2)); } @@ -120,96 +123,72 @@ class SpeechProfileProvider extends ChangeNotifier with MessageNotifierMixin imp ServiceManager.instance().device.subscribe(this, this); } - Future initiateWebsocket(bool isFromOnboarding) async { - await webSocketProvider?.initWebSocket( - codec: BleAudioCodec.opus, - sampleRate: 16000, - includeSpeechProfile: false, - newMemoryWatch: false, - onConnectionSuccess: () { - print('Websocket connected in speech profile'); - notifyListeners(); - }, - onConnectionFailed: (err) { - notifyError('WS_ERR'); - }, - onConnectionClosed: (int? closeCode, String? closeReason) {}, - onConnectionError: (err) { - notifyError('WS_ERR'); - }, - onMessageReceived: (List newSegments) { - if (newSegments.isEmpty) return; - if (segments.isEmpty) { - audioStorage.removeFramesRange(fromSecond: 0, toSecond: newSegments[0].start.toInt()); - } - streamStartedAtSecond ??= newSegments[0].start; - - TranscriptSegment.combineSegments( - segments, - newSegments, - toRemoveSeconds: streamStartedAtSecond ?? 0, - ); - updateProgressMessage(); - _validateSingleSpeaker(); - _handleCompletion(isFromOnboarding); - notifyInfo('SCROLL_DOWN'); - debugPrint('Memory creation timer restarted'); - }, - ); + Future _initiateWebsocket({bool force = false}) async { + _socket = await ServiceManager.instance() + .socket + .speechProfile(codec: BleAudioCodec.opus, sampleRate: 16000, force: force); + if (_socket == null) { + throw Exception("Can not create new speech profile socket"); + } + _socket?.subscribe(this, this); } - _handleCompletion(bool isFromOnboarding) async { + _handleCompletion() async { if (uploadingProfile || profileCompleted) return; String text = segments.map((e) => e.text).join(' ').trim(); int wordsCount = text.split(' ').length; percentageCompleted = (wordsCount / targetWordsCount).clamp(0, 1); notifyListeners(); if (percentageCompleted == 1) { - await finalize(isFromOnboarding); + await finalize(); } notifyListeners(); } - Future finalize(bool isFromOnboarding) async { - if (uploadingProfile || profileCompleted) return; - - int duration = segments.isEmpty ? 0 : segments.last.end.toInt(); - if (duration < 5 || duration > 120) { - notifyError('INVALID_RECORDING'); - } + Future finalize() async { + try { + if (uploadingProfile || profileCompleted) return; - String text = segments.map((e) => e.text).join(' ').trim(); - if (text.split(' ').length < (targetWordsCount / 2)) { - // 25 words - notifyError('TOO_SHORT'); - } - uploadingProfile = true; - notifyListeners(); - await webSocketProvider?.closeWebSocketWithoutReconnect('finalizing'); - forceCompletionTimer?.cancel(); - connectionStateListener?.cancel(); - _bleBytesStream?.cancel(); + int duration = segments.isEmpty ? 0 : segments.last.end.toInt(); + if (duration < 5 || duration > 120) { + notifyError('INVALID_RECORDING'); + } - updateLoadingText('Memorizing your voice...'); - List> raw = List.from(audioStorage.rawPackets); - var data = await audioStorage.createWavFile(filename: 'speaker_profile.wav'); - try { - await uploadProfile(data.item1); - await uploadProfileBytes(raw, duration); - } catch (e) {} - - updateLoadingText('Personalizing your experience...'); - SharedPreferencesUtil().hasSpeakerProfile = true; - if (isFromOnboarding) { - await createMemory(); - captureProvider?.clearTranscripts(); + String text = segments.map((e) => e.text).join(' ').trim(); + if (text.split(' ').length < (targetWordsCount / 2)) { + // 25 words + notifyError('TOO_SHORT'); + } + uploadingProfile = true; + notifyListeners(); + await _socket?.stop(reason: 'finalizing'); + forceCompletionTimer?.cancel(); + connectionStateListener?.cancel(); + _bleBytesStream?.cancel(); + + updateLoadingText('Memorizing your voice...'); + List> raw = List.from(audioStorage.rawPackets); + var data = await audioStorage.createWavFile(filename: 'speaker_profile.wav'); + try { + await uploadProfile(data.item1); + await uploadProfileBytes(raw, duration); + } catch (e) {} + + updateLoadingText('Personalizing your experience...'); + SharedPreferencesUtil().hasSpeakerProfile = true; + if (_isFromOnboarding) { + await createMemory(); + } + uploadingProfile = false; + profileCompleted = true; + text = ''; + updateLoadingText("You're all set!"); + notifyListeners(); + } finally { + if (_finalizedCallback != null) { + _finalizedCallback!(); + } } - await captureProvider?.resetState(restartBytesProcessing: true); - uploadingProfile = false; - profileCompleted = true; - text = ''; - updateLoadingText("You're all set!"); - notifyListeners(); } // TODO: use connection directly @@ -230,9 +209,10 @@ class SpeechProfileProvider extends ChangeNotifier with MessageNotifierMixin imp onAudioBytesReceived: (List value) { if (value.isEmpty) return; audioStorage.storeFramePacket(value); + value.removeRange(0, 3); - if (webSocketProvider?.wsConnectionState == WebsocketConnectionStatus.connected) { - webSocketProvider?.websocketChannel?.sink.add(value); + if (_socket?.state == SocketServiceState.connected) { + _socket?.send(value); } }, ); @@ -296,8 +276,7 @@ class SpeechProfileProvider extends ChangeNotifier with MessageNotifierMixin imp percentageCompleted = 0; uploadingProfile = false; profileCompleted = false; - await webSocketProvider?.closeWebSocketWithoutReconnect('closing'); - await captureProvider?.resetState(restartBytesProcessing: true, isFromSpeechProfile: true); + await _socket?.stop(reason: 'closing'); notifyListeners(); } @@ -355,7 +334,10 @@ class SpeechProfileProvider extends ChangeNotifier with MessageNotifierMixin imp connectionStateListener?.cancel(); _bleBytesStream?.cancel(); forceCompletionTimer?.cancel(); + _finalizedCallback = null; + _socket?.unsubscribe(this); ServiceManager.instance().device.unsubscribe(this); + super.dispose(); } @@ -386,4 +368,39 @@ class SpeechProfileProvider extends ChangeNotifier with MessageNotifierMixin imp @override void onStatusChanged(DeviceServiceStatus status) {} + + @override + void onClosed() { + // TODO: implement onClosed + } + + @override + void onError(Object err) { + notifyError('WS_ERR'); + } + + @override + void onMessageEventReceived(ServerMessageEvent event) { + // TODO: implement onMessageEventReceived + } + + @override + void onSegmentReceived(List newSegments) { + if (newSegments.isEmpty) return; + if (segments.isEmpty) { + audioStorage.removeFramesRange(fromSecond: 0, toSecond: newSegments[0].start.toInt()); + } + streamStartedAtSecond ??= newSegments[0].start; + + TranscriptSegment.combineSegments( + segments, + newSegments, + toRemoveSeconds: streamStartedAtSecond ?? 0, + ); + updateProgressMessage(); + _validateSingleSpeaker(); + _handleCompletion(); + notifyInfo('SCROLL_DOWN'); + debugPrint('Memory creation timer restarted'); + } } diff --git a/app/lib/providers/websocket_provider.dart b/app/lib/providers/websocket_provider.dart index 1d138b34c..b316956ed 100644 --- a/app/lib/providers/websocket_provider.dart +++ b/app/lib/providers/websocket_provider.dart @@ -10,6 +10,7 @@ import 'package:friend_private/utils/websockets.dart'; import 'package:internet_connection_checker_plus/internet_connection_checker_plus.dart'; import 'package:web_socket_channel/io.dart'; +@Deprecated("Use the socket service") class WebSocketProvider with ChangeNotifier { WebsocketConnectionStatus wsConnectionState = WebsocketConnectionStatus.notConnected; bool websocketReconnecting = false; diff --git a/app/lib/services/device_connections.dart b/app/lib/services/device_connections.dart index 19c1d828f..aab1fef8f 100644 --- a/app/lib/services/device_connections.dart +++ b/app/lib/services/device_connections.dart @@ -45,6 +45,8 @@ abstract class DeviceConnection { DeviceConnectionState get connectionState => _connectionState; + Function(String deviceId, DeviceConnectionState state)? _connectionStateChangedCallback; + DateTime? get pongAt => _pongAt; late StreamSubscription _connectionStateSubscription; @@ -62,8 +64,9 @@ abstract class DeviceConnection { } // Connect + _connectionStateChangedCallback = onConnectionStateChanged; _connectionStateSubscription = bleDevice.connectionState.listen((BluetoothConnectionState state) async { - _onBleConnectionStateChanged(state, onConnectionStateChanged); + _onBleConnectionStateChanged(state); }); await FlutterBluePlus.adapterState.where((val) => val == BluetoothAdapterState.on).first; @@ -82,26 +85,26 @@ abstract class DeviceConnection { _services = await bleDevice.discoverServices(); } - void _onBleConnectionStateChanged( - BluetoothConnectionState state, Function(String deviceId, DeviceConnectionState state)? callback) async { + void _onBleConnectionStateChanged(BluetoothConnectionState state) async { if (state == BluetoothConnectionState.disconnected && _connectionState == DeviceConnectionState.connected) { _connectionState = DeviceConnectionState.disconnected; - await disconnect(callback: callback); + await disconnect(); return; } if (state == BluetoothConnectionState.connected && _connectionState == DeviceConnectionState.disconnected) { _connectionState = DeviceConnectionState.connected; - if (callback != null) { - callback(device.id, _connectionState); + if (_connectionStateChangedCallback != null) { + _connectionStateChangedCallback!(device.id, _connectionState); } } } - Future disconnect({Function(String deviceId, DeviceConnectionState state)? callback}) async { + Future disconnect() async { _connectionState = DeviceConnectionState.disconnected; - if (callback != null) { - callback(device.id, _connectionState); + if (_connectionStateChangedCallback != null) { + _connectionStateChangedCallback!(device.id, _connectionState); + _connectionStateChangedCallback = null; } await bleDevice.disconnect(); _connectionStateSubscription.cancel(); diff --git a/app/lib/services/services.dart b/app/lib/services/services.dart index dc489b15e..127fbeb3d 100644 --- a/app/lib/services/services.dart +++ b/app/lib/services/services.dart @@ -6,10 +6,12 @@ import 'package:flutter/material.dart'; import 'package:flutter_background_service/flutter_background_service.dart'; import 'package:flutter_sound/flutter_sound.dart'; import 'package:friend_private/services/devices.dart'; +import 'package:friend_private/services/sockets.dart'; class ServiceManager { late IMicRecorderService _mic; late IDeviceService _device; + late ISocketService _socket; static ServiceManager? _instance; @@ -19,6 +21,7 @@ class ServiceManager { runner: BackgroundService(), ); sm._device = DeviceService(); + sm._socket = SocketServicePool(); return sm; } @@ -35,6 +38,8 @@ class ServiceManager { IDeviceService get device => _device; + ISocketService get socket => _socket; + static void init() { if (_instance != null) { throw Exception("Service manager is initiated"); diff --git a/app/lib/services/sockets.dart b/app/lib/services/sockets.dart new file mode 100644 index 000000000..4cf3e0a34 --- /dev/null +++ b/app/lib/services/sockets.dart @@ -0,0 +1,77 @@ +import 'package:flutter/material.dart'; +import 'package:friend_private/backend/schema/bt_device.dart'; +import 'package:friend_private/utils/pure_socket.dart'; + +abstract class ISocketService { + void start(); + void stop(); + + Future memory( + {required BleAudioCodec codec, required int sampleRate, bool force = false}); + Future speechProfile( + {required BleAudioCodec codec, required int sampleRate, bool force = false}); +} + +abstract interface class ISocketServiceSubsciption {} + +class SocketServicePool extends ISocketService { + TranscripSegmentSocketService? _socket; + + @override + void start() { + // TODO: implement start + } + + @override + void stop() async { + await _socket?.stop(); + } + + // Warn: Should use a better solution to prevent race conditions + bool mutex = false; + Future socket( + {required BleAudioCodec codec, required int sampleRate, bool force = false}) async { + while (mutex) { + await Future.delayed(const Duration(milliseconds: 50)); + } + mutex = true; + + try { + if (!force && + _socket?.codec == codec && + _socket?.sampleRate == sampleRate && + _socket?.state == SocketServiceState.connected) { + return _socket; + } + + // new socket + await _socket?.stop(); + + _socket = MemoryTranscripSegmentSocketService.create(sampleRate, codec); + await _socket?.start(); + if (_socket?.state != SocketServiceState.connected) { + return null; + } + + return _socket; + } finally { + mutex = false; + } + + return null; + } + + @override + Future memory( + {required BleAudioCodec codec, required int sampleRate, bool force = false}) async { + debugPrint("socket memory > $codec $sampleRate $force"); + return await socket(codec: codec, sampleRate: sampleRate, force: force); + } + + @override + Future speechProfile( + {required BleAudioCodec codec, required int sampleRate, bool force = false}) async { + debugPrint("socket speech profile > $codec $sampleRate $force"); + return await socket(codec: codec, sampleRate: sampleRate, force: force); + } +} diff --git a/app/lib/utils/pure_socket.dart b/app/lib/utils/pure_socket.dart new file mode 100644 index 000000000..33189e4d3 --- /dev/null +++ b/app/lib/utils/pure_socket.dart @@ -0,0 +1,416 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:math'; + +import 'package:flutter/material.dart'; +import 'package:friend_private/backend/preferences.dart'; +import 'package:friend_private/backend/schema/bt_device.dart'; +import 'package:friend_private/backend/schema/message_event.dart'; +import 'package:friend_private/backend/schema/transcript_segment.dart'; +import 'package:friend_private/env/env.dart'; +import 'package:friend_private/services/notification_service.dart'; +import 'package:instabug_flutter/instabug_flutter.dart'; +import 'package:internet_connection_checker_plus/internet_connection_checker_plus.dart'; +import 'package:web_socket_channel/io.dart'; +import 'package:web_socket_channel/status.dart' as socket_channel_status; +import 'package:web_socket_channel/web_socket_channel.dart'; + +enum PureSocketStatus { notConnected, connecting, connected, disconnected } + +abstract class IPureSocketListener { + void onMessage(dynamic message); + void onClosed(); + void onError(Object err, StackTrace trace); + + void onInternetConnectionFailed() {} + + void onMaxRetriesReach() {} +} + +abstract class IPureSocket { + Future connect(); + Future disconnect(); + void send(dynamic message); + + void onInternetSatusChanged(InternetStatus status); + + void onMessage(dynamic message); + void onClosed(); + void onError(Object err, StackTrace trace); +} + +class PureSocketMessage { + String? raw; +} + +class PureCore { + late InternetConnection internetConnection; + + factory PureCore() => _instance; + + /// The singleton instance of [PureCore]. + static final _instance = PureCore.createInstance(); + + PureCore.createInstance() { + internetConnection = InternetConnection.createInstance( + /* + customCheckOptions: [ + InternetCheckOption( + uri: Uri.parse(Env.apiBaseUrl!), + timeout: const Duration( + seconds: 30, + ), + responseStatusFn: (resp) { + return resp.statusCode < 500; + }, + ), + ], + */ + ); + } +} + +class PureSocket implements IPureSocket { + StreamSubscription? _internetStatusListener; + InternetStatus? _internetStatus; + Timer? _internetLostDelayTimer; + + WebSocketChannel? _channel; + WebSocketChannel get channel { + if (_channel == null) { + throw Exception('Socket is not connected'); + } + return _channel!; + } + + PureSocketStatus _status = PureSocketStatus.notConnected; + PureSocketStatus get status => _status; + + IPureSocketListener? _listener; + + int _retries = 0; + + String url; + + PureSocket(this.url) { + _internetStatusListener = PureCore().internetConnection.onStatusChange.listen((InternetStatus status) { + onInternetSatusChanged(status); + }); + } + + void setListener(IPureSocketListener listener) { + _listener = listener; + } + + @override + Future connect() async { + return await _connect(); + } + + Future _connect() async { + if (_status == PureSocketStatus.connecting || _status == PureSocketStatus.connected) { + return false; + } + + _channel = IOWebSocketChannel.connect( + url, + pingInterval: const Duration(seconds: 10), + connectTimeout: const Duration(seconds: 30), + ); + if (_channel?.ready == null) { + return false; + } + + _status = PureSocketStatus.connecting; + await _channel?.ready; + _status = PureSocketStatus.connected; + _retries = 0; + + final that = this; + + _channel?.stream.listen( + (message) { + that.onMessage(message); + }, + onError: (err, trace) { + that.onError(err, trace); + }, + onDone: () { + that.onClosed(); + }, + cancelOnError: true, + ); + + return true; + } + + @override + Future disconnect() async { + if (_status == PureSocketStatus.connected) { + // Warn: should not use await cause dead end by socket closed. + _channel?.sink.close(socket_channel_status.normalClosure); + } + _status = PureSocketStatus.disconnected; + onClosed(); + } + + Future _cleanUp() async { + _internetLostDelayTimer?.cancel(); + _internetStatusListener?.cancel(); + } + + Future stop() async { + await disconnect(); + await _cleanUp(); + } + + @override + void onClosed() { + _status = PureSocketStatus.disconnected; + debugPrint("Socket closed"); + _listener?.onClosed(); + } + + @override + void onError(Object err, StackTrace trace) { + _status = PureSocketStatus.disconnected; + print("Error: ${err}"); + debugPrintStack(stackTrace: trace); + + _listener?.onError(err, trace); + + CrashReporting.reportHandledCrash(err, trace, level: NonFatalExceptionLevel.error); + } + + @override + void onMessage(dynamic message) { + debugPrint("[Socket] Message $message"); + _listener?.onMessage(message); + } + + @override + void send(message) { + _channel?.sink.add(message); + } + + void _reconnect() async { + debugPrint("[Socket] reconnect...${_retries + 1}..."); + const int initialBackoffTimeMs = 1000; // 1 second + const double multiplier = 1.5; + const int maxRetries = 7; + + if (_status == PureSocketStatus.connecting || _status == PureSocketStatus.connected) { + debugPrint("[Socket] Can not reconnect, because socket is $_status"); + return; + } + + await _cleanUp(); + + var ok = await _connect(); + if (ok) { + return; + } + + // retry + int waitInMilliseconds = pow(multiplier, _retries).toInt() * initialBackoffTimeMs; + await Future.delayed(Duration(milliseconds: waitInMilliseconds)); + _retries++; + if (_retries >= maxRetries) { + debugPrint("[Socket] Reach max retries $maxRetries"); + _listener?.onMaxRetriesReach(); + return; + } + _reconnect(); + } + + @override + void onInternetSatusChanged(InternetStatus status) { + debugPrint("[Socket] Internet connection changed $status"); + _internetStatus = status; + switch (status) { + case InternetStatus.connected: + if (_status == PureSocketStatus.connected || _status == PureSocketStatus.connecting) { + return; + } + _reconnect(); + break; + case InternetStatus.disconnected: + var that = this; + _internetLostDelayTimer?.cancel(); + _internetLostDelayTimer = Timer(const Duration(seconds: 60), () async { + if (_internetStatus != InternetStatus.disconnected) { + return; + } + + await that.disconnect(); + _listener?.onInternetConnectionFailed(); + }); + + break; + } + } +} + +abstract interface class ITransctipSegmentSocketServiceListener { + void onMessageEventReceived(ServerMessageEvent event); + void onSegmentReceived(List segments); + void onError(Object err); + void onClosed(); +} + +class SpeechProfileTranscripSegmentSocketService extends TranscripSegmentSocketService { + SpeechProfileTranscripSegmentSocketService.create(super.sampleRate, super.codec) + : super.create(includeSpeechProfile: false, newMemoryWatch: false); +} + +class MemoryTranscripSegmentSocketService extends TranscripSegmentSocketService { + MemoryTranscripSegmentSocketService.create(super.sampleRate, super.codec) + : super.create(includeSpeechProfile: true, newMemoryWatch: true); +} + +enum SocketServiceState { + connected, + disconnected, +} + +class TranscripSegmentSocketService implements IPureSocketListener { + late PureSocket _socket; + final Map _listeners = {}; + + SocketServiceState get state => + _socket.status == PureSocketStatus.connected ? SocketServiceState.connected : SocketServiceState.disconnected; + + int sampleRate; + BleAudioCodec codec; + bool includeSpeechProfile; + bool newMemoryWatch; + + TranscripSegmentSocketService.create( + this.sampleRate, + this.codec, { + this.includeSpeechProfile = false, + this.newMemoryWatch = true, + }) { + final recordingsLanguage = SharedPreferencesUtil().recordingsLanguage; + var params = '?language=$recordingsLanguage&sample_rate=$sampleRate&codec=$codec&uid=${SharedPreferencesUtil().uid}' + '&include_speech_profile=$includeSpeechProfile&new_memory_watch=$newMemoryWatch&stt_service=${SharedPreferencesUtil().transcriptionModel}'; + String url = '${Env.apiBaseUrl!.replaceAll('https', 'wss')}listen$params'; + + _socket = PureSocket(url); + _socket.setListener(this); + } + + void subscribe(Object context, ITransctipSegmentSocketServiceListener listener) { + _listeners.remove(context.hashCode); + _listeners.putIfAbsent(context.hashCode, () => listener); + } + + void unsubscribe(Object context) { + _listeners.remove(context.hashCode); + } + + Future start() async { + bool ok = await _socket.connect(); + if (!ok) { + debugPrint("Can not connect to websocket"); + } + } + + Future stop({String? reason}) async { + await _socket.stop(); + _listeners.clear(); + + if (reason != null) { + debugPrint(reason); + } + } + + Future send(dynamic message) async { + _socket.send(message); + return; + } + + @override + void onClosed() { + _listeners.forEach((k, v) { + v.onClosed(); + }); + } + + @override + void onError(Object err, StackTrace trace) { + _listeners.forEach((k, v) { + v.onError(err); + }); + } + + @override + void onMessage(event) { + debugPrint("[TranscriptSegmentService] onMessage ${event}"); + if (event == 'ping') return; + + // Decode json + dynamic jsonEvent; + try { + jsonEvent = jsonDecode(event); + } on FormatException catch (e) { + debugPrint(e.toString()); + } + if (jsonEvent == null) { + debugPrint("Can not decode message event json $event"); + return; + } + + // Transcript segments + if (jsonEvent is List) { + var segments = jsonEvent; + if (segments.isEmpty) { + return; + } + _listeners.forEach((k, v) { + v.onSegmentReceived(segments.map((e) => TranscriptSegment.fromJson(e)).toList()); + }); + return; + } + + debugPrint(event); + + // Message event + if (jsonEvent.containsKey("type")) { + var event = ServerMessageEvent.fromJson(jsonEvent); + _listeners.forEach((k, v) { + v.onMessageEventReceived(event); + }); + return; + } + + debugPrint(event.toString()); + } + + @override + void onInternetConnectionFailed() { + debugPrint("onInternetConnectionFailed"); + + // Send notification + NotificationService.instance.clearNotification(3); + NotificationService.instance.createNotification( + notificationId: 3, + title: 'Internet Connection Lost', + body: 'Your device is offline. Transcription is paused until connection is restored.', + ); + } + + @override + void onMaxRetriesReach() { + debugPrint("onMaxRetriesReach"); + + // Send notification + NotificationService.instance.clearNotification(2); + NotificationService.instance.createNotification( + notificationId: 2, + title: 'Connection Issue 🚨', + body: 'Unable to connect to the transcript service.' + ' Please restart the app or contact support if the problem persists.', + ); + } +} diff --git a/docs/_assembly/Compile_firmware.md b/docs/_assembly/Compile_firmware.md index bd6d7278b..3677e208f 100644 --- a/docs/_assembly/Compile_firmware.md +++ b/docs/_assembly/Compile_firmware.md @@ -11,11 +11,10 @@ Important: If you purchased an assembled device please skip this step If you purchased an unassembled Friend device or built it yourself using our hardware guide, follow the steps below to flash the firmware: -### Official firmware: +### Want to install a pre-built firmware? Navigate [here](https://docs.omi.me/get_started/Flash_device/) -Go to [Releases](https://github.com/BasedHardware/Omi/releases) in Github, and find the latest official firmware release. To use this firmware, simply download it and skip to step 6. If you would like to build the firmware yourself please follow all the steps below. -### Build your own firmware: +## Build your own firmware: 1. Set up nRF Connect by following the tutorial in this video: [https://youtu.be/EAJdOqsL9m8](https://youtu.be/EAJdOqsL9m8) diff --git a/docs/_assembly/Install_firmware.md b/docs/_assembly/Install_firmware.md new file mode 100644 index 000000000..3ac6a266b --- /dev/null +++ b/docs/_assembly/Install_firmware.md @@ -0,0 +1,7 @@ +--- +layout: default +title: Install firmware (old) +nav_order: 3 +--- + +We've moved! please navigate [here](https://docs.omi.me/get_started/Flash_device/) diff --git a/docs/_get_started/Flash_device.md b/docs/_get_started/Flash_device.md index 2f9bc4ba9..b2ee810ce 100644 --- a/docs/_get_started/Flash_device.md +++ b/docs/_get_started/Flash_device.md @@ -1,6 +1,6 @@ --- layout: default -title: Flashing FRIEND Firmware +title: Update FRIEND Firmware nav_order: 3 --- # Video Tutorial @@ -13,9 +13,13 @@ This guide will walk you through the process of flashing the latest firmware ont ## Downloading the Firmware -1. Go to the [FRIEND GitHub repository](https://github.com/BasedHardware/Omi) and navigate to the "Devices > FRIEND > firmware" section. +1. Go to the [FRIEND GitHub repository](https://github.com/BasedHardware/Omi) and navigate to the " FRIEND > firmware" section. 2. Find the latest firmware release and bootloader, then download the corresponding `.uf2` files. +Or download these files + - **Bootloader:** [bootloader0.9.0.uf2](https://github.com/ebowwa/omi/blob/firmware-flashing-readme/devices/Friend/firmware/bootloader/bootloader0.9.0.uf2) + - **Firmware:** [firmware1.0.4.uf2](https://github.com/ebowwa/omi/blob/firmware-flashing-readme/devices/Friend/firmware/firmware1.0.4.uf2) + ## Putting FRIEND into DFU Mode 1. **Locate the DFU Button:** Find the small pin-sized button on the FRIEND device's circuit board (refer to the image below if needed). @@ -45,4 +49,4 @@ You have successfully flashed the latest firmware onto your FRIEND device. You c Once you've installed the app, follow the in-app instructions to connect your FRIEND device and start exploring its features. -i just added this video to the repo docs/images/updating_your_friend.mov add it to this \ No newline at end of file +i just added this video to the repo docs/images/updating_your_friend.mov add it to this