Skip to content

Commit

Permalink
Keep capturing memory when disconnects happened
Browse files Browse the repository at this point in the history
  • Loading branch information
@beastoin
beastoin committed Sep 27, 2024
1 parent 7ac3a81 commit 6e6e88e
Show file tree
Hide file tree
Showing 6 changed files with 159 additions and 47 deletions.
27 changes: 27 additions & 0 deletions app/lib/backend/http/api/processing_memories.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,3 +46,30 @@ Future<UpdateProcessingMemoryResponse?> updateProcessingMemoryServer({
}
return null;
}

Future<ProcessingMemoryResponse?> fetchProcessingMemoryServer({
required String id,
}) async {
var response = await makeApiCall(
url: '${Env.apiBaseUrl}v1/processing-memories/$id',
headers: {},
method: 'GET',
body: "",
);
if (response == null) return null;
debugPrint('updateProcessingMemoryServer: ${response.body}');
if (response.statusCode == 200) {
return ProcessingMemoryResponse.fromJson(jsonDecode(response.body));
} else {
// TODO: Server returns 304 doesn't recover
CrashReporting.reportHandledCrash(
Exception('Failed to create memory'),
StackTrace.current,
level: NonFatalExceptionLevel.info,
userAttributes: {
'response': response.body,
},
);
}
return null;
}
37 changes: 37 additions & 0 deletions app/lib/backend/schema/memory.dart
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import 'dart:convert';
import 'dart:math';

import 'package:collection/collection.dart';
import 'package:flutter/material.dart';
import 'package:friend_private/backend/schema/geolocation.dart';
import 'package:friend_private/backend/schema/message.dart';
Expand Down Expand Up @@ -57,22 +58,44 @@ class MemoryPostProcessing {
toJson() => {'status': status.toString().split('.').last, 'model': model.toString().split('.').last};
}

enum ServerProcessingMemoryStatus {
capturing('capturing'),
processing('processing'),
done('done'),
unknown('unknown'),
;

final String value;
const ServerProcessingMemoryStatus(this.value);

static ServerProcessingMemoryStatus valuesFromString(String value) {
return ServerProcessingMemoryStatus.values.firstWhereOrNull((e) => e.value == value) ??
ServerProcessingMemoryStatus.unknown;
}
}

class ServerProcessingMemory {
final String id;
final DateTime createdAt;
final DateTime? startedAt;
final DateTime? capturingTo;
final ServerProcessingMemoryStatus? status;

ServerProcessingMemory({
required this.id,
required this.createdAt,
this.startedAt,
this.capturingTo,
this.status,
});

factory ServerProcessingMemory.fromJson(Map<String, dynamic> json) {
return ServerProcessingMemory(
id: json['id'],
createdAt: DateTime.parse(json['created_at']).toLocal(),
startedAt: json['started_at'] != null ? DateTime.parse(json['started_at']).toLocal() : null,
capturingTo: json['capturing_to'] != null ? DateTime.parse(json['capturing_to']).toLocal() : null,
status: json['status'] != null ? ServerProcessingMemoryStatus.valuesFromString(json['status']) : null,
);
}

Expand All @@ -81,6 +104,8 @@ class ServerProcessingMemory {
'id': id,
'created_at': createdAt.toUtc().toIso8601String(),
'started_at': startedAt?.toUtc().toIso8601String(),
'capturing_to': capturingTo?.toUtc().toIso8601String(),
'status': status.toString(),
};
}

Expand All @@ -97,6 +122,18 @@ class ServerProcessingMemory {
}
}

class ProcessingMemoryResponse {
final ServerProcessingMemory? result;

ProcessingMemoryResponse({required this.result});

factory ProcessingMemoryResponse.fromJson(Map<String, dynamic> json) {
return ProcessingMemoryResponse(
result: json['result'] != null ? ServerProcessingMemory.fromJson(json['result']) : null,
);
}
}

class UpdateProcessingMemoryResponse {
final ServerProcessingMemory? result;

Expand Down
37 changes: 0 additions & 37 deletions app/lib/pages/memories/widgets/capture.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,50 +77,13 @@ class LiteCaptureWidgetState extends State<LiteCaptureWidget>
super.dispose();
}

// TODO: use connection directly
Future<BleAudioCodec> _getAudioCodec(String deviceId) async {
var connection = await ServiceManager.instance().device.ensureConnection(deviceId);
if (connection == null) {
return BleAudioCodec.pcm8;
}
return connection.getAudioCodec();
}
// Future requestLocationPermission() async {
// LocationService locationService = LocationService();
// bool serviceEnabled = await locationService.enableService();
// if (!serviceEnabled) {
// debugPrint('Location service not enabled');
// if (mounted) {
// ScaffoldMessenger.of(context).showSnackBar(
// const SnackBar(
// content: Text(
// 'Location services are disabled. Enable them for a better experience.',
// style: TextStyle(color: Colors.white, fontSize: 14),
// ),
// ),
// );
// }
// } else {
// PermissionStatus permissionGranted = await locationService.requestPermission();
// SharedPreferencesUtil().locationEnabled = permissionGranted == PermissionStatus.granted;
// MixpanelManager().setUserProperty('Location Enabled', SharedPreferencesUtil().locationEnabled);
// if (permissionGranted == PermissionStatus.denied) {
// debugPrint('Location permission not granted');
// } else if (permissionGranted == PermissionStatus.deniedForever) {
// debugPrint('Location permission denied forever');
// if (mounted) {
// ScaffoldMessenger.of(context).showSnackBar(
// const SnackBar(
// content: Text(
// 'If you change your mind, you can enable location services in your device settings.',
// style: TextStyle(color: Colors.white, fontSize: 14),
// ),
// ),
// );
// }
// }
// }
// }

@override
Widget build(BuildContext context) {
Expand Down
36 changes: 33 additions & 3 deletions app/lib/pages/memories/widgets/processing_capture.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ class MemoryCaptureWidget extends StatefulWidget {
}

class _MemoryCaptureWidgetState extends State<MemoryCaptureWidget> {

@override
Widget build(BuildContext context) {
return Consumer3<CaptureProvider, DeviceProvider, ConnectivityProvider>(
Expand All @@ -38,8 +37,9 @@ class _MemoryCaptureWidgetState extends State<MemoryCaptureWidget> {
bool isConnected = deviceProvider.connectedDevice != null ||
provider.recordingState == RecordingState.record ||
(provider.memoryCreating && deviceProvider.connectedDevice != null);
bool havingCapturingMemory = provider.capturingProcessingMemory != null;

return (showPhoneMic || isConnected)
return (showPhoneMic || isConnected || havingCapturingMemory)
? GestureDetector(
onTap: () async {
if (provider.segments.isEmpty && provider.photos.isEmpty) return;
Expand Down Expand Up @@ -108,6 +108,10 @@ class _MemoryCaptureWidgetState extends State<MemoryCaptureWidget> {
}

_getMemoryHeader(BuildContext context) {
// Processing memory
var provider = context.read<CaptureProvider>();
bool havingCapturingMemory = provider.capturingProcessingMemory != null;

// Connected device
var deviceProvider = context.read<DeviceProvider>();

Expand Down Expand Up @@ -138,9 +142,35 @@ class _MemoryCaptureWidgetState extends State<MemoryCaptureWidget> {
child: Row(
mainAxisAlignment: MainAxisAlignment.spaceBetween,
children: [
deviceProvider.connectedDevice == null &&
!deviceProvider.isConnecting &&
!isUsingPhoneMic &&
havingCapturingMemory
? Row(
children: [
const Text(
'🎙️',
style: TextStyle(color: Colors.white, fontSize: 22, fontWeight: FontWeight.w600),
),
const SizedBox(width: 12),
Container(
decoration: BoxDecoration(
color: Colors.grey.shade800,
borderRadius: BorderRadius.circular(16),
),
padding: const EdgeInsets.symmetric(horizontal: 12, vertical: 6),
child: Text(
'Waiting for reconnect...',
style: Theme.of(context).textTheme.bodyMedium!.copyWith(color: Colors.white),
maxLines: 1,
),
),
],
)
: const SizedBox.shrink(),
// mic
// TODO: improve phone recording UI
deviceProvider.connectedDevice == null && !deviceProvider.isConnecting
deviceProvider.connectedDevice == null && !deviceProvider.isConnecting && !havingCapturingMemory
? Center(
child: getPhoneMicRecordingButton(
context,
Expand Down
67 changes: 61 additions & 6 deletions app/lib/providers/capture_provider.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,8 @@ class CaptureProvider extends ChangeNotifier
// -----------------------

String? processingMemoryId;
ServerProcessingMemory? capturingProcessingMemory;
Timer? _processingMemoryWatchTimer;

String dateTimeStorageString = "";

Expand Down Expand Up @@ -151,9 +153,57 @@ class CaptureProvider extends ChangeNotifier

Future<void> _onProcessingMemoryCreated(String processingMemoryId) async {
this.processingMemoryId = processingMemoryId;

// Fetch and watch capturing status
ProcessingMemoryResponse? result = await fetchProcessingMemoryServer(
id: processingMemoryId,
);
if (result?.result == null) {
debugPrint("Can not fetch processing memory, result null");
}
_setCapturingProcessingMemory(result?.result);

// Update processing memory
_updateProcessingMemory();
}

void _setCapturingProcessingMemory(ServerProcessingMemory? pm) {
debugPrint("${pm?.toJson()}");
if (pm != null &&
pm.status == ServerProcessingMemoryStatus.capturing &&
pm.capturingTo != null &&
pm.capturingTo!.isAfter(DateTime.now())) {
capturingProcessingMemory = pm;

// TODO: thinh, might set pre-segments
} else {
capturingProcessingMemory = null;
}
notifyListeners();

// End
if (capturingProcessingMemory == null) {
return;
}

// Or watch
var id = capturingProcessingMemory!.id;
var delayMs = capturingProcessingMemory?.capturingTo != null
? capturingProcessingMemory!.capturingTo!.millisecondsSinceEpoch - DateTime.now().millisecondsSinceEpoch
: 2 * 60 * 1000; // 2m
if (delayMs > 0) {
_processingMemoryWatchTimer?.cancel();
_processingMemoryWatchTimer = Timer(Duration(milliseconds: delayMs), () async {
ProcessingMemoryResponse? result = await fetchProcessingMemoryServer(id: id);
if (result?.result == null) {
debugPrint("Can not fetch processing memory, result null");
return;
}
_setCapturingProcessingMemory(result?.result);
});
}
}

Future<void> _onMemoryCreated(ServerMessageEvent event) async {
if (event.memory == null) {
debugPrint("Memory is not found, processing memory ${event.processingMemoryId}");
Expand Down Expand Up @@ -313,7 +363,10 @@ class CaptureProvider extends ChangeNotifier
secondsMissedOnReconnect = null;
photos = [];
conversationId = const Uuid().v4();

processingMemoryId = null;
capturingProcessingMemory = null;
_processingMemoryWatchTimer?.cancel();
}

Future _cleanNew() async {
Expand Down Expand Up @@ -656,6 +709,7 @@ class CaptureProvider extends ChangeNotifier
_memoryCreationTimer?.cancel();
_socket?.unsubscribe(this);
_keepAliveTimer?.cancel();
_processingMemoryWatchTimer?.cancel();
super.dispose();
}

Expand Down Expand Up @@ -715,12 +769,13 @@ class CaptureProvider extends ChangeNotifier
_transcriptServiceReady = false;
debugPrint('[Provider] Socket is closed');

_clean();

// Notify
setMemoryCreating(false);
setHasTranscripts(false);
notifyListeners();
// Wait reconnect
if (capturingProcessingMemory == null) {
_clean();
setMemoryCreating(false);
setHasTranscripts(false);
notifyListeners();
}

// Keep alived
_startKeepAlivedServices();
Expand Down
2 changes: 1 addition & 1 deletion backend/models/processing_memory.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,4 +60,4 @@ class BasicProcessingMemoryResponse(BaseModel):
result: BasicProcessingMemory

class BasicProcessingMemoriesResponse(BaseModel):
result: [BasicProcessingMemory]
result: List[BasicProcessingMemory]

0 comments on commit 6e6e88e

Please sign in to comment.