From ef122aa34b001f8b8b58489e316c648e82f891ee Mon Sep 17 00:00:00 2001 From: kevvz Date: Wed, 28 Aug 2024 19:00:58 -0700 Subject: [PATCH 1/2] make the thing compile first try --- .../prj_xiao_ble_sense_devkitv2-adafruit.conf | 12 ++++++------ Friend/firmware/firmware_v1.0/src/transport.c | 1 - 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Friend/firmware/firmware_v1.0/prj_xiao_ble_sense_devkitv2-adafruit.conf b/Friend/firmware/firmware_v1.0/prj_xiao_ble_sense_devkitv2-adafruit.conf index 3ff66375a..6c2ce270a 100644 --- a/Friend/firmware/firmware_v1.0/prj_xiao_ble_sense_devkitv2-adafruit.conf +++ b/Friend/firmware/firmware_v1.0/prj_xiao_ble_sense_devkitv2-adafruit.conf @@ -119,12 +119,12 @@ CONFIG_OFFLINE_STORAGE=y # CONFIG_SPI_SDHC=y # SD Card Support -CONFIG_DISK_DRIVER_SDMMC=y -CONFIG_MMC_STACK=y -CONFIG_SDMMC_STACK=y -CONFIG_SPI=y -CONFIG_SDHC=y -CONFIG_SPI_SDHC=y +# CONFIG_DISK_DRIVER_SDMMC=y +# CONFIG_MMC_STACK=y +# CONFIG_SDMMC_STACK=y +# CONFIG_SPI=y +# CONFIG_SDHC=y +# CONFIG_SPI_SDHC=y # File System CONFIG_FILE_SYSTEM=y diff --git a/Friend/firmware/firmware_v1.0/src/transport.c b/Friend/firmware/firmware_v1.0/src/transport.c index 5b5f24d18..e9f1aedb3 100644 --- a/Friend/firmware/firmware_v1.0/src/transport.c +++ b/Friend/firmware/firmware_v1.0/src/transport.c @@ -29,7 +29,6 @@ uint16_t current_package_index = 0; // static struct bt_conn_cb _callback_references; -struct bt_conn *current_connection = NULL; static void audio_ccc_config_changed_handler(const struct bt_gatt_attr *attr, uint16_t value); static ssize_t audio_data_read_characteristic(struct bt_conn *conn, const struct bt_gatt_attr *attr, void *buf, uint16_t len, uint16_t offset); static ssize_t audio_codec_read_characteristic(struct bt_conn *conn, const struct bt_gatt_attr *attr, void *buf, uint16_t len, uint16_t offset); From 5810845e4e304a1723aa9c327d7b2fa83ebfcb78 Mon Sep 17 00:00:00 2001 From: kevvz Date: Sat, 7 Sep 2024 18:14:23 -0700 Subject: [PATCH 2/2] added commands to storage write + sd card support on app --- Friend/firmware/firmware_v1.0/src/sdcard.c | 19 +++ Friend/firmware/firmware_v1.0/src/storage.c | 109 ++++++++++++++--- Friend/firmware/firmware_v1.0/src/transport.c | 4 +- Friend/firmware/testing/decode_audio.py | 29 +++-- Friend/firmware/testing/get_audio_file.py | 18 +-- Friend/firmware/testing/storage.py | 45 +++++++ app/lib/backend/http/api/memories.dart | 29 ++++- app/lib/providers/capture_provider.dart | 55 ++++++++- app/lib/utils/audio/wav_bytes.dart | 95 +++++++++++++++ app/lib/utils/ble/communication.dart | 19 ++- app/lib/utils/ble/device_base.dart | 36 ++++++ app/lib/utils/ble/frame_communication.dart | 21 ++++ app/lib/utils/ble/friend_communication.dart | 113 ++++++++++++++++++ app/lib/utils/ble/gatt_utils.dart | 4 + 14 files changed, 546 insertions(+), 50 deletions(-) create mode 100644 Friend/firmware/testing/storage.py diff --git a/Friend/firmware/firmware_v1.0/src/sdcard.c b/Friend/firmware/firmware_v1.0/src/sdcard.c index f0f6daea3..6a53413e1 100644 --- a/Friend/firmware/firmware_v1.0/src/sdcard.c +++ b/Friend/firmware/firmware_v1.0/src/sdcard.c @@ -297,4 +297,23 @@ int get_next_item(struct fs_dir_t *zdp, struct fs_dirent *entry) { count++; } return count; +} +//we should clear instead of delete since we lose fifo structure +int clear_audio_file(uint8_t num) { + + char *ptr = generate_new_audio_header(num); + snprintf(current_full_path, sizeof(current_full_path), "%s%s", disk_mount_pt, ptr); + free(ptr); + int res = fs_unlink(¤t_full_path); + if (res) { + printk("error deleting file\n"); + return -1; + } + res = create_file(¤t_full_path); + if (res) { + printk("error creating file\n"); + return -1; + } + + return 0; } \ No newline at end of file diff --git a/Friend/firmware/firmware_v1.0/src/storage.c b/Friend/firmware/firmware_v1.0/src/storage.c index 27e043839..102d47f8c 100644 --- a/Friend/firmware/firmware_v1.0/src/storage.c +++ b/Friend/firmware/firmware_v1.0/src/storage.c @@ -18,6 +18,8 @@ LOG_MODULE_REGISTER(storage, CONFIG_LOG_DEFAULT_LEVEL); #define OPUS_ENTRY_LENGTH 100 #define FRAME_PREFIX_LENGTH 3 +#define READ_COMMAND 0 +#define DELETE_COMMAND 1 static void storage_config_changed_handler(const struct bt_gatt_attr *attr, uint16_t value); static ssize_t storage_write_handler(struct bt_conn *conn, const struct bt_gatt_attr *attr, const void *buf, uint16_t len, uint16_t offset, uint8_t flags); @@ -68,7 +70,7 @@ static void storage_config_changed_handler(const struct bt_gatt_attr *attr, uint static ssize_t storage_read_characteristic(struct bt_conn *conn, const struct bt_gatt_attr *attr, void *buf, uint16_t len, uint16_t offset) { k_msleep(10); // char amount[1] = {file_count}; - uint32_t amount[20] = {0}; + uint32_t amount[50] = {0}; for (int i = 0; i < file_count; i++) { amount[i] = file_num_array[i]; } @@ -79,15 +81,6 @@ static ssize_t storage_read_characteristic(struct bt_conn *conn, const struct bt uint8_t transport_started = 0; -static ssize_t storage_write_handler(struct bt_conn *conn, const struct bt_gatt_attr *attr, const void *buf, uint16_t len, uint16_t offset, uint8_t flags) { - - LOG_INF("about to schedule the storage"); - transport_started = 1; - k_msleep(1000); - - - return len; -} static uint16_t packet_next_index = 0; @@ -98,17 +91,18 @@ static uint8_t index = 0; static uint8_t current_packet_size = 0; static uint8_t tx_buffer_size = 0; - +static uint8_t delete_started = 0; static uint8_t current_read_num = 1; uint32_t remaining_length = 0; -static int setup_storage_tx(void) { +static int setup_storage_tx() { transport_started= (uint8_t)0; - offset = 0; - + // offset = 0; + LOG_INF("about to transmit storage\n"); + k_msleep(1000); int res = move_read_pointer(current_read_num); if (res) { - printk("bad pointer\n"); + LOG_ERR("bad pointer\n"); transport_started = 0; // current_read_num++; current_read_num = 1; @@ -116,20 +110,89 @@ static int setup_storage_tx(void) { return -1; } else { - if ((uint32_t)get_file_size(current_read_num) == 0 ) { + if (file_num_array[current_read_num-1] == 0 ) { // LOG_ERR("bad file size, moving again..."); - current_read_num++; + // file_num++; move_read_pointer(current_read_num); } printk("current read ptr %d\n",current_read_num); - remaining_length = get_file_size(current_read_num); - LOG_INF("remaining length: %d",remaining_length); + remaining_length = file_num_array[current_read_num-1]; + remaining_length = remaining_length - offset; + // offset=offset_; + printk("remaining length: %d\n",remaining_length); + LOG_INF("offset: %d\n",offset); + printk("file: %d\n",current_read_num); + } + return 0; + +} + +static int parse_storage_command(void *buf,uint16_t len) { + + if (len != 6 && len != 2) { + printk("not the exact length for command"); + return -2; + } + uint8_t command = ((uint8_t*)buf)[0]; + uint8_t file_num = ((uint8_t*)buf)[1]; + uint32_t size = 0; + if ( len == 6 ) { + + size = ((uint8_t*)buf)[2] <<24 |((uint8_t*)buf)[3] << 16 | ((uint8_t*)buf)[4] << 8 | ((uint8_t*)buf)[5]; + + } + printk("command successful: command: %d file: %d size: %d \n",command,file_num,size); + + if (file_num == 0) { + printk("invalid file count 0\n"); + return -3; + } + if (file_num > file_count) { //invalid file count + printk("invalid file count\n"); + return -3; +//add audio all? + } + if (command == READ_COMMAND) { //read + uint32_t temp = file_num_array[file_num-1]; + if (temp == 0) { + printk("file size is 0\n"); + return -4; + } + if (size > temp) { + printk("requested size is too large\n"); + return -5; + } + else { + printk("valid command, setting up \n"); + offset = size; + current_read_num = file_num; + transport_started = 1; + } + } + else if (command == DELETE_COMMAND) { + printk("delete something\n"); + + } + else { + printk("invalid command \n"); + return -6; } return 0; } +static ssize_t storage_write_handler(struct bt_conn *conn, const struct bt_gatt_attr *attr, const void *buf, uint16_t len, uint16_t offset, uint8_t flags) { + + LOG_INF("about to schedule the storage"); + printk("was sent %d \n ", ((uint8_t*)buf)[0] ); + int result = parse_storage_command(buf,len); + printk("length of storage write: %d\n",len); + printk("result: %d \n", result); + + k_msleep(1000); + return len; +} static void write_to_gatt(struct bt_conn *conn) { uint32_t id = packet_next_index++; @@ -158,6 +221,11 @@ void storage_write(void) { setup_storage_tx(); } + // if (delete_started) { + + // delete_started = 0; + // } + if(remaining_length > 0 ) { struct bt_conn *conn = get_current_connection(); @@ -170,6 +238,9 @@ void storage_write(void) { transport_started = 0; if (remaining_length == 0) { printk("done. attempting to download more files\n"); + uint8_t stop_result[2] = {100,100}; + int err = bt_gatt_notify(conn, &storage_service.attrs[1], &stop_result,1); + // current_read_num++; k_sleep(K_MSEC(10)); diff --git a/Friend/firmware/firmware_v1.0/src/transport.c b/Friend/firmware/firmware_v1.0/src/transport.c index c7ccd454c..a24b88f93 100644 --- a/Friend/firmware/firmware_v1.0/src/transport.c +++ b/Friend/firmware/firmware_v1.0/src/transport.c @@ -540,7 +540,7 @@ bool write_to_storage(void) { return false; } - uint8_t *buffer = tx_buffer+3; + uint8_t *buffer = tx_buffer+2; uint32_t packet_size = tx_buffer_size; memset(storage_temp_data, 0, OPUS_PADDED_LENGTH); @@ -597,7 +597,7 @@ void pusher(void) bool result = write_to_storage(); file_num_array[file_count-1] = get_file_size(file_count); - printk("file size for file count %d %d\n",file_count,file_num_array[file_count-1]); + // printk("file size for file count %d %d\n",file_count,file_num_array[file_count-1]); if (result) diff --git a/Friend/firmware/testing/decode_audio.py b/Friend/firmware/testing/decode_audio.py index 2a0f1a967..97fd4b0f1 100644 --- a/Friend/firmware/testing/decode_audio.py +++ b/Friend/firmware/testing/decode_audio.py @@ -15,25 +15,24 @@ sample_frame = info_char[i*103:(i+1)*103] amount = int(sample_frame[3]) frame_to_decode = bytes(list(sample_frame[4:4+amount])) - print(frame_to_decode) - print(len(frame_to_decode)) + f.append(frame_to_decode) - print(np.frombuffer(frame_to_decode,dtype=np.uint8)) + # print(i) opus_frame = opus_decoder.decode(bytes(frame_to_decode), 160,decode_fec=False) - # for frame in f: - # try: - # decoded_frame = opus_decoder.decode(bytes(frame), 960) - # pcm_data.extend(decoded_frame) - # except Exception as e: - # print(f"Error decoding frame: {e}") - # count+=1 + for frame in f: + try: + decoded_frame = opus_decoder.decode(bytes(frame), 320) + pcm_data.extend(decoded_frame) + except Exception as e: + print(f"Error decoding frame: {e}") + count+=1 - # with wave.open('decoded_audio.wav', 'wb') as wav_file: - # wav_file.setnchannels(1) # Mono - # wav_file.setsampwidth(2) # 16-bit - # wav_file.setframerate(16000) # Sample rate - # wav_file.writeframes(pcm_data) + with wave.open('decoded_audio.wav', 'wb') as wav_file: + wav_file.setnchannels(1) # Mono + wav_file.setsampwidth(2) # 16-bit + wav_file.setframerate(16000) # Sample rate + wav_file.writeframes(pcm_data) # print(count) diff --git a/Friend/firmware/testing/get_audio_file.py b/Friend/firmware/testing/get_audio_file.py index 39988089b..bab67d246 100644 --- a/Friend/firmware/testing/get_audio_file.py +++ b/Friend/firmware/testing/get_audio_file.py @@ -36,18 +36,22 @@ async def on_notify(sender: bleak.BleakGATTCharacteristic, data: bytearray): global count global done print(len(data)) - amount_to_append = data[3] - audio_frames.append(data[4:data[3]+4]) - count +=1 - print(np.frombuffer(data,dtype=np.uint8)) + if (len(data)==1): + print(data[0]) + else: + amount_to_append = data[3] + audio_frames.append(data[4:data[3]+4]) + count +=1 + print(np.frombuffer(data,dtype=np.uint8)) stuff = await client.start_notify(storage_uuid, on_notify) - print('b') + # await client.read_gatt_char(storage_read_uuid) await asyncio.sleep(1) - await client.write_gatt_char(storage_uuid, b'd', response=True) - print('c') + command = bytearray([0,1,0,0,0,0]) + await client.write_gatt_char(storage_uuid, command, response=True) + print(stuff) await asyncio.sleep(1) while True: diff --git a/Friend/firmware/testing/storage.py b/Friend/firmware/testing/storage.py new file mode 100644 index 000000000..ec538d0a8 --- /dev/null +++ b/Friend/firmware/testing/storage.py @@ -0,0 +1,45 @@ +import asyncio +import os +import threading +import time +# from deepgram import ( +# DeepgramClient, +# PrerecordedOptions, +# FileSource, +# ) + +from fastapi import APIRouter, FastAPI,Depends, HTTPException, UploadFile + +app = FastAPI() + + +@app.get("/memory") +async def root(): + return {"message": "sexp"} + +@app.post("/download_wav") +async def download_wav(file: UploadFile): + with open("downloaded_wav_file.wav", "wb") as f: + f.write(file.file.read()) + # try: + # deepgram = DeepgramClient("DEEPGRAM_API_KEY") + + # with open(AUDIO_FILE, "rb") as file: + # buffer_data = file.read() + + # payload: FileSource = { + # "buffer": buffer_data, + # } + + # options = PrerecordedOptions( + # model="nova-2", + # smart_format=True, + # ) + + # response = deepgram.listen.rest.v("1").transcribe_file(payload, options) + + # print(response.to_json(indent=4)) + + # except Exception as e: + # print(f"Exception: {e}") + \ No newline at end of file diff --git a/app/lib/backend/http/api/memories.dart b/app/lib/backend/http/api/memories.dart index 343e3ecf1..36aa243bd 100644 --- a/app/lib/backend/http/api/memories.dart +++ b/app/lib/backend/http/api/memories.dart @@ -78,7 +78,7 @@ Future memoryPostProcessing(File file, String memoryId) async { ); request.files.add(await http.MultipartFile.fromPath('file', file.path, filename: basename(file.path))); request.headers.addAll({'Authorization': await getAuthHeader()}); - + try { var streamedResponse = await request.send(); var response = await http.Response.fromStream(streamedResponse); @@ -234,3 +234,30 @@ Future setMemoryEventsState( debugPrint('setMemoryEventsState: ${response.body}'); return response.statusCode == 200; } + + +Future sendStorageToBackend(File file, String memoryId) async { + var request = http.MultipartRequest( + 'POST', + // Uri.parse('${Env.apiBaseUrl}v1/memories/$memoryId/post-processing?emotional_feedback=$optEmotionalFeedback'), + Uri.parse('http://127.0.0.1:8000/download_wav'), + ); + request.files.add(await http.MultipartFile.fromPath('file', file.path, filename: basename(file.path))); + + try { + var streamedResponse = await request.send(); + var response = await http.Response.fromStream(streamedResponse); + + if (response.statusCode == 200) { + debugPrint('storageSend Response body: ${jsonDecode(response.body)}'); + + } else { + debugPrint('Failed to storageSend. Status code: ${response.statusCode}'); + return null; + } + } catch (e) { + debugPrint('An error occurred storageSend: $e'); + return null; + } + +} \ No newline at end of file diff --git a/app/lib/providers/capture_provider.dart b/app/lib/providers/capture_provider.dart index 223853bad..49ee883f2 100644 --- a/app/lib/providers/capture_provider.dart +++ b/app/lib/providers/capture_provider.dart @@ -62,6 +62,9 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie get bleBytesStream => _bleBytesStream; + StreamSubscription? _storageStream; + get storageStream => _storageStream; + var record = AudioRecorder(); RecordingState recordingState = RecordingState.stop; @@ -76,7 +79,8 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie DateTime? currentTranscriptStartedAt; DateTime? currentTranscriptFinishedAt; int elapsedSeconds = 0; - + List currentStorageFiles = []; + StorageBytesUtil? storageUtil; // ----------------------- bool resetStateAlreadyCalled = false; @@ -111,7 +115,7 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie connectedDevice = device; notifyListeners(); } - + //create the memory here Future createMemory({bool forcedCreation = false}) async { debugPrint('_createMemory forcedCreation: $forcedCreation'); if (memoryCreating) return null; @@ -130,7 +134,7 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie } // in case was a local recording and not a BLE recording } - ServerMemory? memory = await processTranscriptContent( + ServerMemory? memory = await processTranscriptContent( //create mmeory "shell" segments: segments, startedAt: currentTranscriptStartedAt, finishedAt: currentTranscriptFinishedAt, @@ -178,7 +182,7 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie if (memory != null && !memory.failed && file != null && segments.isNotEmpty && !memory.discarded) { setMemoryCreating(false); try { - memoryPostProcessing(file, memory.id).then((postProcessed) { + memoryPostProcessing(file, memory.id).then((postProcessed) { //fill the shell with audio file if (postProcessed != null) { memoryProvider?.updateMemory(postProcessed); } else { @@ -331,6 +335,36 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie notifyListeners(); } + Future sendStorage(String id) async { + storageUtil = StorageBytesUtil(); + // storageUtil.currentStorageList = currentStorageFiles; + + if (_storageStream != null) { + _storageStream?.cancel(); + } + _storageStream = await getBleStorageBytesListener( + id, + onStorageBytesReceived: (List value) async { + if (value.isEmpty) return; + storageUtil!.storeFrameStoragePacket(value); + if (value.length == 1) { + if (value[0] == 100) { + debugPrint('done. sending to backend....'); + File storageFile = (await storageUtil!.createWavFile(removeLastNSeconds:0)).item1; + sendStorageToBackend(storageFile, "hi"); + } + } + } + ); + + writeToStorage(id); + + // notifyListeners(); + } + + // Future saveAndSendStorageWav() async { + // } + void clearTranscripts() { segments = []; SharedPreferencesUtil().transcriptSegments = []; @@ -370,6 +404,7 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie } await initiateFriendAudioStreaming(isFromSpeechProfile); + await initiateStorageBytesStreaming(); setResetStateAlreadyCalled(false); notifyListeners(); @@ -416,7 +451,7 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie Future initiateFriendAudioStreaming(bool isFromSpeechProfile) async { print('connectedDevice: $connectedDevice in initiateFriendAudioStreaming'); if (connectedDevice == null) return; - + BleAudioCodec codec = await getAudioCodec(connectedDevice!.id); if (SharedPreferencesUtil().deviceCodec != codec) { debugPrint('Device codec changed from ${SharedPreferencesUtil().deviceCodec} to $codec'); @@ -432,6 +467,14 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie notifyListeners(); } + Future initiateStorageBytesStreaming() async { + debugPrint('initiateStorageBytesStreaming'); + currentStorageFiles = await getStorageList(connectedDevice!.id); + debugPrint('Storage files: $currentStorageFiles'); + await sendStorage(connectedDevice!.id); + notifyListeners(); + } + processCachedTranscript() async { // TODO: only applies to friend, not openglass, fix it var segments = SharedPreferencesUtil().transcriptSegments; @@ -526,4 +569,6 @@ class CaptureProvider extends ChangeNotifier with OpenGlassMixin, MessageNotifie stopStreamRecordingOnAndroid() { stopBackgroundService(); } + + } diff --git a/app/lib/utils/audio/wav_bytes.dart b/app/lib/utils/audio/wav_bytes.dart index 8d8bd5770..9027561db 100644 --- a/app/lib/utils/audio/wav_bytes.dart +++ b/app/lib/utils/audio/wav_bytes.dart @@ -181,12 +181,15 @@ class WavBytesUtil { debugPrint('createWavFile $filename'); List> framesCopy; if (removeLastNSeconds > 0) { + debugPrint(' in this branch'); removeFramesRange(fromSecond: (frames.length ~/ 100) - removeLastNSeconds, toSecond: frames.length ~/ 100); framesCopy = List>.from(frames); // after trimming, copy the frames } else { + debugPrint(' in other branch'); framesCopy = List>.from(frames); // copy the frames before clearing all clearAudioBytes(); } + debugPrint('about to write to other codec'); File file = await createWavByCodec(framesCopy, filename: filename); return Tuple2(file, framesCopy); } @@ -356,3 +359,95 @@ class ImageBytesUtil { return null; } } + +class StorageBytesUtil extends WavBytesUtil { + + StorageBytesUtil() : super(); +// @override + int count = 0; + List pending = []; + List currentStorageList = []; + int currentStorageCount = 0; + + void storeFrameStoragePacket(value) { + if (value.length == 1) { + debugPrint('stop command'); + return; + } + if (value.length < 100) { + debugPrint('packet too small'); + return; + } + rawPackets.add(value); + int index = value[0] + (value[1] << 8); + int internal = value[2]; + int amount = value[3]; + List content = value.sublist(4,4+amount); + count = count +1; + // debugPrint('current count: $count'); + + // Start of a new frame + if (lastPacketIndex == -1 && internal == 0) { + lastPacketIndex = index; + lastFrameId = internal; + pending = content; + // debugPrint('discsrd'); + return; + } + + if (lastPacketIndex == -1) return; + + // Lost frame - reset state + if (index != lastPacketIndex + 1 || (internal != 0 && internal != lastFrameId + 1)) { + // debugPrint('Lost frame'); + lastPacketIndex = -1; + pending = []; + lost += 1; + return; + } + + // Start of a new frame + if (internal == 0) { + frames.add(pending); // Save frame + pending = content; // Start new frame + lastFrameId = internal; // Update internal frame id + lastPacketIndex = index; // Update packet id + // debugPrint('Frames received: ${frames.length} && Lost: $lost'); + // debugPrint('new frame'); + return; + } + + // Continue frame + pending.addAll(content); + lastFrameId = internal; // Update internal frame id + lastPacketIndex = index; // Update packet id + // debugPrint('reached end'); + } + +@override +Future>>> createWavFile({String? filename, int removeLastNSeconds = 0}) async { + debugPrint('createWavFile $filename'); + List> framesCopy; + if (removeLastNSeconds > 0) { + removeFramesRange(fromSecond: (frames.length ~/ 100) - removeLastNSeconds, toSecond: frames.length ~/ 100); + framesCopy = List>.from(frames); // after trimming, copy the frames + } else { + framesCopy = List>.from(frames); // copy the frames before clearing all + clearAudioBytes(); + } + File file = await createWavByCodec(framesCopy, filename: filename); + return Tuple2(file, framesCopy); +} +@override +Future createWavByCodec(List> frames, {String? filename}) async { + Uint8List wavBytes; + + List decodedSamples = []; + for (var frame in frames) { + decodedSamples.addAll(opusDecoder.decode(input: Uint8List.fromList(frame))); + } + wavBytes = getUInt8ListBytes(decodedSamples, 16000); + return createWav(wavBytes, filename: filename); +} + +} \ No newline at end of file diff --git a/app/lib/utils/ble/communication.dart b/app/lib/utils/ble/communication.dart index 62cb89977..4b1b2aa7c 100644 --- a/app/lib/utils/ble/communication.dart +++ b/app/lib/utils/ble/communication.dart @@ -22,7 +22,7 @@ Future getDevice(String deviceId) async { return null; } } - + //where the codecs get initialized? final deviceType = deviceTypeMap[deviceId]; if (deviceType == DeviceType.friend) { deviceMap[deviceId] = FriendDevice(deviceId); @@ -54,9 +54,26 @@ Future getBleAudioBytesListener( onAudioBytesReceived: onAudioBytesReceived) ?? Future.value(null); + Future getAudioCodec(String deviceId) async => (await getDevice(deviceId))?.getAudioCodec() ?? Future.value(BleAudioCodec.pcm8); + + +Future getBleStorageBytesListener( + String deviceId, { + required void Function(List) onStorageBytesReceived, +}) async => + (await getDevice(deviceId))?.getBleStorageBytesListener( + onStorageBytesReceived: onStorageBytesReceived) ?? + Future.value(null); + +Future writeToStorage(String deviceId) async => + (await getDevice(deviceId))?.writeToStorage() ?? Future.value(false); + +Future> getStorageList(String deviceId) async => + (await getDevice(deviceId))?.getStorageList() ?? Future.value(null); + Future cameraStartPhotoController(String deviceId) async => (await getDevice(deviceId))?.cameraStartPhotoController() ?? Future.value(null); diff --git a/app/lib/utils/ble/device_base.dart b/app/lib/utils/ble/device_base.dart index 19e3ae4b7..83cfd2455 100644 --- a/app/lib/utils/ble/device_base.dart +++ b/app/lib/utils/ble/device_base.dart @@ -59,6 +59,42 @@ abstract class DeviceBase { } Future performGetAudioCodec(); +//storage here + + Future> getStorageList() async { + if (await isConnected()) { + return await performGetStorageList(); + } + _showDeviceDisconnectedNotification(); + return Future.value([]); + } + + Future> performGetStorageList(); + + Future performWriteToStorage(); + + Future writeToStorage() async { + + if (await isConnected()) { + return await performWriteToStorage(); + } + _showDeviceDisconnectedNotification(); + return Future.value(false); + } + + Future getBleStorageBytesListener({ + required void Function(List) onStorageBytesReceived, + }) async { + if (await isConnected()) { + return await performGetBleStorageBytesListener(onStorageBytesReceived: onStorageBytesReceived); + } + _showDeviceDisconnectedNotification(); + return null; + } + + Future performGetBleStorageBytesListener({ + required void Function(List) onStorageBytesReceived, + }); Future cameraStartPhotoController() async { if (await isConnected()) { diff --git a/app/lib/utils/ble/frame_communication.dart b/app/lib/utils/ble/frame_communication.dart index 593256c74..b782c8215 100644 --- a/app/lib/utils/ble/frame_communication.dart +++ b/app/lib/utils/ble/frame_communication.dart @@ -466,4 +466,25 @@ class FrameDevice extends DeviceBase { // not yet implemented return null; } + @override + Future> performGetStorageList() { + + return Future.value([]); + } + + // @override + // Future> performGetStorageList() { + + // return []; + // } +@override +Future performGetBleStorageBytesListener({ + required void Function(List) onStorageBytesReceived, +}) { + return Future.value(null); +} +@override +Future performWriteToStorage() { + return Future.value(false); +} } diff --git a/app/lib/utils/ble/friend_communication.dart b/app/lib/utils/ble/friend_communication.dart index 33652a0de..0089dcdc8 100644 --- a/app/lib/utils/ble/friend_communication.dart +++ b/app/lib/utils/ble/friend_communication.dart @@ -12,6 +12,9 @@ import 'package:friend_private/utils/ble/device_base.dart'; import 'package:friend_private/utils/ble/errors.dart'; import 'package:friend_private/utils/ble/gatt_utils.dart'; +import 'package:friend_private/backend/schema/bt_device.dart'; +import 'package:friend_private/services/notification_service.dart'; + class FriendDevice extends DeviceBase { @override final String deviceId; @@ -170,6 +173,116 @@ class FriendDevice extends DeviceBase { // debugPrint('Codec is $codec'); return codec; } + Future> getStorageList() async { + if (await isConnected()) { + debugPrint('storage list called'); + return await performGetStorageList(); + } + // _showDeviceDisconnectedNotification(); + debugPrint('storage list error'); + return Future.value([]); + } + + // @override + Future> performGetStorageList() async { + debugPrint(' perform storage list called'); + final storageService = await getServiceByUuid(deviceId, storageDataStreamServiceUuid); + if (storageService == null) { + logServiceNotFoundError('Friend', deviceId); + return Future.value([]); + } + + var storageListCharacteristic = getCharacteristicByUuid(storageService, storageReadControlCharacteristicUuid); + if (storageListCharacteristic == null) { + logCharacteristicNotFoundError('Storage List', deviceId); + return Future.value([]); + } + var storageValue = await storageListCharacteristic.read(); + List storageLengths = []; + if (storageValue.isNotEmpty) { + //parse the list + int totalEntries = (storageValue.length / 4).toInt(); + debugPrint('Storage list: ${totalEntries} items'); + + for (int i = 0; i < totalEntries; i++) { + int baseIndex = i * 4; + var result = ((storageValue[baseIndex] | + (storageValue[baseIndex + 1] << 8) | + (storageValue[baseIndex + 2] << 16) | + (storageValue[baseIndex + 3] << 24)) & + 0xFFFFFFFF as int) + .toSigned(32); + storageLengths.add(result); + } + } + debugPrint('storage list finished'); + debugPrint('Storage lengths: ${storageLengths.length} items: ${storageLengths.join(', ')}'); + return storageLengths; + } + +@override +Future performGetBleStorageBytesListener({ + required void Function(List) onStorageBytesReceived, +}) async { + final storageService = await getServiceByUuid(deviceId, storageDataStreamServiceUuid); + if (storageService == null) { + logServiceNotFoundError('Storage Write', deviceId); + return null; + } + + var storageDataStreamCharacteristic = getCharacteristicByUuid(storageService, storageDataStreamCharacteristicUuid); + if (storageDataStreamCharacteristic == null) { + logCharacteristicNotFoundError('Storage data stream', deviceId); + return null; + } + + try { + await storageDataStreamCharacteristic.setNotifyValue(true); // device could be disconnected here. + } catch (e, stackTrace) { + logSubscribeError('Storage data stream', deviceId, e, stackTrace); + return null; + } + + debugPrint('Subscribed to StorageBytes stream from Friend Device'); + var listener = storageDataStreamCharacteristic.lastValueStream.listen((value) { + if (value.isNotEmpty) onStorageBytesReceived(value); + }); + + final device = BluetoothDevice.fromId(deviceId); + device.cancelWhenDisconnected(listener); + + // await storageDataStreamCharacteristic.write([0x00,0x01]); + + // This will cause a crash in OpenGlass devices + // due to a race with discoverServices() that triggers + // a bug in the device firmware. + if (Platform.isAndroid) await device.requestMtu(512); + + return listener; + } + + +Future performWriteToStorage( + +) async { + final storageService = await getServiceByUuid(deviceId, storageDataStreamServiceUuid); + if (storageService == null) { + logServiceNotFoundError('Storage Write', deviceId); + return false; + } + + var storageDataStreamCharacteristic = getCharacteristicByUuid(storageService, storageDataStreamCharacteristicUuid); + if (storageDataStreamCharacteristic == null) { + logCharacteristicNotFoundError('Storage data stream', deviceId); + return false; + } + debugPrint('About to write to storage bytes'); + + await storageDataStreamCharacteristic.write([0x00,0x01]); + + return true; + } + // Future> performGetStorageList(); @override Future performCameraStartPhotoController() async { diff --git a/app/lib/utils/ble/gatt_utils.dart b/app/lib/utils/ble/gatt_utils.dart index 72132c760..ce490c4c7 100644 --- a/app/lib/utils/ble/gatt_utils.dart +++ b/app/lib/utils/ble/gatt_utils.dart @@ -15,6 +15,10 @@ const String buttonTriggerCharacteristicUuid = '23ba7925-0000-1000-7450-346eac49 const String imageDataStreamCharacteristicUuid = '19b10005-e8f2-537e-4f6c-d104768a1214'; const String imageCaptureControlCharacteristicUuid = '19b10006-e8f2-537e-4f6c-d104768a1214'; +const String storageDataStreamServiceUuid = '30295780-4301-eabd-2904-2849adfeae43'; +const String storageDataStreamCharacteristicUuid = '30295781-4301-eabd-2904-2849adfeae43'; +const String storageReadControlCharacteristicUuid = '30295782-4301-eabd-2904-2849adfeae43'; + const String accelDataStreamServiceUuid = '32403790-0000-1000-7450-bf445e5829a2'; const String accelDataStreamCharacteristicUuid = '32403791-0000-1000-7450-bf445e5829a2';