From c86fcf54b2941cc0d6254d1a4373544d167b1e37 Mon Sep 17 00:00:00 2001 From: Shankari Date: Sat, 13 Apr 2024 17:56:29 -0700 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=97=83=EF=B8=8F=20Plumb=20through=20t?= =?UTF-8?q?he=20BLE=20objects=20to=20the=20server?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These are the server side changes related to https://github.com/e-mission/e-mission-docs/issues/1062 The changes are fairly straightforward, and consistent with https://github.com/e-mission/e-mission-docs/blob/2665b39e1335ea04896b6944a4a065e7887b6cdc/docs/dev/back/adding_a_new_data_type.md?plain=1#L4 Concretely: - we add a new `bluetoothble` wrapper - we add references to it to `entry.py` and `builtin_timeseries.py` - we add formatters for both android and iOS A new wrinkle this time is that we are modifying the FSM, so there are also new transitions. Those needed to be added to the enums in the transition wrapper, and to the maps in the formatters so that the enums could be created properly. Bonus fix: check for the `None` transition properly on android and iOS to avoid spurious errors ``` >>> broken_transition_example = {'_id': ObjectId('661b129fc271a44bb612b464'), 'metadata': {'time_zone': 'America/Los_Angeles', 'plugin': 'none', 'write_ts': 1713050268.574551, 'platform': 'ios', 'read_ts': 0, 'key': 'statemachine/transition', 'type': 'message'}, 'user_id': UUID('f1aaae55-fc42-4527-bf7f-33f84d7c8c2f'), 'data': {'currState': 'STATE_ONGOING_TRIP', 'transition': None, 'ts': 1713050268.574418}} >>> broken_transition_example_entry = ad.AttrDict(broken_transition_example) >>> enufit.format(broken_transition_example_entry) Traceback (most recent call last): File "", line 1, in File "/Users/kshankar/Desktop/data/e-mission/gis_branch_tests/emission/net/usercache/formatters/ios/transition.py", line 64, in format data.transition = transition_map[entry.data.transition].value KeyError: None ----- fixed code ------ >>> importlib.reload(enufit) >>> enufit.format(broken_transition_example_entry) AttrDict({'_id': ObjectId('661b129fc271a44bb612b464'), 'user_id': UUID('f1aaae55-fc42-4527-bf7f-33f84d7c8c2f'), 'metadata': AttrDict({'time_zone': 'America/Los_Angeles', 'plugin': 'none', 'write_ts': 1713050268.574551, 'platform': 'ios', 'read_ts': 0, 'key': 'statemachine/transition', 'type': 'message', 'write_local_dt': LocalDate({'year': 2024, 'month': 4, 'day': 13, 'hour': 16, 'minute': 17, 'second': 48, 'weekday': 5, 'timezone': 'America/Los_Angeles'}), 'write_fmt_time': '2024-04-13T16:17:48.574551-07:00'}), 'data': AttrDict({'curr_state': 2, 'transition': None, 'ts': 1713050268.574551, 'local_dt': AttrDict({'year': 2024, 'month': 4, 'day': 13, 'hour': 16, 'minute': 17, 'second': 48, 'weekday': 5, 'timezone': 'America/Los_Angeles'}), 'fmt_time': '2024-04-13T16:17:48.574551-07:00'})}) ``` Testing done: Used the corresponding changes in https://github.com/e-mission/e-mission-phone/pull/1144 to simulate BLE as follows: - Region exit - A few range updates until the `ble_beacon_found` transition was generated - Turned on location mocking from the android and iOS simulators, and manually generated the start trip transition on android - Clicked "range update" at random times during the simulated trip - BLE beacon lost from the UI - Turn off location mocking - Force end trip transition Testing Results: Android: ``` START 2024-04-13 17:36:02.096342 POST /usercache/put END 2024-04-13 17:36:02.313529 POST /usercache/put ebc13f1b-671b-4094-bce6-fed342da7e9c 0.2171182632446289 START 2024-04-13 17:36:02.583812 POST /usercache/get END 2024-04-13 17:36:02.591868 POST /usercache/get ebc13f1b-671b-4094-bce6-fed342da7e9c 0.007989168167114258 ``` ``` >>> edb.get_usercache_db().count_documents({"metadata.key": "background/bluetooth_ble"}) 57 >>> edb.get_timeseries_db().count_documents({"metadata.key": "background/bluetooth_ble"}) 0 ``` ``` 2024-04-13 17:37:57,635:DEBUG:140704655566784:write_ts = 1713054811.255 2024-04-13 17:37:57,635:DEBUG:140704655566784:module_name = emission.net.userca che.formatters.android.bluetooth_ble 2024-04-13 17:37:57,636:DEBUG:140704655566784:write_ts = 1713054811.294 2024-04-13 17:37:57,636:DEBUG:140704655566784:module_name = emission.net.userca che.formatters.android.bluetooth_ble 2024-04-13 17:37:57,636:DEBUG:140704655566784:write_ts = 1713054811.316 2024-04-13 17:37:57,636:DEBUG:140704655566784:module_name = emission.net.userca che.formatters.android.bluetooth_ble 2024-04-13 17:37:57,637:DEBUG:140704655566784:write_ts = 1713054811.339 2024-04-13 17:37:57,637:DEBUG:140704655566784:module_name = emission.net.userca che.formatters.android.bluetooth_ble 2024-04-13 17:37:57,637:DEBUG:140704655566784:write_ts = 1713054811.369 ``` ``` >>> edb.get_usercache_db().count_documents({"metadata.key": "background/bluetooth_ble"}) 0 >>> edb.get_timeseries_db().count_documents({"metadata.key": "background/bluetooth_ble"}) 57 ``` iOS ``` START 2024-04-13 16:17:50.707151 POST /usercache/put START 2024-04-13 16:17:50.737703 POST /usercache/get END 2024-04-13 16:17:50.763880 POST /usercache/get f1aaae55-fc42-4527-bf7f-33f84d7c8c2f 0.026064157485961914 END 2024-04-13 16:17:51.340867 POST /usercache/put f1aaae55-fc42-4527-bf7f-33f84d7c8c2f 0.6329052448272705 ``` ``` >>> edb.get_usercache_db().count_documents({"metadata.key": "background/bluetooth_ble"}) 74 ``` ``` DEBUG:root:module_name = emission.net.usercache.formatters.ios.bluetooth_ble DEBUG:root:write_ts = 1713050204.075974 DEBUG:root:module_name = emission.net.usercache.formatters.ios.bluetooth_ble DEBUG:root:write_ts = 1713050204.077563 DEBUG:root:module_name = emission.net.usercache.formatters.ios.bluetooth_ble DEBUG:root:write_ts = 1713050204.078974 DEBUG:root:module_name = emission.net.usercache.formatters.ios.bluetooth_ble DEBUG:root:write_ts = 1713050207.32417 DEBUG:root:module_name = emission.net.usercache.formatters.ios.bluetooth_ble DEBUG:root:write_ts = 1713050207.326033 ``` ``` >>> edb.get_usercache_db().count_documents({"metadata.key": "background/bluetooth_ble"}) 0 >>> edb.get_timeseries_db().count_documents({"metadata.key": "background/bluetooth_ble"}) 74 >>> edb.get_timeseries_error_db().count_documents({"metadata.key": "background/bluetooth_ble"}) 0 ``` --- emission/core/wrapper/bluetoothble.py | 29 +++++++++++++++++++ emission/core/wrapper/entry.py | 2 ++ emission/core/wrapper/transition.py | 2 ++ .../formatters/android/bluetooth_ble.py | 26 +++++++++++++++++ .../formatters/android/transition.py | 8 ++++- .../usercache/formatters/ios/bluetooth_ble.py | 24 +++++++++++++++ .../usercache/formatters/ios/transition.py | 3 +- .../storage/timeseries/builtin_timeseries.py | 1 + 8 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 emission/core/wrapper/bluetoothble.py create mode 100644 emission/net/usercache/formatters/android/bluetooth_ble.py create mode 100644 emission/net/usercache/formatters/ios/bluetooth_ble.py diff --git a/emission/core/wrapper/bluetoothble.py b/emission/core/wrapper/bluetoothble.py new file mode 100644 index 000000000..a814e4303 --- /dev/null +++ b/emission/core/wrapper/bluetoothble.py @@ -0,0 +1,29 @@ +import logging +import enum as enum +import emission.core.wrapper.wrapperbase as ecwb + +class BLEEventTypes(enum.Enum): + REGION_ENTER = 0 + REGION_EXIT = 1 + RANGE_UPDATE = 2 + +class Bluetoothble(ecwb.WrapperBase): + props = {"eventType": ecwb.WrapperBase.Access.RO, # the type of event + "uuid": ecwb.WrapperBase.Access.RO, # UUID of the beacon. Will be a constant for beacons used by e-mission, consistent with https://github.com/e-mission/e-mission-docs/issues/1062#issuecomment-2026359038 + "major": ecwb.WrapperBase.Access.RO, # major value (matches deployment) + "minor": ecwb.WrapperBase.Access.RO, # minor value (matches the actual vehicle) + "ts": ecwb.WrapperBase.Access.RO, # timestamp (in seconds) + "proximity": ecwb.WrapperBase.Access.RO, # how close the beacon is (used as the second step in the process, https://github.com/e-mission/e-mission-docs/issues/1062#issuecomment-2026359038 + "local_dt": ecwb.WrapperBase.Access.RO, # searchable datetime in local time + "fmt_time": ecwb.WrapperBase.Access.RO, # formatted time + "accuracy": ecwb.WrapperBase.Access.RO, # only available for range updats + "rssi": ecwb.WrapperBase.Access.RO, # signal strength, only available for range updates + } + + enums = {"eventType": BLEEventTypes} + geojson = [] + nullable = ["major", "minor", "proximity", "accuracy", "rssi"] + local_dates = ['local_dt'] + + def _populateDependencies(self): + pass diff --git a/emission/core/wrapper/entry.py b/emission/core/wrapper/entry.py index b4d8520f7..c6f7b7dd3 100644 --- a/emission/core/wrapper/entry.py +++ b/emission/core/wrapper/entry.py @@ -42,6 +42,8 @@ def _getData2Wrapper(): "background/motion_activity": "motionactivity", # battery readings, to determine power drain empirically "background/battery": "battery", + # BLE events, including enter, exiting and ranging beacons + "background/bluetooth_ble": "bluetoothble", # transition events for the tracking finite state machine on the phone "statemachine/transition": "transition", # phone sensing configuration (e.g. sensing frequency, geofencing,...) diff --git a/emission/core/wrapper/transition.py b/emission/core/wrapper/transition.py index 9c6247272..634f7ac42 100644 --- a/emission/core/wrapper/transition.py +++ b/emission/core/wrapper/transition.py @@ -39,6 +39,8 @@ class TransitionType(enum.Enum): DATA_PUSHED = 16 # joint transition again START_TRACKING = 17 + BLE_BEACON_FOUND = 18 + BLE_BEACON_LOST = 19 class Transition(ecwb.WrapperBase): props = {"curr_state": ecwb.WrapperBase.Access.RO, diff --git a/emission/net/usercache/formatters/android/bluetooth_ble.py b/emission/net/usercache/formatters/android/bluetooth_ble.py new file mode 100644 index 000000000..bf43951eb --- /dev/null +++ b/emission/net/usercache/formatters/android/bluetooth_ble.py @@ -0,0 +1,26 @@ +import logging + +import emission.core.wrapper.bluetoothble as ecwb +import emission.net.usercache.formatters.common as fc +import attrdict as ad + +def format(entry): + formatted_entry = ad.AttrDict() + formatted_entry["_id"] = entry["_id"] + formatted_entry.user_id = entry.user_id + + metadata = entry.metadata + if "time_zone" not in metadata: + metadata.time_zone = "America/Los_Angeles" + fc.expand_metadata_times(metadata) + formatted_entry.metadata = metadata + + #logging.info('*** Motion Data write_ts: %d' % metadata.write_ts) + + data = entry.data + data.local_dt = formatted_entry.metadata.write_local_dt + data.fmt_time = formatted_entry.metadata.write_fmt_time + data.eventType = ecwb.BLEEventTypes[entry.data.eventType].value + formatted_entry.data = data + + return formatted_entry diff --git a/emission/net/usercache/formatters/android/transition.py b/emission/net/usercache/formatters/android/transition.py index 8352c6013..8dd689882 100644 --- a/emission/net/usercache/formatters/android/transition.py +++ b/emission/net/usercache/formatters/android/transition.py @@ -26,6 +26,8 @@ "local.transition.stopped_moving": et.TransitionType.STOPPED_MOVING, "local.transition.stop_tracking": et.TransitionType.STOP_TRACKING, "local.transition.start_tracking": et.TransitionType.START_TRACKING, + "local.transition.ble_beacon_found": et.TransitionType.BLE_BEACON_FOUND, + "local.transition.ble_beacon_lost": et.TransitionType.BLE_BEACON_LOST, "local.transition.tracking_error": et.TransitionType.TRACKING_ERROR } @@ -44,7 +46,11 @@ def format(entry): data = ad.AttrDict() data.curr_state = state_map[entry.data.currState].value logging.debug("Mapped %s -> %s" % (entry.data.currState, data.curr_state)) - data.transition = transition_map[entry.data.transition].value + if entry.data.transition is not None: + data.transition = transition_map[entry.data.transition].value + else: + data.transition = None + if "ts" not in data: data.ts = formatted_entry.metadata.write_ts logging.debug("No existing timestamp, copyied from metadata%s" % data.ts) diff --git a/emission/net/usercache/formatters/ios/bluetooth_ble.py b/emission/net/usercache/formatters/ios/bluetooth_ble.py new file mode 100644 index 000000000..a0883d424 --- /dev/null +++ b/emission/net/usercache/formatters/ios/bluetooth_ble.py @@ -0,0 +1,24 @@ +import logging + +import emission.core.wrapper.bluetoothble as ecwb +import emission.net.usercache.formatters.common as fc +import attrdict as ad +import pandas as pd +import numpy as np + +def format(entry): + formatted_entry = ad.AttrDict() + formatted_entry["_id"] = entry["_id"] + formatted_entry.user_id = entry.user_id + + metadata = entry.metadata + fc.expand_metadata_times(metadata) + formatted_entry.metadata = metadata + + data = entry.data + data.local_dt = formatted_entry.metadata.write_local_dt + data.fmt_time = formatted_entry.metadata.write_fmt_time + data.eventType = ecwb.BLEEventTypes[entry.data.eventType].value + formatted_entry.data = data + + return formatted_entry diff --git a/emission/net/usercache/formatters/ios/transition.py b/emission/net/usercache/formatters/ios/transition.py index 5c4f3cacd..04c19dcb1 100644 --- a/emission/net/usercache/formatters/ios/transition.py +++ b/emission/net/usercache/formatters/ios/transition.py @@ -34,6 +34,8 @@ "T_TRACKING_STOPPED": et.TransitionType.TRACKING_STOPPED, "T_VISIT_STARTED": et.TransitionType.VISIT_STARTED, "T_VISIT_ENDED": et.TransitionType.VISIT_ENDED, + "T_BLE_BEACON_FOUND": et.TransitionType.BLE_BEACON_FOUND, + "T_BLE_BEACON_LOST": et.TransitionType.BLE_BEACON_LOST, "T_NOP": et.TransitionType.NOP, "T_START_TRACKING": et.TransitionType.START_TRACKING } @@ -59,7 +61,6 @@ def format(entry): # deal with collapsing later # data.transition_raw = entry.data.transition - data.transition = transition_map[entry.data.transition].value if entry.data.transition is not None: data.transition = transition_map[entry.data.transition].value else: diff --git a/emission/storage/timeseries/builtin_timeseries.py b/emission/storage/timeseries/builtin_timeseries.py index 8204b4e29..15ba9f0ec 100644 --- a/emission/storage/timeseries/builtin_timeseries.py +++ b/emission/storage/timeseries/builtin_timeseries.py @@ -45,6 +45,7 @@ def __init__(self, user_id): "background/filtered_location": self.timeseries_db, "background/motion_activity": self.timeseries_db, "background/battery": self.timeseries_db, + "background/bluetooth_ble": self.timeseries_db, "statemachine/transition": self.timeseries_db, "config/sensor_config": self.timeseries_db, "config/sync_config": self.timeseries_db, From de11585a9b0fd53143fe7e9c66e0a1a48b63f3ef Mon Sep 17 00:00:00 2001 From: Shankari Date: Sat, 13 Apr 2024 19:12:17 -0700 Subject: [PATCH 2/3] =?UTF-8?q?=E2=9C=85=20Change=20the=20transition=20for?= =?UTF-8?q?=20the=20negative=20test=20to=20one=20that=20actually=20fails?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Our negative test in this test suite used `transition': none` as an invalid transition. However, in https://github.com/e-mission/e-mission-server/pull/963/commits/c86fcf54b2941cc0d6254d1a4373544d167b1e37 we started handling `None` transitions properly So we need to change it to a transition that is actually invalid. Testing done: Before this, `testMoveWhenEmpty` failed. Now, `testMoveWhenEmpty` passes --- emission/tests/netTests/TestBuiltinUserCacheHandlerInput.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emission/tests/netTests/TestBuiltinUserCacheHandlerInput.py b/emission/tests/netTests/TestBuiltinUserCacheHandlerInput.py index b7b2e8b97..3d024be06 100644 --- a/emission/tests/netTests/TestBuiltinUserCacheHandlerInput.py +++ b/emission/tests/netTests/TestBuiltinUserCacheHandlerInput.py @@ -138,7 +138,7 @@ def testMoveWhenEmpty(self): edb.get_usercache_db().insert_one({ 'user_id': self.testUserUUID1, '_id': boi.ObjectId('572d3621d282b8f30def7e85'), - 'data': {u'transition': None, + 'data': {u'transition': "MY_FAKE_TRANSITION", 'currState': u'STATE_ONGOING_TRIP'}, 'metadata': {'plugin': 'none', 'write_ts': self.curr_ts - 25, From ae7996c796f94160c6f90c80531670a86945f2f1 Mon Sep 17 00:00:00 2001 From: Shankari Date: Sat, 13 Apr 2024 19:18:52 -0700 Subject: [PATCH 3/3] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20=20Upgrade=20the=20pym?= =?UTF-8?q?ongo=20calls=20to=20the=20latest=20version?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `save` -> `replace_one` with upsert `remove` -> `delete_one` --- bin/debug/fix_usercache_processing.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/debug/fix_usercache_processing.py b/bin/debug/fix_usercache_processing.py index 7187177e6..0158705b3 100644 --- a/bin/debug/fix_usercache_processing.py +++ b/bin/debug/fix_usercache_processing.py @@ -27,6 +27,7 @@ def fix_usercache_errors(): copy_to_usercache() + print(">" * 30) move_to_long_term() def copy_to_usercache(): @@ -37,8 +38,8 @@ def copy_to_usercache(): logging.info("Found %d errors in this round" % edb.get_timeseries_error_db().estimated_document_count()) for error in error_it: logging.debug("Copying entry %s" % error["metadata"]) - save_result = uc.save(error) - remove_result = te.remove(error["_id"]) + save_result = uc.replace_one({"_id": error['_id']}, error, upsert=True) + remove_result = te.delete_one({"_id": error["_id"]}) logging.debug("save_result = %s, remove_result = %s" % (save_result, remove_result)) logging.info("step copy_to_usercache DONE")