🗃️ Plumb through the BLE objects to the server

These are the server side changes related to e-mission/e-mission-docs#1062 The changes are fairly straightforward, and consistent with https://github.com/e-mission/e-mission-docs/blob/2665b39e1335ea04896b6944a4a065e7887b6cdc/docs/dev/back/adding_a_new_data_type.md?plain=1#L4 Concretely: - we add a new `bluetoothble` wrapper - we add references to it to `entry.py` and `builtin_timeseries.py` - we add formatters for both android and iOS A new wrinkle this time is that we are modifying the FSM, so there are also new transitions. Those needed to be added to the enums in the transition wrapper, and to the maps in the formatters so that the enums could be created properly. Bonus fix: check for the `None` transition properly on android and iOS to avoid spurious errors ``` >>> broken_transition_example = {'_id': ObjectId('661b129fc271a44bb612b464'), 'metadata': {'time_zone': 'America/Los_Angeles', 'plugin': 'none', 'write_ts': 1713050268.574551, 'platform': 'ios', 'read_ts': 0, 'key': 'statemachine/transition', 'type': 'message'}, 'user_id': UUID('f1aaae55-fc42-4527-bf7f-33f84d7c8c2f'), 'data': {'currState': 'STATE_ONGOING_TRIP', 'transition': None, 'ts': 1713050268.574418}} >>> broken_transition_example_entry = ad.AttrDict(broken_transition_example) >>> enufit.format(broken_transition_example_entry) Traceback (most recent call last): File "<stdin>", line 1, in <module> File "/Users/kshankar/Desktop/data/e-mission/gis_branch_tests/emission/net/usercache/formatters/ios/transition.py", line 64, in format data.transition = transition_map[entry.data.transition].value KeyError: None ----- fixed code ------ >>> importlib.reload(enufit) <module 'emission.net.usercache.formatters.ios.transition' from '/Users/kshankar/Desktop/data/e-mission/gis_branch_tests/emission/net/usercache/formatters/ios/transition.py'> >>> enufit.format(broken_transition_example_entry) AttrDict({'_id': ObjectId('661b129fc271a44bb612b464'), 'user_id': UUID('f1aaae55-fc42-4527-bf7f-33f84d7c8c2f'), 'metadata': AttrDict({'time_zone': 'America/Los_Angeles', 'plugin': 'none', 'write_ts': 1713050268.574551, 'platform': 'ios', 'read_ts': 0, 'key': 'statemachine/transition', 'type': 'message', 'write_local_dt': LocalDate({'year': 2024, 'month': 4, 'day': 13, 'hour': 16, 'minute': 17, 'second': 48, 'weekday': 5, 'timezone': 'America/Los_Angeles'}), 'write_fmt_time': '2024-04-13T16:17:48.574551-07:00'}), 'data': AttrDict({'curr_state': 2, 'transition': None, 'ts': 1713050268.574551, 'local_dt': AttrDict({'year': 2024, 'month': 4, 'day': 13, 'hour': 16, 'minute': 17, 'second': 48, 'weekday': 5, 'timezone': 'America/Los_Angeles'}), 'fmt_time': '2024-04-13T16:17:48.574551-07:00'})}) ``` Testing done: Used the corresponding changes in e-mission/e-mission-phone#1144 to simulate BLE as follows: - Region exit - A few range updates until the `ble_beacon_found` transition was generated - Turned on location mocking from the android and iOS simulators, and manually generated the start trip transition on android - Clicked "range update" at random times during the simulated trip - BLE beacon lost from the UI - Turn off location mocking - Force end trip transition Testing Results: Android: ``` START 2024-04-13 17:36:02.096342 POST /usercache/put END 2024-04-13 17:36:02.313529 POST /usercache/put ebc13f1b-671b-4094-bce6-fed342da7e9c 0.2171182632446289 START 2024-04-13 17:36:02.583812 POST /usercache/get END 2024-04-13 17:36:02.591868 POST /usercache/get ebc13f1b-671b-4094-bce6-fed342da7e9c 0.007989168167114258 ``` ``` >>> edb.get_usercache_db().count_documents({"metadata.key": "background/bluetooth_ble"}) 57 >>> edb.get_timeseries_db().count_documents({"metadata.key": "background/bluetooth_ble"}) 0 ``` ``` 2024-04-13 17:37:57,635:DEBUG:140704655566784:write_ts = 1713054811.255 2024-04-13 17:37:57,635:DEBUG:140704655566784:module_name = emission.net.userca che.formatters.android.bluetooth_ble 2024-04-13 17:37:57,636:DEBUG:140704655566784:write_ts = 1713054811.294 2024-04-13 17:37:57,636:DEBUG:140704655566784:module_name = emission.net.userca che.formatters.android.bluetooth_ble 2024-04-13 17:37:57,636:DEBUG:140704655566784:write_ts = 1713054811.316 2024-04-13 17:37:57,636:DEBUG:140704655566784:module_name = emission.net.userca che.formatters.android.bluetooth_ble 2024-04-13 17:37:57,637:DEBUG:140704655566784:write_ts = 1713054811.339 2024-04-13 17:37:57,637:DEBUG:140704655566784:module_name = emission.net.userca che.formatters.android.bluetooth_ble 2024-04-13 17:37:57,637:DEBUG:140704655566784:write_ts = 1713054811.369 ``` ``` >>> edb.get_usercache_db().count_documents({"metadata.key": "background/bluetooth_ble"}) 0 >>> edb.get_timeseries_db().count_documents({"metadata.key": "background/bluetooth_ble"}) 57 ``` iOS ``` START 2024-04-13 16:17:50.707151 POST /usercache/put START 2024-04-13 16:17:50.737703 POST /usercache/get END 2024-04-13 16:17:50.763880 POST /usercache/get f1aaae55-fc42-4527-bf7f-33f84d7c8c2f 0.026064157485961914 END 2024-04-13 16:17:51.340867 POST /usercache/put f1aaae55-fc42-4527-bf7f-33f84d7c8c2f 0.6329052448272705 ``` ``` >>> edb.get_usercache_db().count_documents({"metadata.key": "background/bluetooth_ble"}) 74 ``` ``` DEBUG:root:module_name = emission.net.usercache.formatters.ios.bluetooth_ble DEBUG:root:write_ts = 1713050204.075974 DEBUG:root:module_name = emission.net.usercache.formatters.ios.bluetooth_ble DEBUG:root:write_ts = 1713050204.077563 DEBUG:root:module_name = emission.net.usercache.formatters.ios.bluetooth_ble DEBUG:root:write_ts = 1713050204.078974 DEBUG:root:module_name = emission.net.usercache.formatters.ios.bluetooth_ble DEBUG:root:write_ts = 1713050207.32417 DEBUG:root:module_name = emission.net.usercache.formatters.ios.bluetooth_ble DEBUG:root:write_ts = 1713050207.326033 ``` ``` >>> edb.get_usercache_db().count_documents({"metadata.key": "background/bluetooth_ble"}) 0 >>> edb.get_timeseries_db().count_documents({"metadata.key": "background/bluetooth_ble"}) 74 >>> edb.get_timeseries_error_db().count_documents({"metadata.key": "background/bluetooth_ble"}) 0 ```
e-mission · Apr 14, 2024 · c86fcf5 · c86fcf5
1 parent 54659fb
commit c86fcf5
Show file tree

Hide file tree

Showing 8 changed files with 93 additions and 2 deletions.
diff --git a/emission/core/wrapper/bluetoothble.py b/emission/core/wrapper/bluetoothble.py
@@ -0,0 +1,29 @@
+import logging
+import enum as enum
+import emission.core.wrapper.wrapperbase as ecwb
+
+class BLEEventTypes(enum.Enum):
+    REGION_ENTER = 0
+    REGION_EXIT = 1
+    RANGE_UPDATE = 2
+
+class Bluetoothble(ecwb.WrapperBase):
+    props = {"eventType": ecwb.WrapperBase.Access.RO, # the type of event
+             "uuid": ecwb.WrapperBase.Access.RO,  # UUID of the beacon. Will be a constant for beacons used by e-mission, consistent with https://github.com/e-mission/e-mission-docs/issues/1062#issuecomment-2026359038
+             "major": ecwb.WrapperBase.Access.RO, # major value (matches deployment)
+             "minor": ecwb.WrapperBase.Access.RO,       # minor value (matches the actual vehicle)
+             "ts": ecwb.WrapperBase.Access.RO,        # timestamp (in seconds)
+             "proximity": ecwb.WrapperBase.Access.RO,  # how close the beacon is (used as the second step in the process, https://github.com/e-mission/e-mission-docs/issues/1062#issuecomment-2026359038
+             "local_dt": ecwb.WrapperBase.Access.RO, # searchable datetime in local time
+             "fmt_time": ecwb.WrapperBase.Access.RO,  # formatted time
+             "accuracy": ecwb.WrapperBase.Access.RO,  # only available for range updats
+             "rssi": ecwb.WrapperBase.Access.RO, # signal strength, only available for range updates
+    }
+
+    enums = {"eventType": BLEEventTypes}
+    geojson = []
+    nullable = ["major", "minor", "proximity", "accuracy", "rssi"]
+    local_dates = ['local_dt']
+
+    def _populateDependencies(self):
+        pass
diff --git a/emission/core/wrapper/entry.py b/emission/core/wrapper/entry.py
@@ -42,6 +42,8 @@ def _getData2Wrapper():
             "background/motion_activity": "motionactivity",
             # battery readings, to determine power drain empirically
             "background/battery": "battery",
+            # BLE events, including enter, exiting and ranging beacons
+            "background/bluetooth_ble": "bluetoothble",
             # transition events for the tracking finite state machine on the phone
             "statemachine/transition": "transition",
             # phone sensing configuration (e.g. sensing frequency, geofencing,...)

diff --git a/emission/core/wrapper/transition.py b/emission/core/wrapper/transition.py
@@ -39,6 +39,8 @@ class TransitionType(enum.Enum):
     DATA_PUSHED = 16
     # joint transition again
     START_TRACKING = 17
+    BLE_BEACON_FOUND = 18
+    BLE_BEACON_LOST = 19
 
 class Transition(ecwb.WrapperBase):
     props = {"curr_state": ecwb.WrapperBase.Access.RO,

diff --git a/emission/net/usercache/formatters/android/bluetooth_ble.py b/emission/net/usercache/formatters/android/bluetooth_ble.py
@@ -0,0 +1,26 @@
+import logging
+
+import emission.core.wrapper.bluetoothble as ecwb
+import emission.net.usercache.formatters.common as fc
+import attrdict as ad
+
+def format(entry):
+    formatted_entry = ad.AttrDict()
+    formatted_entry["_id"] = entry["_id"]
+    formatted_entry.user_id = entry.user_id
+
+    metadata = entry.metadata
+    if "time_zone" not in metadata:
+        metadata.time_zone = "America/Los_Angeles" 
+    fc.expand_metadata_times(metadata)
+    formatted_entry.metadata = metadata
+
+    #logging.info('*** Motion Data write_ts: %d' % metadata.write_ts)
+
+    data = entry.data
+    data.local_dt = formatted_entry.metadata.write_local_dt
+    data.fmt_time = formatted_entry.metadata.write_fmt_time
+    data.eventType = ecwb.BLEEventTypes[entry.data.eventType].value
+    formatted_entry.data = data
+
+    return formatted_entry
diff --git a/emission/net/usercache/formatters/android/transition.py b/emission/net/usercache/formatters/android/transition.py
@@ -26,6 +26,8 @@
     "local.transition.stopped_moving": et.TransitionType.STOPPED_MOVING,
     "local.transition.stop_tracking": et.TransitionType.STOP_TRACKING,
     "local.transition.start_tracking": et.TransitionType.START_TRACKING,
+    "local.transition.ble_beacon_found": et.TransitionType.BLE_BEACON_FOUND,
+    "local.transition.ble_beacon_lost": et.TransitionType.BLE_BEACON_LOST,
     "local.transition.tracking_error": et.TransitionType.TRACKING_ERROR
 }
 
@@ -44,7 +46,11 @@ def format(entry):
     data = ad.AttrDict()
     data.curr_state = state_map[entry.data.currState].value
     logging.debug("Mapped %s -> %s" % (entry.data.currState, data.curr_state))
-    data.transition = transition_map[entry.data.transition].value
+    if entry.data.transition is not None:
+        data.transition = transition_map[entry.data.transition].value
+    else:
+        data.transition = None
+
     if "ts" not in data:
         data.ts = formatted_entry.metadata.write_ts
         logging.debug("No existing timestamp, copyied from metadata%s" % data.ts)

diff --git a/emission/net/usercache/formatters/ios/bluetooth_ble.py b/emission/net/usercache/formatters/ios/bluetooth_ble.py
@@ -0,0 +1,24 @@
+import logging
+
+import emission.core.wrapper.bluetoothble as ecwb
+import emission.net.usercache.formatters.common as fc
+import attrdict as ad
+import pandas as pd
+import numpy as np
+
+def format(entry):
+    formatted_entry = ad.AttrDict()
+    formatted_entry["_id"] = entry["_id"]
+    formatted_entry.user_id = entry.user_id
+
+    metadata = entry.metadata
+    fc.expand_metadata_times(metadata)
+    formatted_entry.metadata = metadata
+
+    data = entry.data
+    data.local_dt = formatted_entry.metadata.write_local_dt
+    data.fmt_time = formatted_entry.metadata.write_fmt_time
+    data.eventType = ecwb.BLEEventTypes[entry.data.eventType].value
+    formatted_entry.data = data
+
+    return formatted_entry
diff --git a/emission/net/usercache/formatters/ios/transition.py b/emission/net/usercache/formatters/ios/transition.py
@@ -34,6 +34,8 @@
     "T_TRACKING_STOPPED": et.TransitionType.TRACKING_STOPPED,
     "T_VISIT_STARTED": et.TransitionType.VISIT_STARTED,
     "T_VISIT_ENDED": et.TransitionType.VISIT_ENDED,
+    "T_BLE_BEACON_FOUND": et.TransitionType.BLE_BEACON_FOUND,
+    "T_BLE_BEACON_LOST": et.TransitionType.BLE_BEACON_LOST,
     "T_NOP": et.TransitionType.NOP,
     "T_START_TRACKING": et.TransitionType.START_TRACKING
 }
@@ -59,7 +61,6 @@ def format(entry):
     # deal with collapsing later
     # data.transition_raw = entry.data.transition
 
-    data.transition = transition_map[entry.data.transition].value
     if entry.data.transition is not None:
         data.transition = transition_map[entry.data.transition].value
     else:

diff --git a/emission/storage/timeseries/builtin_timeseries.py b/emission/storage/timeseries/builtin_timeseries.py
@@ -45,6 +45,7 @@ def __init__(self, user_id):
                 "background/filtered_location": self.timeseries_db,
                 "background/motion_activity": self.timeseries_db,
                 "background/battery": self.timeseries_db,
+                "background/bluetooth_ble": self.timeseries_db,
                 "statemachine/transition": self.timeseries_db,
                 "config/sensor_config": self.timeseries_db,
                 "config/sync_config": self.timeseries_db,