🗃️ Plumb through the BLE objects to the server

bin/debug/fix_usercache_processing.py

@@ -27,6 +27,7 @@
 
 def fix_usercache_errors():
     copy_to_usercache()
+    print(">" * 30)
     move_to_long_term()
 
 def copy_to_usercache():       
@@ -37,8 +38,8 @@ def copy_to_usercache():
     logging.info("Found %d errors in this round" % edb.get_timeseries_error_db().estimated_document_count())
     for error in error_it:
         logging.debug("Copying entry %s" % error["metadata"])
-        save_result = uc.save(error)
-        remove_result = te.remove(error["_id"])    
+        save_result = uc.replace_one({"_id": error['_id']}, error, upsert=True)
+        remove_result = te.delete_one({"_id": error["_id"]})
         logging.debug("save_result = %s, remove_result = %s" % (save_result, remove_result))
     logging.info("step copy_to_usercache DONE")
 

emission/core/wrapper/bluetoothble.py

@@ -0,0 +1,29 @@
+import logging
+import enum as enum
+import emission.core.wrapper.wrapperbase as ecwb
+
+class BLEEventTypes(enum.Enum):
+    REGION_ENTER = 0
+    REGION_EXIT = 1
+    RANGE_UPDATE = 2
+
+class Bluetoothble(ecwb.WrapperBase):
+    props = {"eventType": ecwb.WrapperBase.Access.RO, # the type of event
+             "uuid": ecwb.WrapperBase.Access.RO,  # UUID of the beacon. Will be a constant for beacons used by e-mission, consistent with https://github.com/e-mission/e-mission-docs/issues/1062#issuecomment-2026359038
+             "major": ecwb.WrapperBase.Access.RO, # major value (matches deployment)
+             "minor": ecwb.WrapperBase.Access.RO,       # minor value (matches the actual vehicle)
+             "ts": ecwb.WrapperBase.Access.RO,        # timestamp (in seconds)
+             "proximity": ecwb.WrapperBase.Access.RO,  # how close the beacon is (used as the second step in the process, https://github.com/e-mission/e-mission-docs/issues/1062#issuecomment-2026359038
+             "local_dt": ecwb.WrapperBase.Access.RO, # searchable datetime in local time
+             "fmt_time": ecwb.WrapperBase.Access.RO,  # formatted time
+             "accuracy": ecwb.WrapperBase.Access.RO,  # only available for range updats
+             "rssi": ecwb.WrapperBase.Access.RO, # signal strength, only available for range updates
+    }
+
+    enums = {"eventType": BLEEventTypes}
+    geojson = []
+    nullable = ["major", "minor", "proximity", "accuracy", "rssi"]
+    local_dates = ['local_dt']
+
+    def _populateDependencies(self):
+        pass

emission/core/wrapper/entry.py

@@ -42,6 +42,8 @@ def _getData2Wrapper():
             "background/motion_activity": "motionactivity",
             # battery readings, to determine power drain empirically
             "background/battery": "battery",
+            # BLE events, including enter, exiting and ranging beacons
+            "background/bluetooth_ble": "bluetoothble",
             # transition events for the tracking finite state machine on the phone
             "statemachine/transition": "transition",
             # phone sensing configuration (e.g. sensing frequency, geofencing,...)

emission/core/wrapper/transition.py

@@ -39,6 +39,8 @@ class TransitionType(enum.Enum):
     DATA_PUSHED = 16
     # joint transition again
     START_TRACKING = 17
+    BLE_BEACON_FOUND = 18
+    BLE_BEACON_LOST = 19
 
 class Transition(ecwb.WrapperBase):
     props = {"curr_state": ecwb.WrapperBase.Access.RO,

emission/net/usercache/formatters/android/bluetooth_ble.py

@@ -0,0 +1,26 @@
+import logging
+
+import emission.core.wrapper.bluetoothble as ecwb
+import emission.net.usercache.formatters.common as fc
+import attrdict as ad
+
+def format(entry):
+    formatted_entry = ad.AttrDict()
+    formatted_entry["_id"] = entry["_id"]
+    formatted_entry.user_id = entry.user_id
+
+    metadata = entry.metadata
+    if "time_zone" not in metadata:
+        metadata.time_zone = "America/Los_Angeles" 
+    fc.expand_metadata_times(metadata)
+    formatted_entry.metadata = metadata
+
+    #logging.info('*** Motion Data write_ts: %d' % metadata.write_ts)
+
+    data = entry.data
+    data.local_dt = formatted_entry.metadata.write_local_dt
+    data.fmt_time = formatted_entry.metadata.write_fmt_time
+    data.eventType = ecwb.BLEEventTypes[entry.data.eventType].value
+    formatted_entry.data = data
+
+    return formatted_entry

emission/net/usercache/formatters/android/transition.py

@@ -26,6 +26,8 @@
     "local.transition.stopped_moving": et.TransitionType.STOPPED_MOVING,
     "local.transition.stop_tracking": et.TransitionType.STOP_TRACKING,
     "local.transition.start_tracking": et.TransitionType.START_TRACKING,
+    "local.transition.ble_beacon_found": et.TransitionType.BLE_BEACON_FOUND,
+    "local.transition.ble_beacon_lost": et.TransitionType.BLE_BEACON_LOST,
     "local.transition.tracking_error": et.TransitionType.TRACKING_ERROR
 }
 
@@ -44,7 +46,11 @@ def format(entry):
     data = ad.AttrDict()
     data.curr_state = state_map[entry.data.currState].value
     logging.debug("Mapped %s -> %s" % (entry.data.currState, data.curr_state))
-    data.transition = transition_map[entry.data.transition].value
+    if entry.data.transition is not None:
+        data.transition = transition_map[entry.data.transition].value
+    else:
+        data.transition = None
+
     if "ts" not in data:
         data.ts = formatted_entry.metadata.write_ts
         logging.debug("No existing timestamp, copyied from metadata%s" % data.ts)

emission/net/usercache/formatters/ios/bluetooth_ble.py

@@ -0,0 +1,24 @@
+import logging
+
+import emission.core.wrapper.bluetoothble as ecwb
+import emission.net.usercache.formatters.common as fc
+import attrdict as ad
+import pandas as pd
+import numpy as np
+
+def format(entry):
+    formatted_entry = ad.AttrDict()
+    formatted_entry["_id"] = entry["_id"]
+    formatted_entry.user_id = entry.user_id
+
+    metadata = entry.metadata
+    fc.expand_metadata_times(metadata)
+    formatted_entry.metadata = metadata
+
+    data = entry.data
+    data.local_dt = formatted_entry.metadata.write_local_dt
+    data.fmt_time = formatted_entry.metadata.write_fmt_time
+    data.eventType = ecwb.BLEEventTypes[entry.data.eventType].value
+    formatted_entry.data = data
+
+    return formatted_entry

emission/net/usercache/formatters/ios/transition.py

@@ -34,6 +34,8 @@
     "T_TRACKING_STOPPED": et.TransitionType.TRACKING_STOPPED,
     "T_VISIT_STARTED": et.TransitionType.VISIT_STARTED,
     "T_VISIT_ENDED": et.TransitionType.VISIT_ENDED,
+    "T_BLE_BEACON_FOUND": et.TransitionType.BLE_BEACON_FOUND,
+    "T_BLE_BEACON_LOST": et.TransitionType.BLE_BEACON_LOST,
     "T_NOP": et.TransitionType.NOP,
     "T_START_TRACKING": et.TransitionType.START_TRACKING
 }
@@ -59,7 +61,6 @@ def format(entry):
     # deal with collapsing later
     # data.transition_raw = entry.data.transition
 
-    data.transition = transition_map[entry.data.transition].value
     if entry.data.transition is not None:
         data.transition = transition_map[entry.data.transition].value
     else:

emission/storage/timeseries/builtin_timeseries.py

@@ -45,6 +45,7 @@ def __init__(self, user_id):
                 "background/filtered_location": self.timeseries_db,
                 "background/motion_activity": self.timeseries_db,
                 "background/battery": self.timeseries_db,
+                "background/bluetooth_ble": self.timeseries_db,
                 "statemachine/transition": self.timeseries_db,
                 "config/sensor_config": self.timeseries_db,
                 "config/sync_config": self.timeseries_db,

emission/tests/netTests/TestBuiltinUserCacheHandlerInput.py

@@ -138,7 +138,7 @@ def testMoveWhenEmpty(self):
         edb.get_usercache_db().insert_one({
             'user_id': self.testUserUUID1,
             '_id': boi.ObjectId('572d3621d282b8f30def7e85'),
-            'data': {u'transition': None,
+            'data': {u'transition': "MY_FAKE_TRANSITION",
                      'currState': u'STATE_ONGOING_TRIP'},
             'metadata': {'plugin': 'none',
                          'write_ts': self.curr_ts - 25,