diff --git a/emission/analysis/configs/dynamic_config.py b/emission/analysis/configs/dynamic_config.py new file mode 100644 index 000000000..d62e441f6 --- /dev/null +++ b/emission/analysis/configs/dynamic_config.py @@ -0,0 +1,30 @@ +import sys +import os +import logging + +import json +import requests + +STUDY_CONFIG = os.getenv('STUDY_CONFIG', "stage-program") + +dynamic_config = None +def get_dynamic_config(): + global dynamic_config + if dynamic_config is not None: + logging.debug("Returning cached dynamic config for %s at version %s" % (STUDY_CONFIG, dynamic_config['version'])) + return dynamic_config + logging.debug("No cached dynamic config for %s, downloading from server" % STUDY_CONFIG) + download_url = "https://raw.githubusercontent.com/e-mission/nrel-openpath-deploy-configs/main/configs/" + STUDY_CONFIG + ".nrel-op.json" + logging.debug("About to download config from %s" % download_url) + r = requests.get(download_url) + if r.status_code != 200: + logging.debug(f"Unable to download study config, status code: {r.status_code}") + # sys.exit(1) + # TODO what to do here? What if Github is down or something? + # If we terminate, will the pipeline just try again later? + else: + dynamic_config = json.loads(r.text) + logging.debug(f"Successfully downloaded config with version {dynamic_config['version']} "\ + f"for {dynamic_config['intro']['translated_text']['en']['deployment_name']} "\ + f"and data collection URL {dynamic_config['server']['connectUrl']}") + return dynamic_config diff --git a/emission/analysis/intake/segmentation/section_segmentation.py b/emission/analysis/intake/segmentation/section_segmentation.py index 609e1737c..ffcd2dadb 100644 --- a/emission/analysis/intake/segmentation/section_segmentation.py +++ b/emission/analysis/intake/segmentation/section_segmentation.py @@ -10,6 +10,7 @@ import logging # Our imports +import emission.analysis.configs.dynamic_config as eadc import emission.storage.pipeline_queries as epq import emission.storage.decorations.analysis_timeseries_queries as esda @@ -22,6 +23,7 @@ import emission.core.wrapper.entry as ecwe import emission.core.common as ecc +import emcommon.bluetooth.ble_matching as emcble class SectionSegmentationMethod(object): def segment_into_sections(self, timeseries, distance_from_place, time_query): @@ -64,6 +66,7 @@ def segment_trip_into_sections(user_id, trip_entry, trip_source): ts = esta.TimeSeries.get_time_series(user_id) time_query = esda.get_time_query_for_trip_like(esda.RAW_TRIP_KEY, trip_entry.get_id()) distance_from_place = _get_distance_from_start_place_to_end(trip_entry) + ble_entries_during_trip = ts.find_entries(["background/bluetooth_ble"], time_query) if (trip_source == "DwellSegmentationTimeFilter"): import emission.analysis.intake.segmentation.section_segmentation_methods.smoothed_high_confidence_motion as shcm @@ -118,7 +121,16 @@ def segment_trip_into_sections(user_id, trip_entry, trip_source): # Particularly in this case, if we don't do this, then the trip end may overshoot the section end end_loc = trip_end_loc - fill_section(section, start_loc, end_loc, sensed_mode) + # ble_sensed_mode represents the vehicle that was sensed via BLE beacon during the section. + # For now, we are going to rely on the current segmentation implementation and then fill in + # ble_sensed_mode by looking at scans within the timestamp range of the section. + # Later, we may want to actually use BLE sensor data as part of the basis for segmentation + dynamic_config = eadc.get_dynamic_config() + ble_sensed_mode = emcble.get_ble_sensed_vehicle_for_section( + ble_entries_during_trip, start_loc.ts, end_loc.ts, dynamic_config + ) + + fill_section(section, start_loc, end_loc, sensed_mode, ble_sensed_mode) # We create the entry after filling in the section so that we know # that the data is included properly section_entry = ecwe.Entry.create_entry(user_id, esda.RAW_SECTION_KEY, @@ -143,7 +155,7 @@ def segment_trip_into_sections(user_id, trip_entry, trip_source): prev_section_entry = section_entry -def fill_section(section, start_loc, end_loc, sensed_mode): +def fill_section(section, start_loc, end_loc, sensed_mode, ble_sensed_mode=None): section.start_ts = start_loc.ts section.start_local_dt = start_loc.local_dt section.start_fmt_time = start_loc.fmt_time @@ -161,6 +173,7 @@ def fill_section(section, start_loc, end_loc, sensed_mode): section.duration = end_loc.ts - start_loc.ts section.source = "SmoothedHighConfidenceMotion" section.sensed_mode = sensed_mode + section.ble_sensed_mode = ble_sensed_mode def stitch_together(ending_section_entry, stop_entry, starting_section_entry): diff --git a/emission/analysis/userinput/matcher.py b/emission/analysis/userinput/matcher.py index ee4365261..9263d6c79 100644 --- a/emission/analysis/userinput/matcher.py +++ b/emission/analysis/userinput/matcher.py @@ -193,12 +193,13 @@ def create_and_link_timeline(ts, timeline, last_confirmed_place): confirmed_places, confirmed_trips) return confirmed_tl -def get_section_summary(ts, cleaned_trip, section_key): +def get_section_summary(ts, cleaned_trip, section_key, mode_prop="sensed_mode"): """ Returns the proportions of the distance, duration and count for each mode in this trip. Note that sections are unimodal by definition. cleaned_trip: the cleaned trip object associated with the sections section_key: 'inferred_section' or 'cleaned_section' + mode_prop: the section property used as the basis for mode: 'sensed_mode' or 'ble_sensed_mode'. """ logging.debug(f"get_section_summary({cleaned_trip['_id']}, {section_key}) called") sections = esdt.get_sections_for_trip(key = section_key, @@ -207,12 +208,14 @@ def get_section_summary(ts, cleaned_trip, section_key): logging.warning("While getting section summary, section length = 0. This should never happen, but let's not crash if it does") return {"distance": {}, "duration": {}, "count": {}} sections_df = ts.to_data_df(section_key, sections) - cleaned_section_mapper = lambda sm: ecwm.MotionTypes(sm).name - inferred_section_mapper = lambda sm: ecwmp.PredictedModeTypes(sm).name - sel_section_mapper = cleaned_section_mapper \ - if section_key == "analysis/cleaned_section" else inferred_section_mapper - sections_df["sensed_mode_str"] = sections_df["sensed_mode"].apply(sel_section_mapper) - grouped_section_df = sections_df.groupby("sensed_mode_str") + if mode_prop == "ble_sensed_mode": + mapper = lambda bsm: bsm['baseMode'] if bsm is not None else ecwm.MotionTypes.UNKNOWN.name + elif section_key == "analysis/cleaned_section": + mapper = lambda sm: ecwm.MotionTypes(sm).name + else: + mapper = lambda sm: ecwmp.PredictedModeTypes(sm).name + sections_df[mode_prop + "_str"] = sections_df[mode_prop].apply(mapper) + grouped_section_df = sections_df.groupby(mode_prop + "_str") retVal = { "distance": grouped_section_df.distance.sum().to_dict(), "duration": grouped_section_df.duration.sum().to_dict(), @@ -233,6 +236,7 @@ def create_confirmed_entry(ts, tce, confirmed_key, input_key_list): tce["data"]["cleaned_trip"]) confirmed_object_data['inferred_section_summary'] = get_section_summary(ts, cleaned_trip, "analysis/inferred_section") confirmed_object_data['cleaned_section_summary'] = get_section_summary(ts, cleaned_trip, "analysis/cleaned_section") + confirmed_object_data['ble_sensed_summary'] = get_section_summary(ts, cleaned_trip, "analysis/inferred_section", mode_prop="ble_sensed_mode") elif (confirmed_key == esda.CONFIRMED_PLACE_KEY): confirmed_object_data["cleaned_place"] = tce.get_id() confirmed_object_data["user_input"] = \ diff --git a/emission/core/wrapper/confirmedtrip.py b/emission/core/wrapper/confirmedtrip.py index 5a67c89fa..9a83efb2c 100644 --- a/emission/core/wrapper/confirmedtrip.py +++ b/emission/core/wrapper/confirmedtrip.py @@ -19,6 +19,7 @@ class Confirmedtrip(ecwt.Trip): "expected_trip": ecwb.WrapperBase.Access.WORM, "inferred_section_summary": ecwb.WrapperBase.Access.WORM, "cleaned_section_summary": ecwb.WrapperBase.Access.WORM, + "ble_sensed_summary": ecwb.WrapperBase.Access.WORM, # the user input will have all `manual/*` entries # let's make that be somewhat flexible instead of hardcoding into the data model "user_input": ecwb.WrapperBase.Access.WORM, diff --git a/emission/core/wrapper/section.py b/emission/core/wrapper/section.py index a6f059b9d..5547d2880 100644 --- a/emission/core/wrapper/section.py +++ b/emission/core/wrapper/section.py @@ -25,6 +25,7 @@ class Section(ecwb.WrapperBase): "end_loc": ecwb.WrapperBase.Access.WORM, # location of end point in geojson format "duration": ecwb.WrapperBase.Access.WORM, # duration of the trip in secs "sensed_mode": ecwb.WrapperBase.Access.WORM, # the sensed mode used for the segmentation + "ble_sensed_mode": ecwb.WrapperBase.Access.WORM, # the mode sensed from BLE beacon scans "source": ecwb.WrapperBase.Access.WORM} # the method used to generate this trip enums = {"sensed_mode": ecwm.MotionTypes} diff --git a/emission/tests/analysisTests/userInputTests/TestUserInput.py b/emission/tests/analysisTests/userInputTests/TestUserInput.py index 49a9ffce4..c5455ab2a 100644 --- a/emission/tests/analysisTests/userInputTests/TestUserInput.py +++ b/emission/tests/analysisTests/userInputTests/TestUserInput.py @@ -90,6 +90,8 @@ def compare_confirmed_objs_result(self, result_dicts, expect_dicts, manual_keys self.assertEqual(rt.data["inferred_section_summary"], et.data["inferred_section_summary"]) if "cleaned_section_summary" in et.data: self.assertEqual(rt.data["cleaned_section_summary"], et.data["cleaned_section_summary"]) + if 'ble_sensed_summary' in et.data: + self.assertEqual(rt.data["ble_sensed_summary"], et.data["ble_sensed_summary"]) logging.debug(20 * "=") def compare_section_result(self, result, expect): diff --git a/emission/tests/data/real_examples/shankari_2016-06-20.expected_confirmed_trips b/emission/tests/data/real_examples/shankari_2016-06-20.expected_confirmed_trips index c2a170e78..c4262cebb 100644 --- a/emission/tests/data/real_examples/shankari_2016-06-20.expected_confirmed_trips +++ b/emission/tests/data/real_examples/shankari_2016-06-20.expected_confirmed_trips @@ -1,27 +1,27 @@ [ { "_id": { - "$oid": "6493309c15a3780e6be29fc9" + "$oid": "6635321269e6b3b5d07065f2" }, "user_id": { - "$uuid": "aa9fdec9-2944-446c-8ee2-50d79b3044d3" + "$uuid": "344d603232dc4e6983e4eecdddf1f6f5" }, "metadata": { "key": "analysis/confirmed_trip", "platform": "server", - "write_ts": 1687367836.921912, + "write_ts": 1714762258.733232, "time_zone": "America/Los_Angeles", "write_local_dt": { - "year": 2023, - "month": 6, - "day": 21, - "hour": 10, - "minute": 17, - "second": 16, - "weekday": 2, + "year": 2024, + "month": 5, + "day": 3, + "hour": 11, + "minute": 50, + "second": 58, + "weekday": 4, "timezone": "America/Los_Angeles" }, - "write_fmt_time": "2023-06-21T10:17:16.921912-07:00" + "write_fmt_time": "2024-05-03T11:50:58.733232-07:00" }, "data": { "source": "DwellSegmentationTimeFilter", @@ -45,7 +45,7 @@ ] }, "raw_trip": { - "$oid": "6493309815a3780e6be29ecf" + "$oid": "6635321069e6b3b5d07064f9" }, "start_ts": 1466436483.395, "start_local_dt": { @@ -69,24 +69,24 @@ "duration": 792.4609999656677, "distance": 1047.1630675866315, "start_place": { - "$oid": "6493309c15a3780e6be29fc8" + "$oid": "6635321269e6b3b5d07065f1" }, "end_place": { - "$oid": "6493309c15a3780e6be29fca" + "$oid": "6635321269e6b3b5d07065f3" }, "cleaned_trip": { - "$oid": "6493309915a3780e6be29ef2" + "$oid": "6635321169e6b3b5d070651c" }, "inferred_labels": [], "inferred_trip": { - "$oid": "6493309c15a3780e6be29fae" + "$oid": "6635321269e6b3b5d07065d7" }, "expectation": { "to_label": true }, "confidence_threshold": 0.55, "expected_trip": { - "$oid": "6493309c15a3780e6be29fc1" + "$oid": "6635321269e6b3b5d07065ea" }, "inferred_section_summary": { "distance": {}, @@ -104,6 +104,11 @@ "ON_FOOT": 1 } }, + "ble_sensed_summary": { + "distance": {}, + "duration": {}, + "count": {} + }, "user_input": { "mode_confirm": "walk", "purpose_confirm": "library" @@ -113,27 +118,27 @@ }, { "_id": { - "$oid": "6493309d15a3780e6be29fcb" + "$oid": "6635321269e6b3b5d07065f4" }, "user_id": { - "$uuid": "aa9fdec9-2944-446c-8ee2-50d79b3044d3" + "$uuid": "344d603232dc4e6983e4eecdddf1f6f5" }, "metadata": { "key": "analysis/confirmed_trip", "platform": "server", - "write_ts": 1687367837.162344, + "write_ts": 1714762258.883084, "time_zone": "America/Los_Angeles", "write_local_dt": { - "year": 2023, - "month": 6, - "day": 21, - "hour": 10, - "minute": 17, - "second": 17, - "weekday": 2, + "year": 2024, + "month": 5, + "day": 3, + "hour": 11, + "minute": 50, + "second": 58, + "weekday": 4, "timezone": "America/Los_Angeles" }, - "write_fmt_time": "2023-06-21T10:17:17.162344-07:00" + "write_fmt_time": "2024-05-03T11:50:58.883084-07:00" }, "data": { "source": "DwellSegmentationTimeFilter", @@ -157,7 +162,7 @@ ] }, "raw_trip": { - "$oid": "6493309815a3780e6be29ed1" + "$oid": "6635321069e6b3b5d07064fb" }, "start_ts": 1466437438.6453953, "start_local_dt": { @@ -181,24 +186,24 @@ "duration": 584.3136048316956, "distance": 886.4937093667857, "start_place": { - "$oid": "6493309c15a3780e6be29fca" + "$oid": "6635321269e6b3b5d07065f3" }, "end_place": { - "$oid": "6493309d15a3780e6be29fcc" + "$oid": "6635321269e6b3b5d07065f5" }, "cleaned_trip": { - "$oid": "6493309a15a3780e6be29f11" + "$oid": "6635321169e6b3b5d070653b" }, "inferred_labels": [], "inferred_trip": { - "$oid": "6493309c15a3780e6be29fb1" + "$oid": "6635321269e6b3b5d07065d8" }, "expectation": { "to_label": true }, "confidence_threshold": 0.55, "expected_trip": { - "$oid": "6493309c15a3780e6be29fc2" + "$oid": "6635321269e6b3b5d07065eb" }, "inferred_section_summary": { "distance": {}, @@ -216,33 +221,38 @@ "ON_FOOT": 1 } }, + "ble_sensed_summary": { + "distance": {}, + "duration": {}, + "count": {} + }, "user_input": {}, "additions": [] } }, { "_id": { - "$oid": "6493309d15a3780e6be29fcd" + "$oid": "6635321369e6b3b5d07065f6" }, "user_id": { - "$uuid": "aa9fdec9-2944-446c-8ee2-50d79b3044d3" + "$uuid": "344d603232dc4e6983e4eecdddf1f6f5" }, "metadata": { "key": "analysis/confirmed_trip", "platform": "server", - "write_ts": 1687367837.410047, + "write_ts": 1714762259.064074, "time_zone": "America/Los_Angeles", "write_local_dt": { - "year": 2023, - "month": 6, - "day": 21, - "hour": 10, - "minute": 17, - "second": 17, - "weekday": 2, + "year": 2024, + "month": 5, + "day": 3, + "hour": 11, + "minute": 50, + "second": 59, + "weekday": 4, "timezone": "America/Los_Angeles" }, - "write_fmt_time": "2023-06-21T10:17:17.410047-07:00" + "write_fmt_time": "2024-05-03T11:50:59.064074-07:00" }, "data": { "source": "DwellSegmentationTimeFilter", @@ -266,7 +276,7 @@ ] }, "raw_trip": { - "$oid": "6493309815a3780e6be29ed3" + "$oid": "6635321069e6b3b5d07064fd" }, "start_ts": 1466461623.1195338, "start_local_dt": { @@ -290,24 +300,24 @@ "duration": 343.25946617126465, "distance": 610.2234223038181, "start_place": { - "$oid": "6493309d15a3780e6be29fcc" + "$oid": "6635321269e6b3b5d07065f5" }, "end_place": { - "$oid": "6493309d15a3780e6be29fce" + "$oid": "6635321369e6b3b5d07065f7" }, "cleaned_trip": { - "$oid": "6493309a15a3780e6be29f29" + "$oid": "6635321169e6b3b5d0706553" }, "inferred_labels": [], "inferred_trip": { - "$oid": "6493309c15a3780e6be29fb4" + "$oid": "6635321269e6b3b5d07065d9" }, "expectation": { "to_label": true }, "confidence_threshold": 0.55, "expected_trip": { - "$oid": "6493309c15a3780e6be29fc3" + "$oid": "6635321269e6b3b5d07065ec" }, "inferred_section_summary": { "distance": {}, @@ -325,33 +335,38 @@ "ON_FOOT": 1 } }, + "ble_sensed_summary": { + "distance": {}, + "duration": {}, + "count": {} + }, "user_input": {}, "additions": [] } }, { "_id": { - "$oid": "6493309d15a3780e6be29fcf" + "$oid": "6635321369e6b3b5d07065f8" }, "user_id": { - "$uuid": "aa9fdec9-2944-446c-8ee2-50d79b3044d3" + "$uuid": "344d603232dc4e6983e4eecdddf1f6f5" }, "metadata": { "key": "analysis/confirmed_trip", "platform": "server", - "write_ts": 1687367837.665754, + "write_ts": 1714762259.2089221, "time_zone": "America/Los_Angeles", "write_local_dt": { - "year": 2023, - "month": 6, - "day": 21, - "hour": 10, - "minute": 17, - "second": 17, - "weekday": 2, + "year": 2024, + "month": 5, + "day": 3, + "hour": 11, + "minute": 50, + "second": 59, + "weekday": 4, "timezone": "America/Los_Angeles" }, - "write_fmt_time": "2023-06-21T10:17:17.665754-07:00" + "write_fmt_time": "2024-05-03T11:50:59.208922-07:00" }, "data": { "source": "DwellSegmentationTimeFilter", @@ -375,7 +390,7 @@ ] }, "raw_trip": { - "$oid": "6493309815a3780e6be29ed5" + "$oid": "6635321069e6b3b5d07064ff" }, "start_ts": 1466462052.158904, "start_local_dt": { @@ -399,24 +414,24 @@ "duration": 400.54909586906433, "distance": 405.97685486691756, "start_place": { - "$oid": "6493309d15a3780e6be29fce" + "$oid": "6635321369e6b3b5d07065f7" }, "end_place": { - "$oid": "6493309d15a3780e6be29fd0" + "$oid": "6635321369e6b3b5d07065f9" }, "cleaned_trip": { - "$oid": "6493309a15a3780e6be29f39" + "$oid": "6635321169e6b3b5d0706563" }, "inferred_labels": [], "inferred_trip": { - "$oid": "6493309c15a3780e6be29fb7" + "$oid": "6635321269e6b3b5d07065da" }, "expectation": { "to_label": true }, "confidence_threshold": 0.55, "expected_trip": { - "$oid": "6493309c15a3780e6be29fc4" + "$oid": "6635321269e6b3b5d07065ed" }, "inferred_section_summary": { "distance": {}, @@ -434,6 +449,11 @@ "ON_FOOT": 1 } }, + "ble_sensed_summary": { + "distance": {}, + "duration": {}, + "count": {} + }, "user_input": { "mode_confirm": "walk", "purpose_confirm": "home" @@ -443,27 +463,27 @@ }, { "_id": { - "$oid": "6493309d15a3780e6be29fd1" + "$oid": "6635321369e6b3b5d07065fa" }, "user_id": { - "$uuid": "aa9fdec9-2944-446c-8ee2-50d79b3044d3" + "$uuid": "344d603232dc4e6983e4eecdddf1f6f5" }, "metadata": { "key": "analysis/confirmed_trip", "platform": "server", - "write_ts": 1687367837.881526, + "write_ts": 1714762259.4899352, "time_zone": "America/Los_Angeles", "write_local_dt": { - "year": 2023, - "month": 6, - "day": 21, - "hour": 10, - "minute": 17, - "second": 17, - "weekday": 2, + "year": 2024, + "month": 5, + "day": 3, + "hour": 11, + "minute": 50, + "second": 59, + "weekday": 4, "timezone": "America/Los_Angeles" }, - "write_fmt_time": "2023-06-21T10:17:17.881526-07:00" + "write_fmt_time": "2024-05-03T11:50:59.489935-07:00" }, "data": { "source": "DwellSegmentationTimeFilter", @@ -487,7 +507,7 @@ ] }, "raw_trip": { - "$oid": "6493309815a3780e6be29ed7" + "$oid": "6635321069e6b3b5d0706501" }, "start_ts": 1466462970.2807262, "start_local_dt": { @@ -511,24 +531,24 @@ "duration": 865.4322738647461, "distance": 4521.417177464177, "start_place": { - "$oid": "6493309d15a3780e6be29fd0" + "$oid": "6635321369e6b3b5d07065f9" }, "end_place": { - "$oid": "6493309d15a3780e6be29fd2" + "$oid": "6635321369e6b3b5d07065fb" }, "cleaned_trip": { - "$oid": "6493309b15a3780e6be29f4b" + "$oid": "6635321269e6b3b5d0706575" }, "inferred_labels": [], "inferred_trip": { - "$oid": "6493309c15a3780e6be29fba" + "$oid": "6635321269e6b3b5d07065db" }, "expectation": { "to_label": true }, "confidence_threshold": 0.55, "expected_trip": { - "$oid": "6493309c15a3780e6be29fc5" + "$oid": "6635321269e6b3b5d07065ee" }, "inferred_section_summary": { "distance": {}, @@ -549,6 +569,11 @@ "ON_FOOT": 1 } }, + "ble_sensed_summary": { + "distance": {}, + "duration": {}, + "count": {} + }, "user_input": { "mode_confirm": "shared_ride", "purpose_confirm": "karate" @@ -558,27 +583,27 @@ }, { "_id": { - "$oid": "6493309e15a3780e6be29fd3" + "$oid": "6635321369e6b3b5d07065fc" }, "user_id": { - "$uuid": "aa9fdec9-2944-446c-8ee2-50d79b3044d3" + "$uuid": "344d603232dc4e6983e4eecdddf1f6f5" }, "metadata": { "key": "analysis/confirmed_trip", "platform": "server", - "write_ts": 1687367838.1055691, + "write_ts": 1714762259.66389, "time_zone": "America/Los_Angeles", "write_local_dt": { - "year": 2023, - "month": 6, - "day": 21, - "hour": 10, - "minute": 17, - "second": 18, - "weekday": 2, + "year": 2024, + "month": 5, + "day": 3, + "hour": 11, + "minute": 50, + "second": 59, + "weekday": 4, "timezone": "America/Los_Angeles" }, - "write_fmt_time": "2023-06-21T10:17:18.105569-07:00" + "write_fmt_time": "2024-05-03T11:50:59.663890-07:00" }, "data": { "source": "DwellSegmentationTimeFilter", @@ -602,7 +627,7 @@ ] }, "raw_trip": { - "$oid": "6493309815a3780e6be29edb" + "$oid": "6635321069e6b3b5d0706505" }, "start_ts": 1466466589.3710227, "start_local_dt": { @@ -626,24 +651,24 @@ "duration": 1370.3959772586823, "distance": 5396.705710302932, "start_place": { - "$oid": "6493309d15a3780e6be29fd2" + "$oid": "6635321369e6b3b5d07065fb" }, "end_place": { - "$oid": "6493309e15a3780e6be29fd4" + "$oid": "6635321369e6b3b5d07065fd" }, "cleaned_trip": { - "$oid": "6493309b15a3780e6be29f6f" + "$oid": "6635321269e6b3b5d0706599" }, "inferred_labels": [], "inferred_trip": { - "$oid": "6493309c15a3780e6be29fbd" + "$oid": "6635321269e6b3b5d07065dc" }, "expectation": { "to_label": true }, "confidence_threshold": 0.55, "expected_trip": { - "$oid": "6493309c15a3780e6be29fc6" + "$oid": "6635321269e6b3b5d07065ef" }, "inferred_section_summary": { "distance": {}, @@ -664,6 +689,11 @@ "ON_FOOT": 1 } }, + "ble_sensed_summary": { + "distance": {}, + "duration": {}, + "count": {} + }, "user_input": { "mode_confirm": "shared_ride", "purpose_confirm": "home" @@ -671,4 +701,4 @@ "additions": [] } } -] +] \ No newline at end of file diff --git a/emission/tests/data/real_examples/shankari_2016-06-20.expected_confirmed_trips.manual_trip_addition_input b/emission/tests/data/real_examples/shankari_2016-06-20.expected_confirmed_trips.manual_trip_addition_input index d6d96ec67..fb4711e7b 100644 --- a/emission/tests/data/real_examples/shankari_2016-06-20.expected_confirmed_trips.manual_trip_addition_input +++ b/emission/tests/data/real_examples/shankari_2016-06-20.expected_confirmed_trips.manual_trip_addition_input @@ -1,27 +1,27 @@ [ { "_id": { - "$oid": "64933307ddae82d93a5158db" + "$oid": "6635321a69e6b3b5d0706702" }, "user_id": { - "$uuid": "4ef98df66c794a4ea4af156986d5e167" + "$uuid": "233a2d2f55184118a8071aa93d92d35a" }, "metadata": { "key": "analysis/confirmed_trip", "platform": "server", - "write_ts": 1687368455.0237079, + "write_ts": 1714762266.4695292, "time_zone": "America/Los_Angeles", "write_local_dt": { - "year": 2023, - "month": 6, - "day": 21, - "hour": 10, - "minute": 27, - "second": 35, - "weekday": 2, + "year": 2024, + "month": 5, + "day": 3, + "hour": 11, + "minute": 51, + "second": 6, + "weekday": 4, "timezone": "America/Los_Angeles" }, - "write_fmt_time": "2023-06-21T10:27:35.023708-07:00" + "write_fmt_time": "2024-05-03T11:51:06.469529-07:00" }, "data": { "source": "DwellSegmentationTimeFilter", @@ -45,7 +45,7 @@ ] }, "raw_trip": { - "$oid": "64933303ddae82d93a5157e1" + "$oid": "6635321869e6b3b5d0706609" }, "start_ts": 1466436483.395, "start_local_dt": { @@ -69,24 +69,24 @@ "duration": 792.4609999656677, "distance": 1047.1630675866315, "start_place": { - "$oid": "64933306ddae82d93a5158da" + "$oid": "6635321a69e6b3b5d0706701" }, "end_place": { - "$oid": "64933307ddae82d93a5158dc" + "$oid": "6635321a69e6b3b5d0706703" }, "cleaned_trip": { - "$oid": "64933305ddae82d93a515804" + "$oid": "6635321969e6b3b5d070662c" }, "inferred_labels": [], "inferred_trip": { - "$oid": "64933306ddae82d93a5158c0" + "$oid": "6635321a69e6b3b5d07066e7" }, "expectation": { "to_label": true }, "confidence_threshold": 0.55, "expected_trip": { - "$oid": "64933306ddae82d93a5158d3" + "$oid": "6635321a69e6b3b5d07066fa" }, "inferred_section_summary": { "distance": {}, @@ -104,6 +104,11 @@ "ON_FOOT": 1 } }, + "ble_sensed_summary": { + "distance": {}, + "duration": {}, + "count": {} + }, "user_input": {}, "additions": [ { @@ -111,7 +116,7 @@ "$oid": "640cad8d41c2a574a7b4a801" }, "user_id": { - "$uuid": "4ef98df66c794a4ea4af156986d5e167" + "$uuid": "233a2d2f55184118a8071aa93d92d35a" }, "metadata": { "time_zone": "America/New_York", @@ -198,27 +203,27 @@ }, { "_id": { - "$oid": "64933307ddae82d93a5158dd" + "$oid": "6635321a69e6b3b5d0706704" }, "user_id": { - "$uuid": "4ef98df66c794a4ea4af156986d5e167" + "$uuid": "233a2d2f55184118a8071aa93d92d35a" }, "metadata": { "key": "analysis/confirmed_trip", "platform": "server", - "write_ts": 1687368455.210249, + "write_ts": 1714762266.623338, "time_zone": "America/Los_Angeles", "write_local_dt": { - "year": 2023, - "month": 6, - "day": 21, - "hour": 10, - "minute": 27, - "second": 35, - "weekday": 2, + "year": 2024, + "month": 5, + "day": 3, + "hour": 11, + "minute": 51, + "second": 6, + "weekday": 4, "timezone": "America/Los_Angeles" }, - "write_fmt_time": "2023-06-21T10:27:35.210249-07:00" + "write_fmt_time": "2024-05-03T11:51:06.623338-07:00" }, "data": { "source": "DwellSegmentationTimeFilter", @@ -242,7 +247,7 @@ ] }, "raw_trip": { - "$oid": "64933303ddae82d93a5157e3" + "$oid": "6635321869e6b3b5d070660b" }, "start_ts": 1466437438.6453953, "start_local_dt": { @@ -266,24 +271,24 @@ "duration": 584.3136048316956, "distance": 886.4937093667857, "start_place": { - "$oid": "64933307ddae82d93a5158dc" + "$oid": "6635321a69e6b3b5d0706703" }, "end_place": { - "$oid": "64933307ddae82d93a5158de" + "$oid": "6635321a69e6b3b5d0706705" }, "cleaned_trip": { - "$oid": "64933305ddae82d93a515823" + "$oid": "6635321969e6b3b5d070664b" }, "inferred_labels": [], "inferred_trip": { - "$oid": "64933306ddae82d93a5158c3" + "$oid": "6635321a69e6b3b5d07066e8" }, "expectation": { "to_label": true }, "confidence_threshold": 0.55, "expected_trip": { - "$oid": "64933306ddae82d93a5158d4" + "$oid": "6635321a69e6b3b5d07066fb" }, "inferred_section_summary": { "distance": {}, @@ -301,6 +306,11 @@ "ON_FOOT": 1 } }, + "ble_sensed_summary": { + "distance": {}, + "duration": {}, + "count": {} + }, "user_input": {}, "additions": [ { @@ -308,7 +318,7 @@ "$oid": "640cad8d41c2a574a7b4a809" }, "user_id": { - "$uuid": "4ef98df66c794a4ea4af156986d5e167" + "$uuid": "233a2d2f55184118a8071aa93d92d35a" }, "metadata": { "time_zone": "America/New_York", @@ -395,27 +405,27 @@ }, { "_id": { - "$oid": "64933307ddae82d93a5158df" + "$oid": "6635321a69e6b3b5d0706706" }, "user_id": { - "$uuid": "4ef98df66c794a4ea4af156986d5e167" + "$uuid": "233a2d2f55184118a8071aa93d92d35a" }, "metadata": { "key": "analysis/confirmed_trip", "platform": "server", - "write_ts": 1687368455.392829, + "write_ts": 1714762266.803599, "time_zone": "America/Los_Angeles", "write_local_dt": { - "year": 2023, - "month": 6, - "day": 21, - "hour": 10, - "minute": 27, - "second": 35, - "weekday": 2, + "year": 2024, + "month": 5, + "day": 3, + "hour": 11, + "minute": 51, + "second": 6, + "weekday": 4, "timezone": "America/Los_Angeles" }, - "write_fmt_time": "2023-06-21T10:27:35.392829-07:00" + "write_fmt_time": "2024-05-03T11:51:06.803599-07:00" }, "data": { "source": "DwellSegmentationTimeFilter", @@ -439,7 +449,7 @@ ] }, "raw_trip": { - "$oid": "64933303ddae82d93a5157e5" + "$oid": "6635321869e6b3b5d070660d" }, "start_ts": 1466461623.1195338, "start_local_dt": { @@ -463,24 +473,24 @@ "duration": 343.25946617126465, "distance": 610.2234223038181, "start_place": { - "$oid": "64933307ddae82d93a5158de" + "$oid": "6635321a69e6b3b5d0706705" }, "end_place": { - "$oid": "64933307ddae82d93a5158e0" + "$oid": "6635321a69e6b3b5d0706707" }, "cleaned_trip": { - "$oid": "64933305ddae82d93a51583b" + "$oid": "6635321969e6b3b5d0706663" }, "inferred_labels": [], "inferred_trip": { - "$oid": "64933306ddae82d93a5158c6" + "$oid": "6635321a69e6b3b5d07066e9" }, "expectation": { "to_label": true }, "confidence_threshold": 0.55, "expected_trip": { - "$oid": "64933306ddae82d93a5158d5" + "$oid": "6635321a69e6b3b5d07066fc" }, "inferred_section_summary": { "distance": {}, @@ -498,33 +508,38 @@ "ON_FOOT": 1 } }, + "ble_sensed_summary": { + "distance": {}, + "duration": {}, + "count": {} + }, "user_input": {}, "additions": [] } }, { "_id": { - "$oid": "64933307ddae82d93a5158e1" + "$oid": "6635321a69e6b3b5d0706708" }, "user_id": { - "$uuid": "4ef98df66c794a4ea4af156986d5e167" + "$uuid": "233a2d2f55184118a8071aa93d92d35a" }, "metadata": { "key": "analysis/confirmed_trip", "platform": "server", - "write_ts": 1687368455.578741, + "write_ts": 1714762266.993064, "time_zone": "America/Los_Angeles", "write_local_dt": { - "year": 2023, - "month": 6, - "day": 21, - "hour": 10, - "minute": 27, - "second": 35, - "weekday": 2, + "year": 2024, + "month": 5, + "day": 3, + "hour": 11, + "minute": 51, + "second": 6, + "weekday": 4, "timezone": "America/Los_Angeles" }, - "write_fmt_time": "2023-06-21T10:27:35.578741-07:00" + "write_fmt_time": "2024-05-03T11:51:06.993064-07:00" }, "data": { "source": "DwellSegmentationTimeFilter", @@ -548,7 +563,7 @@ ] }, "raw_trip": { - "$oid": "64933303ddae82d93a5157e7" + "$oid": "6635321869e6b3b5d070660f" }, "start_ts": 1466462052.158904, "start_local_dt": { @@ -572,24 +587,24 @@ "duration": 400.54909586906433, "distance": 405.97685486691756, "start_place": { - "$oid": "64933307ddae82d93a5158e0" + "$oid": "6635321a69e6b3b5d0706707" }, "end_place": { - "$oid": "64933307ddae82d93a5158e2" + "$oid": "6635321b69e6b3b5d0706709" }, "cleaned_trip": { - "$oid": "64933305ddae82d93a51584b" + "$oid": "6635321969e6b3b5d0706673" }, "inferred_labels": [], "inferred_trip": { - "$oid": "64933306ddae82d93a5158c9" + "$oid": "6635321a69e6b3b5d07066ea" }, "expectation": { "to_label": true }, "confidence_threshold": 0.55, "expected_trip": { - "$oid": "64933306ddae82d93a5158d6" + "$oid": "6635321a69e6b3b5d07066fd" }, "inferred_section_summary": { "distance": {}, @@ -607,33 +622,38 @@ "ON_FOOT": 1 } }, + "ble_sensed_summary": { + "distance": {}, + "duration": {}, + "count": {} + }, "user_input": {}, "additions": [] } }, { "_id": { - "$oid": "64933307ddae82d93a5158e3" + "$oid": "6635321b69e6b3b5d070670a" }, "user_id": { - "$uuid": "4ef98df66c794a4ea4af156986d5e167" + "$uuid": "233a2d2f55184118a8071aa93d92d35a" }, "metadata": { "key": "analysis/confirmed_trip", "platform": "server", - "write_ts": 1687368455.806772, + "write_ts": 1714762267.145938, "time_zone": "America/Los_Angeles", "write_local_dt": { - "year": 2023, - "month": 6, - "day": 21, - "hour": 10, - "minute": 27, - "second": 35, - "weekday": 2, + "year": 2024, + "month": 5, + "day": 3, + "hour": 11, + "minute": 51, + "second": 7, + "weekday": 4, "timezone": "America/Los_Angeles" }, - "write_fmt_time": "2023-06-21T10:27:35.806772-07:00" + "write_fmt_time": "2024-05-03T11:51:07.145938-07:00" }, "data": { "source": "DwellSegmentationTimeFilter", @@ -657,7 +677,7 @@ ] }, "raw_trip": { - "$oid": "64933303ddae82d93a5157e9" + "$oid": "6635321869e6b3b5d0706611" }, "start_ts": 1466462970.2807262, "start_local_dt": { @@ -681,24 +701,24 @@ "duration": 865.4322738647461, "distance": 4521.417177464177, "start_place": { - "$oid": "64933307ddae82d93a5158e2" + "$oid": "6635321b69e6b3b5d0706709" }, "end_place": { - "$oid": "64933307ddae82d93a5158e4" + "$oid": "6635321b69e6b3b5d070670b" }, "cleaned_trip": { - "$oid": "64933305ddae82d93a51585d" + "$oid": "6635321969e6b3b5d0706685" }, "inferred_labels": [], "inferred_trip": { - "$oid": "64933306ddae82d93a5158cc" + "$oid": "6635321a69e6b3b5d07066eb" }, "expectation": { "to_label": true }, "confidence_threshold": 0.55, "expected_trip": { - "$oid": "64933306ddae82d93a5158d7" + "$oid": "6635321a69e6b3b5d07066fe" }, "inferred_section_summary": { "distance": {}, @@ -719,33 +739,38 @@ "ON_FOOT": 1 } }, + "ble_sensed_summary": { + "distance": {}, + "duration": {}, + "count": {} + }, "user_input": {}, "additions": [] } }, { "_id": { - "$oid": "64933308ddae82d93a5158e5" + "$oid": "6635321b69e6b3b5d070670c" }, "user_id": { - "$uuid": "4ef98df66c794a4ea4af156986d5e167" + "$uuid": "233a2d2f55184118a8071aa93d92d35a" }, "metadata": { "key": "analysis/confirmed_trip", "platform": "server", - "write_ts": 1687368456.0238578, + "write_ts": 1714762267.5216131, "time_zone": "America/Los_Angeles", "write_local_dt": { - "year": 2023, - "month": 6, - "day": 21, - "hour": 10, - "minute": 27, - "second": 36, - "weekday": 2, + "year": 2024, + "month": 5, + "day": 3, + "hour": 11, + "minute": 51, + "second": 7, + "weekday": 4, "timezone": "America/Los_Angeles" }, - "write_fmt_time": "2023-06-21T10:27:36.023858-07:00" + "write_fmt_time": "2024-05-03T11:51:07.521613-07:00" }, "data": { "source": "DwellSegmentationTimeFilter", @@ -769,7 +794,7 @@ ] }, "raw_trip": { - "$oid": "64933303ddae82d93a5157ed" + "$oid": "6635321869e6b3b5d0706615" }, "start_ts": 1466466589.3710227, "start_local_dt": { @@ -793,24 +818,24 @@ "duration": 1370.3959772586823, "distance": 5396.705710302932, "start_place": { - "$oid": "64933307ddae82d93a5158e4" + "$oid": "6635321b69e6b3b5d070670b" }, "end_place": { - "$oid": "64933308ddae82d93a5158e6" + "$oid": "6635321b69e6b3b5d070670d" }, "cleaned_trip": { - "$oid": "64933306ddae82d93a515881" + "$oid": "6635321969e6b3b5d07066a9" }, "inferred_labels": [], "inferred_trip": { - "$oid": "64933306ddae82d93a5158cf" + "$oid": "6635321a69e6b3b5d07066ec" }, "expectation": { "to_label": true }, "confidence_threshold": 0.55, "expected_trip": { - "$oid": "64933306ddae82d93a5158d8" + "$oid": "6635321a69e6b3b5d07066ff" }, "inferred_section_summary": { "distance": {}, @@ -831,6 +856,11 @@ "ON_FOOT": 1 } }, + "ble_sensed_summary": { + "distance": {}, + "duration": {}, + "count": {} + }, "user_input": {}, "additions": [] } diff --git a/emission/tests/data/real_examples/shankari_2016-06-20.expected_confirmed_trips.manual_trip_user_input b/emission/tests/data/real_examples/shankari_2016-06-20.expected_confirmed_trips.manual_trip_user_input index 04613a97b..33fa4138f 100644 --- a/emission/tests/data/real_examples/shankari_2016-06-20.expected_confirmed_trips.manual_trip_user_input +++ b/emission/tests/data/real_examples/shankari_2016-06-20.expected_confirmed_trips.manual_trip_user_input @@ -1,27 +1,27 @@ [ { "_id": { - "$oid": "64933296eb9c1ad493f8aeb2" + "$oid": "6635322269e6b3b5d0706812" }, "user_id": { - "$uuid": "577bc4f5d3984d3388e1a974b1a98ecd" + "$uuid": "92066e552ad64057b01455c9187e66a5" }, "metadata": { "key": "analysis/confirmed_trip", "platform": "server", - "write_ts": 1687368342.147376, + "write_ts": 1714762274.819944, "time_zone": "America/Los_Angeles", "write_local_dt": { - "year": 2023, - "month": 6, - "day": 21, - "hour": 10, - "minute": 25, - "second": 42, - "weekday": 2, + "year": 2024, + "month": 5, + "day": 3, + "hour": 11, + "minute": 51, + "second": 14, + "weekday": 4, "timezone": "America/Los_Angeles" }, - "write_fmt_time": "2023-06-21T10:25:42.147376-07:00" + "write_fmt_time": "2024-05-03T11:51:14.819944-07:00" }, "data": { "source": "DwellSegmentationTimeFilter", @@ -45,7 +45,7 @@ ] }, "raw_trip": { - "$oid": "64933292eb9c1ad493f8adb8" + "$oid": "6635322069e6b3b5d0706719" }, "start_ts": 1466436483.395, "start_local_dt": { @@ -69,24 +69,24 @@ "duration": 792.4609999656677, "distance": 1047.1630675866315, "start_place": { - "$oid": "64933295eb9c1ad493f8aeb1" + "$oid": "6635322269e6b3b5d0706811" }, "end_place": { - "$oid": "64933296eb9c1ad493f8aeb3" + "$oid": "6635322269e6b3b5d0706813" }, "cleaned_trip": { - "$oid": "64933294eb9c1ad493f8addb" + "$oid": "6635322169e6b3b5d070673c" }, "inferred_labels": [], "inferred_trip": { - "$oid": "64933295eb9c1ad493f8ae97" + "$oid": "6635322269e6b3b5d07067f7" }, "expectation": { "to_label": true }, "confidence_threshold": 0.55, "expected_trip": { - "$oid": "64933295eb9c1ad493f8aeaa" + "$oid": "6635322269e6b3b5d070680a" }, "inferred_section_summary": { "distance": {}, @@ -104,13 +104,18 @@ "ON_FOOT": 1 } }, + "ble_sensed_summary": { + "distance": {}, + "duration": {}, + "count": {} + }, "user_input": { "trip_user_input": { "_id": { "$oid": "5fd7b7bdf181cd8feb045f29" }, "user_id": { - "$uuid": "577bc4f5d3984d3388e1a974b1a98ecd" + "$uuid": "92066e552ad64057b01455c9187e66a5" }, "metadata": { "key": "manual/trip_user_input", @@ -193,27 +198,27 @@ }, { "_id": { - "$oid": "64933296eb9c1ad493f8aeb4" + "$oid": "6635322369e6b3b5d0706814" }, "user_id": { - "$uuid": "577bc4f5d3984d3388e1a974b1a98ecd" + "$uuid": "92066e552ad64057b01455c9187e66a5" }, "metadata": { "key": "analysis/confirmed_trip", "platform": "server", - "write_ts": 1687368342.327698, + "write_ts": 1714762275.027479, "time_zone": "America/Los_Angeles", "write_local_dt": { - "year": 2023, - "month": 6, - "day": 21, - "hour": 10, - "minute": 25, - "second": 42, - "weekday": 2, + "year": 2024, + "month": 5, + "day": 3, + "hour": 11, + "minute": 51, + "second": 15, + "weekday": 4, "timezone": "America/Los_Angeles" }, - "write_fmt_time": "2023-06-21T10:25:42.327698-07:00" + "write_fmt_time": "2024-05-03T11:51:15.027479-07:00" }, "data": { "source": "DwellSegmentationTimeFilter", @@ -237,7 +242,7 @@ ] }, "raw_trip": { - "$oid": "64933292eb9c1ad493f8adba" + "$oid": "6635322069e6b3b5d070671b" }, "start_ts": 1466437438.6453953, "start_local_dt": { @@ -261,24 +266,24 @@ "duration": 584.3136048316956, "distance": 886.4937093667857, "start_place": { - "$oid": "64933296eb9c1ad493f8aeb3" + "$oid": "6635322269e6b3b5d0706813" }, "end_place": { - "$oid": "64933296eb9c1ad493f8aeb5" + "$oid": "6635322369e6b3b5d0706815" }, "cleaned_trip": { - "$oid": "64933294eb9c1ad493f8adfa" + "$oid": "6635322169e6b3b5d070675b" }, "inferred_labels": [], "inferred_trip": { - "$oid": "64933295eb9c1ad493f8ae9a" + "$oid": "6635322269e6b3b5d07067f8" }, "expectation": { "to_label": true }, "confidence_threshold": 0.55, "expected_trip": { - "$oid": "64933295eb9c1ad493f8aeab" + "$oid": "6635322269e6b3b5d070680b" }, "inferred_section_summary": { "distance": {}, @@ -296,33 +301,38 @@ "ON_FOOT": 1 } }, + "ble_sensed_summary": { + "distance": {}, + "duration": {}, + "count": {} + }, "user_input": {}, "additions": [] } }, { "_id": { - "$oid": "64933296eb9c1ad493f8aeb6" + "$oid": "6635322369e6b3b5d0706816" }, "user_id": { - "$uuid": "577bc4f5d3984d3388e1a974b1a98ecd" + "$uuid": "92066e552ad64057b01455c9187e66a5" }, "metadata": { "key": "analysis/confirmed_trip", "platform": "server", - "write_ts": 1687368342.509429, + "write_ts": 1714762275.196306, "time_zone": "America/Los_Angeles", "write_local_dt": { - "year": 2023, - "month": 6, - "day": 21, - "hour": 10, - "minute": 25, - "second": 42, - "weekday": 2, + "year": 2024, + "month": 5, + "day": 3, + "hour": 11, + "minute": 51, + "second": 15, + "weekday": 4, "timezone": "America/Los_Angeles" }, - "write_fmt_time": "2023-06-21T10:25:42.509429-07:00" + "write_fmt_time": "2024-05-03T11:51:15.196306-07:00" }, "data": { "source": "DwellSegmentationTimeFilter", @@ -346,7 +356,7 @@ ] }, "raw_trip": { - "$oid": "64933292eb9c1ad493f8adbc" + "$oid": "6635322069e6b3b5d070671d" }, "start_ts": 1466461623.1195338, "start_local_dt": { @@ -370,24 +380,24 @@ "duration": 343.25946617126465, "distance": 610.2234223038181, "start_place": { - "$oid": "64933296eb9c1ad493f8aeb5" + "$oid": "6635322369e6b3b5d0706815" }, "end_place": { - "$oid": "64933296eb9c1ad493f8aeb7" + "$oid": "6635322369e6b3b5d0706817" }, "cleaned_trip": { - "$oid": "64933294eb9c1ad493f8ae12" + "$oid": "6635322169e6b3b5d0706773" }, "inferred_labels": [], "inferred_trip": { - "$oid": "64933295eb9c1ad493f8ae9d" + "$oid": "6635322269e6b3b5d07067f9" }, "expectation": { "to_label": true }, "confidence_threshold": 0.55, "expected_trip": { - "$oid": "64933295eb9c1ad493f8aeac" + "$oid": "6635322269e6b3b5d070680c" }, "inferred_section_summary": { "distance": {}, @@ -405,33 +415,38 @@ "ON_FOOT": 1 } }, + "ble_sensed_summary": { + "distance": {}, + "duration": {}, + "count": {} + }, "user_input": {}, "additions": [] } }, { "_id": { - "$oid": "64933296eb9c1ad493f8aeb8" + "$oid": "6635322369e6b3b5d0706818" }, "user_id": { - "$uuid": "577bc4f5d3984d3388e1a974b1a98ecd" + "$uuid": "92066e552ad64057b01455c9187e66a5" }, "metadata": { "key": "analysis/confirmed_trip", "platform": "server", - "write_ts": 1687368342.697634, + "write_ts": 1714762275.365001, "time_zone": "America/Los_Angeles", "write_local_dt": { - "year": 2023, - "month": 6, - "day": 21, - "hour": 10, - "minute": 25, - "second": 42, - "weekday": 2, + "year": 2024, + "month": 5, + "day": 3, + "hour": 11, + "minute": 51, + "second": 15, + "weekday": 4, "timezone": "America/Los_Angeles" }, - "write_fmt_time": "2023-06-21T10:25:42.697634-07:00" + "write_fmt_time": "2024-05-03T11:51:15.365001-07:00" }, "data": { "source": "DwellSegmentationTimeFilter", @@ -455,7 +470,7 @@ ] }, "raw_trip": { - "$oid": "64933292eb9c1ad493f8adbe" + "$oid": "6635322069e6b3b5d070671f" }, "start_ts": 1466462052.158904, "start_local_dt": { @@ -479,24 +494,24 @@ "duration": 400.54909586906433, "distance": 405.97685486691756, "start_place": { - "$oid": "64933296eb9c1ad493f8aeb7" + "$oid": "6635322369e6b3b5d0706817" }, "end_place": { - "$oid": "64933296eb9c1ad493f8aeb9" + "$oid": "6635322369e6b3b5d0706819" }, "cleaned_trip": { - "$oid": "64933294eb9c1ad493f8ae22" + "$oid": "6635322169e6b3b5d0706783" }, "inferred_labels": [], "inferred_trip": { - "$oid": "64933295eb9c1ad493f8aea0" + "$oid": "6635322269e6b3b5d07067fa" }, "expectation": { "to_label": true }, "confidence_threshold": 0.55, "expected_trip": { - "$oid": "64933295eb9c1ad493f8aead" + "$oid": "6635322269e6b3b5d070680d" }, "inferred_section_summary": { "distance": {}, @@ -514,13 +529,18 @@ "ON_FOOT": 1 } }, + "ble_sensed_summary": { + "distance": {}, + "duration": {}, + "count": {} + }, "user_input": { "trip_user_input": { "_id": { "$oid": "5fd7b7bdf181cd8feb045f41" }, "user_id": { - "$uuid": "577bc4f5d3984d3388e1a974b1a98ecd" + "$uuid": "92066e552ad64057b01455c9187e66a5" }, "metadata": { "key": "manual/trip_user_input", @@ -603,27 +623,27 @@ }, { "_id": { - "$oid": "64933296eb9c1ad493f8aeba" + "$oid": "6635322369e6b3b5d070681a" }, "user_id": { - "$uuid": "577bc4f5d3984d3388e1a974b1a98ecd" + "$uuid": "92066e552ad64057b01455c9187e66a5" }, "metadata": { "key": "analysis/confirmed_trip", "platform": "server", - "write_ts": 1687368342.882486, + "write_ts": 1714762275.506772, "time_zone": "America/Los_Angeles", "write_local_dt": { - "year": 2023, - "month": 6, - "day": 21, - "hour": 10, - "minute": 25, - "second": 42, - "weekday": 2, + "year": 2024, + "month": 5, + "day": 3, + "hour": 11, + "minute": 51, + "second": 15, + "weekday": 4, "timezone": "America/Los_Angeles" }, - "write_fmt_time": "2023-06-21T10:25:42.882486-07:00" + "write_fmt_time": "2024-05-03T11:51:15.506772-07:00" }, "data": { "source": "DwellSegmentationTimeFilter", @@ -647,7 +667,7 @@ ] }, "raw_trip": { - "$oid": "64933292eb9c1ad493f8adc0" + "$oid": "6635322069e6b3b5d0706721" }, "start_ts": 1466462970.2807262, "start_local_dt": { @@ -671,24 +691,24 @@ "duration": 865.4322738647461, "distance": 4521.417177464177, "start_place": { - "$oid": "64933296eb9c1ad493f8aeb9" + "$oid": "6635322369e6b3b5d0706819" }, "end_place": { - "$oid": "64933296eb9c1ad493f8aebb" + "$oid": "6635322369e6b3b5d070681b" }, "cleaned_trip": { - "$oid": "64933295eb9c1ad493f8ae34" + "$oid": "6635322269e6b3b5d0706795" }, "inferred_labels": [], "inferred_trip": { - "$oid": "64933295eb9c1ad493f8aea3" + "$oid": "6635322269e6b3b5d07067fb" }, "expectation": { "to_label": true }, "confidence_threshold": 0.55, "expected_trip": { - "$oid": "64933295eb9c1ad493f8aeae" + "$oid": "6635322269e6b3b5d070680e" }, "inferred_section_summary": { "distance": {}, @@ -709,13 +729,18 @@ "ON_FOOT": 1 } }, + "ble_sensed_summary": { + "distance": {}, + "duration": {}, + "count": {} + }, "user_input": { "trip_user_input": { "_id": { "$oid": "5fd7b7bdf181cd8feb045f47" }, "user_id": { - "$uuid": "577bc4f5d3984d3388e1a974b1a98ecd" + "$uuid": "92066e552ad64057b01455c9187e66a5" }, "metadata": { "key": "manual/trip_user_input", @@ -798,27 +823,27 @@ }, { "_id": { - "$oid": "64933297eb9c1ad493f8aebc" + "$oid": "6635322369e6b3b5d070681c" }, "user_id": { - "$uuid": "577bc4f5d3984d3388e1a974b1a98ecd" + "$uuid": "92066e552ad64057b01455c9187e66a5" }, "metadata": { "key": "analysis/confirmed_trip", "platform": "server", - "write_ts": 1687368343.071133, + "write_ts": 1714762275.64923, "time_zone": "America/Los_Angeles", "write_local_dt": { - "year": 2023, - "month": 6, - "day": 21, - "hour": 10, - "minute": 25, - "second": 43, - "weekday": 2, + "year": 2024, + "month": 5, + "day": 3, + "hour": 11, + "minute": 51, + "second": 15, + "weekday": 4, "timezone": "America/Los_Angeles" }, - "write_fmt_time": "2023-06-21T10:25:43.071133-07:00" + "write_fmt_time": "2024-05-03T11:51:15.649230-07:00" }, "data": { "source": "DwellSegmentationTimeFilter", @@ -842,7 +867,7 @@ ] }, "raw_trip": { - "$oid": "64933293eb9c1ad493f8adc4" + "$oid": "6635322069e6b3b5d0706725" }, "start_ts": 1466466589.3710227, "start_local_dt": { @@ -866,24 +891,24 @@ "duration": 1370.3959772586823, "distance": 5396.705710302932, "start_place": { - "$oid": "64933296eb9c1ad493f8aebb" + "$oid": "6635322369e6b3b5d070681b" }, "end_place": { - "$oid": "64933297eb9c1ad493f8aebd" + "$oid": "6635322369e6b3b5d070681d" }, "cleaned_trip": { - "$oid": "64933295eb9c1ad493f8ae58" + "$oid": "6635322269e6b3b5d07067b9" }, "inferred_labels": [], "inferred_trip": { - "$oid": "64933295eb9c1ad493f8aea6" + "$oid": "6635322269e6b3b5d07067fc" }, "expectation": { "to_label": true }, "confidence_threshold": 0.55, "expected_trip": { - "$oid": "64933295eb9c1ad493f8aeaf" + "$oid": "6635322269e6b3b5d070680f" }, "inferred_section_summary": { "distance": {}, @@ -904,13 +929,18 @@ "ON_FOOT": 1 } }, + "ble_sensed_summary": { + "distance": {}, + "duration": {}, + "count": {} + }, "user_input": { "trip_user_input": { "_id": { "$oid": "5fd7b7bdf181cd8feb045f37" }, "user_id": { - "$uuid": "577bc4f5d3984d3388e1a974b1a98ecd" + "$uuid": "92066e552ad64057b01455c9187e66a5" }, "metadata": { "key": "manual/trip_user_input", diff --git a/emission/tests/modellingTests/TestRunGreedyIncrementalModel.py b/emission/tests/modellingTests/TestRunGreedyIncrementalModel.py index 1529f8df5..656011f02 100644 --- a/emission/tests/modellingTests/TestRunGreedyIncrementalModel.py +++ b/emission/tests/modellingTests/TestRunGreedyIncrementalModel.py @@ -32,9 +32,15 @@ def setUp(self): logging.basicConfig( format='%(asctime)s:%(levelname)s:%(message)s', level=logging.DEBUG) + + # read test trips from a test file + input_file = 'emission/tests/data/real_examples/shankari_2016-06-20.expected_confirmed_trips' + with open(input_file, 'r') as f: + test_trips_json = json.load(f, object_hook=esj.wrapped_object_hook) + test_trips = [ecwe.Entry(r) for r in test_trips_json] + logging.debug(f'loaded {len(test_trips)} trips from {input_file}') - # emission/tests/data/real_examples/shankari_2016-06-20.expected_confirmed_trips - self.user_id = uuid.UUID('aa9fdec9-2944-446c-8ee2-50d79b3044d3') + self.user_id = test_trips[0]['user_id'] # all trips within the test file have the same user_id self.ts = esta.TimeSeries.get_time_series(self.user_id) self.new_trips_per_invocation = 3 self.model_type = eamumt.ModelType.GREEDY_SIMILARITY_BINNING @@ -52,15 +58,8 @@ def setUp(self): if len(existing_entries_for_user) != 0: raise Exception(f"test invariant failed, there should be no entries for user {self.user_id}") - # load in trips from a test file source - input_file = 'emission/tests/data/real_examples/shankari_2016-06-20.expected_confirmed_trips' - with open(input_file, 'r') as f: - trips_json = json.load(f, object_hook=esj.wrapped_object_hook) - trips = [ecwe.Entry(r) for r in trips_json] - logging.debug(f'loaded {len(trips)} trips from {input_file}') - self.ts.bulk_insert(trips) - - # confirm write to database succeeded + # write trips to database and confirm that they were written + self.ts.bulk_insert(test_trips) self.initial_data = list(self.ts.find_entries([esdatq.CONFIRMED_TRIP_KEY])) if len(self.initial_data) == 0: logging.debug(f'test setup failed while loading trips from file') diff --git a/setup/environment36.yml b/setup/environment36.yml index 66a5dcc89..d02231754 100644 --- a/setup/environment36.yml +++ b/setup/environment36.yml @@ -19,6 +19,7 @@ dependencies: - scipy=1.10.0 - utm=0.7.0 - pip: + - git+https://github.com/JGreenlee/e-mission-common@0.4.3 - pyfcm==1.5.4 - pygeocoder==1.2.5 - pymongo==4.3.3 diff --git a/setup/tests/Dockerfile b/setup/tests/Dockerfile index 81ecf5081..778c3e650 100644 --- a/setup/tests/Dockerfile +++ b/setup/tests/Dockerfile @@ -3,6 +3,7 @@ FROM ubuntu:latest RUN apt-get update RUN apt-get install -y curl +RUN apt-get install -y git # CHANGEME: Create the files that correspond to your configuration in the conf directory # COPY conf/net/auth/google_auth.json /usr/src/app/conf/net/auth/google_auth.json