From 104dd9a1a0bcbcfc73719d534ddd058ff6e26e54 Mon Sep 17 00:00:00 2001 From: $aTyam Date: Mon, 5 Feb 2024 01:17:52 -0500 Subject: [PATCH] Integration Testing for forest model The changes in this iteration are improvements in test for forest model : 1. Post discussion last week, the regression test was removed ( `TestForestModel.py` )since it won't be useful when model performance improves. Rather, the structures of predictions is checked. This check is merged with TestForestModel.py 2. After https://github.com/e-mission/e-mission-server/pull/944 , `predict_labels_with_n` in `run_model.py` expectes a lists and then iterates over it. The forest model and rest of the tests were updated accordingly. --- .../modelling/trip_model/forest_classifier.py | 10 +- .../tests/modellingTests/TestForestModel.py | 151 ------------------ .../TestForestModelIntegration.py | 94 +++++++++-- .../TestForestModelLoadandSave.py | 8 +- .../modellingTests/TestRunForestModel.py | 9 +- 5 files changed, 100 insertions(+), 172 deletions(-) delete mode 100644 emission/tests/modellingTests/TestForestModel.py diff --git a/emission/analysis/modelling/trip_model/forest_classifier.py b/emission/analysis/modelling/trip_model/forest_classifier.py index a8d1dd2de..8e066775d 100644 --- a/emission/analysis/modelling/trip_model/forest_classifier.py +++ b/emission/analysis/modelling/trip_model/forest_classifier.py @@ -103,13 +103,11 @@ def predict(self, trip: List[float]) -> Tuple[List[Dict], int]: #check if theres no trip to predict logging.debug(f"forest classifier predict called with {len(trip)} trips") if len(trip) == 0: - msg = f'model.predict cannot be called with an empty trips' + msg = f'model.predict cannot be called with an empty trip' raise Exception(msg) - # CONVERT LIST OF TRIPS TO dataFrame - test_df = estb.BuiltinTimeSeries.to_data_df("analysis/confirmed_trip",trip) - labeled_trip_df = esdtq.filter_labeled_trips(test_df) - expanded_labeled_trip_df= esdtq.expand_userinputs(labeled_trip_df) - predcitions_df= self.model.predict(expanded_labeled_trip_df) + # CONVERT TRIP TO dataFrame + test_df = estb.BuiltinTimeSeries.to_data_df("analysis/confirmed_trip",[trip]) + predcitions_df= self.model.predict(test_df) # the predictions_df currently holds the highest probable options # individually in all three categories. the predictions_df are in the form diff --git a/emission/tests/modellingTests/TestForestModel.py b/emission/tests/modellingTests/TestForestModel.py deleted file mode 100644 index 07a52aafe..000000000 --- a/emission/tests/modellingTests/TestForestModel.py +++ /dev/null @@ -1,151 +0,0 @@ -import unittest -import logging -import numpy as np -import uuid -import json -import os - -import emission.analysis.modelling.trip_model.run_model as eamur -import emission.analysis.modelling.trip_model.model_type as eamumt -import emission.analysis.modelling.trip_model.model_storage as eamums -import emission.storage.json_wrappers as esj -import emission.storage.timeseries.abstract_timeseries as esta -import emission.tests.modellingTests.modellingTestAssets as etmm -import emission.storage.decorations.analysis_timeseries_queries as esda -import emission.core.get_database as edb -import emission.core.wrapper.entry as ecwe -import emission.storage.decorations.analysis_timeseries_queries as esdatq - -class TestForestModel(unittest.TestCase): - - def setUp(self): - """ - sets up the end-to-end run model test with Confirmedtrip data - """ - logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', - level=logging.DEBUG) - - self.user_id = uuid.UUID('aa9fdec9-2944-446c-8ee2-50d79b3044d3') - self.ts = esta.TimeSeries.get_time_series(self.user_id) - self.new_trips_per_invocation = 3 - self.model_type = eamumt.ModelType.RANDOM_FOREST_CLASSIFIER - self.model_storage = eamums.ModelStorage.DOCUMENT_DATABASE - sim_threshold = 500 # meters - self.forest_model_config= { - "loc_feature" : "coordinates", - "radius": 500, - "size_thresh":1, - "purity_thresh":1.0, - "gamma":0.05, - "C":1, - "n_estimators":100, - "criterion":"gini", - "max_depth":'null', - "min_samples_split":2, - "min_samples_leaf":1, - "max_features":"sqrt", - "bootstrap":True, - "random_state":42, - "use_start_clusters":False, - "use_trip_clusters":True - } - - existing_entries_for_user = list(self.ts.find_entries([esdatq.CONFIRMED_TRIP_KEY])) - if len(existing_entries_for_user) != 0: - raise Exception(f"test invariant failed, there should be no entries for user {self.user_id}") - - # load in trips from a test file source - input_file = 'emission/tests/data/real_examples/shankari_2016-06-20.expected_confirmed_trips' - with open(input_file, 'r') as f: - trips_json = json.load(f, object_hook=esj.wrapped_object_hook) - self.trips = [ecwe.Entry(r) for r in trips_json] - logging.debug(f'loaded {len(self.trips)} trips from {input_file}') - - def tearDown(self): - """ - clean up database - """ - edb.get_analysis_timeseries_db().delete_many({'user_id': self.user_id}) - edb.get_model_db().delete_many({'user_id': self.user_id}) - edb.get_pipeline_state_db().delete_many({'user_id': self.user_id}) - - - - def testRandomForestRegression(self): - """ - test to ensure consistent model results. Load data for a user from json, split - into train and test. After training, we generate predictions and match them with - predictions from last time. If the code is run for the first time, the current predicitons - will be stored as ground truth. - """ - file_path= 'emission/tests/modellingTests/data.json' - split=int(0.9*len(self.trips)) - train_data= self.trips[:split] - - self.ts.bulk_insert(train_data) - - # confirm write to database succeeded - self.initial_data = list(self.ts.find_entries([esdatq.CONFIRMED_TRIP_KEY])) - if len(self.initial_data) == 0: - logging.debug(f'Writing train data failed') - self.fail() - - test_data=self.trips[split:] - logging.debug(f'LENDATA{len(train_data),len(test_data)}') - eamur.update_trip_model( - user_id=self.user_id, - model_type=eamumt.ModelType.RANDOM_FOREST_CLASSIFIER, - model_storage=eamums.ModelStorage.DOCUMENT_DATABASE, - min_trips=4, - model_config=self.forest_model_config - ) - model = eamur._load_stored_trip_model( - user_id=self.user_id, - model_type=eamumt.ModelType.RANDOM_FOREST_CLASSIFIER, - model_storage=eamums.ModelStorage.DOCUMENT_DATABASE, - model_config=self.forest_model_config - ) - - curr_predictions_list = eamur.predict_labels_with_n( - trip_list = [test_data], - model=model - ) - - - ## predictions take the form like : - # - #{'labels': {'mode_confirm': 'ebike', 'replaced_mode': 'walk', 'purpose_confirm': 'dog-park'}, 'p': 1.0} - - #Below are two ways we can store prev. predictions list . Whichever way we finalise, I'll delete the other one. - # - #Method 1 : Run predictions for the first time and hardcode them into - #prev_prdictions_list. For every iteration, simply compare them - # - # for the current data that we read from json, the predictions we get is an empty list. If - # we use a different file with more data, this'll take the for as mentioned above - # - prev_predictions_list= [ - ( - [], - -1 - ) - ] - - self.assertEqual(prev_predictions_list,curr_predictions_list," previous predictions should match current predictions") - - - #Method 2 ( which was failing): Store these predictions into a json and read from - #that json - # - # try: - # if os.path.exists(file_path) and os.path.getsize(file_path)>0: - # with open(file_path, 'r') as f: - # prev_predictions_list = json.load(f) - # logging.debug() - # self.assertEqual(prev_predictions_list,curr_predictions_list," previous predictions should match current predictions") - # else: - # with open(file_path,'w') as file: - # json.dump(curr_predictions_list,file,indent=4) - # logging.debug("Previous predicitons stored for future matching" ) - # except json.JSONDecodeError: - # logging.debug("jsonDecodeErrorError") \ No newline at end of file diff --git a/emission/tests/modellingTests/TestForestModelIntegration.py b/emission/tests/modellingTests/TestForestModelIntegration.py index 90c6fd13b..89d0a639d 100644 --- a/emission/tests/modellingTests/TestForestModelIntegration.py +++ b/emission/tests/modellingTests/TestForestModelIntegration.py @@ -13,16 +13,87 @@ import emission.pipeline.intake_stage as epi import logging -class TestLabelInferencePipeline(unittest.TestCase): - # It is important that these functions be deterministic - +import emission.analysis.modelling.trip_model.run_model as eamur +import emission.analysis.modelling.trip_model.model_type as eamumt +import emission.analysis.modelling.trip_model.model_storage as eamums +import emission.tests.modellingTests.modellingTestAssets as etmm +import emission.storage.timeseries.abstract_timeseries as esta + + +class TestForestModelIntegration(unittest.TestCase): + # Test if the forest model for label prediction is smoothly integrated with the inference pipeline. + # In the initial setup, build a dummy forest model. Then run the pipeline on real example data. + # Finally in the test, assert the type of label predictions expected. def setUp(self): self.reset_all() np.random.seed(91) self.test_algorithms = eacilp.primary_algorithms + + forest_model_config= { + "loc_feature" : "coordinates", + "radius": 500, + "size_thresh":1, + "purity_thresh":1.0, + "gamma":0.05, + "C":1, + "n_estimators":100, + "criterion":"gini", + "max_depth":'null', + "min_samples_split":2, + "min_samples_leaf":1, + "max_features":"sqrt", + "bootstrap":True, + "random_state":42, + "use_start_clusters":False, + "use_trip_clusters":True + } etc.setupRealExample(self, "emission/tests/data/real_examples/shankari_2015-07-22") ##maybe use a different file + ts = esta.TimeSeries.get_time_series(self.testUUID) + label_data = { + "mode_confirm": ['ebike', 'bike'], + "purpose_confirm": ['happy-hour', 'dog-park'], + "replaced_mode": ['walk'], + "mode_weights": [0.9, 0.1], + "purpose_weights": [0.1, 0.9] + } + + self.origin = (-105.1705977, 39.7402654,) + self.destination = (-105.1755606, 39.7673075) + self.min_trips = 14 + self.total_trips = 100 + self.clustered_trips = 33 + self.has_label_percent = 0.9 + ## generate mock trips + train = etmm.generate_mock_trips( + user_id=self.testUUID, + trips=self.total_trips, + origin=self.origin, + destination=self.destination, + trip_part='od', + label_data=label_data, + within_threshold=self.clustered_trips, + threshold=0.004, # ~400m + has_label_p=self.has_label_percent + ) + ts.bulk_insert(train) + # confirm data write did not fail + check_data = esda.get_entries(key="analysis/confirmed_trip", user_id=self.testUUID, time_query=None) + if len(check_data) != self.total_trips: + logging.debug(f'test invariant failed after generating test data') + self.fail() + else: + logging.debug(f'found {self.total_trips} trips in database') + ## Build an already existing model or a new model + eamur.update_trip_model( + user_id=self.testUUID, + model_type=eamumt.ModelType.RANDOM_FOREST_CLASSIFIER, + model_storage=eamums.ModelStorage.DOCUMENT_DATABASE, + min_trips=4, + model_config=forest_model_config + ) + ## run inference pipeline self.run_pipeline(self.test_algorithms) time_range = estt.TimeQuery("metadata.write_ts", None, time.time()) self.inferred_trips = esda.get_entries(esda.INFERRED_TRIP_KEY, self.testUUID, time_query=time_range) @@ -39,16 +110,19 @@ def run_pipeline(self, algorithms): def reset_all(self): etc.dropAllCollections(edb._get_current_db()) - # Tests that algorithm being tested runs and saves to the database correctly - def testIndividualAlgorithms(self): - logging.debug('TEST1') + # Tests that forest algorithm being tested runs successfully + def testForestAlgorithm(self): for trip in self.inferred_trips: entries = esdt.get_sections_for_trip("inference/labels", self.testUUID, trip.get_id()) - logging.debug(f"ENTRIES: {entries}") self.assertEqual(len(entries), len(self.test_algorithms)) - # for entry in entries: - # self.assertGreater(len(entry["data"]["prediction"]), 0) - + for entry in entries: + self.assertGreater(len(entry["data"]["prediction"]), 0) + for singleprediction in entry["data"]["prediction"]: + self.assertIsInstance(singleprediction, dict, " should be an instance of the dictionary class") + self.assertIsInstance(singleprediction['labels'], dict, " should be an instance of the dictionary class") + self.assertIn('mode_confirm',singleprediction['labels'].keys()) + self.assertIn('replaced_mode',singleprediction['labels'].keys()) + self.assertIn('purpose_confirm',singleprediction['labels'].keys()) def main(): etc.configLogging() diff --git a/emission/tests/modellingTests/TestForestModelLoadandSave.py b/emission/tests/modellingTests/TestForestModelLoadandSave.py index e7d5491b9..8da1fce5b 100644 --- a/emission/tests/modellingTests/TestForestModelLoadandSave.py +++ b/emission/tests/modellingTests/TestForestModelLoadandSave.py @@ -134,7 +134,7 @@ def testForestModelRoundTrip(self): # logging.debug(f'Predictions on trips in database') predictions_list = eamur.predict_labels_with_n( - trip_list = [test], + trip_list = test, model=model ) @@ -151,7 +151,7 @@ def testForestModelRoundTrip(self): # logging.debug(f'Predictions on trips using deserialised model') predictions_loaded_model_list = eamur.predict_labels_with_n( - trip_list = [test], + trip_list = test, model=deserialized_model ) # logging.debug(f'Assert that both predictions are the same') @@ -184,7 +184,7 @@ def testForestModelConsistency(self): # logging.debug(f' Model Predictions on trips in database') predictions_list_model1 = eamur.predict_labels_with_n( - trip_list = [test], + trip_list = test, model=model_iter1 ) # logging.debug(f' Loading Model again') @@ -197,7 +197,7 @@ def testForestModelConsistency(self): ) # logging.debug(f' Model Predictions on trips in database') predictions_list_model2 = eamur.predict_labels_with_n( - trip_list = [test], + trip_list = test, model=model_iter2 ) diff --git a/emission/tests/modellingTests/TestRunForestModel.py b/emission/tests/modellingTests/TestRunForestModel.py index 2ca48c4f4..1676e878d 100644 --- a/emission/tests/modellingTests/TestRunForestModel.py +++ b/emission/tests/modellingTests/TestRunForestModel.py @@ -183,9 +183,16 @@ def test1RoundPredictForestModel(self): ) predictions_list = eamur.predict_labels_with_n( - trip_list = [test], + trip_list = test, model=model ) for prediction, n in predictions_list: [logging.debug(p) for p in sorted(prediction, key=lambda r: r['p'], reverse=True)] self.assertNotEqual(len(prediction), 0, "should have a prediction") + self.assertIn('labels',prediction[0].keys()) + self.assertIn('p',prediction[0].keys()) + self.assertIsInstance(prediction[0], dict, " should be an instance of the dictionary class") + self.assertIsInstance(prediction[0]['labels'], dict, " should be an instance of the dictionary class") + self.assertIn('mode_confirm',prediction[0]['labels'].keys()) + self.assertIn('replaced_mode',prediction[0]['labels'].keys()) + self.assertIn('purpose_confirm',prediction[0]['labels'].keys()) \ No newline at end of file