Adding Survey Responses to Public Dashboard

frontend/index.html

@@ -344,6 +344,7 @@
       const end_year = date.getFullYear();
       var current_month = start_month;
       var current_year = start_year;
+
       dates.push([current_month, current_year]);
       while (!(current_month == end_month && current_year == end_year)) {
         current_month += 1;
@@ -356,6 +357,28 @@
       };
       return dates;
     };
+
+    function getDictionaryList(form_list) {
+      var quest_dict = {};
+        return new Promise(async (resolve) => {
+          for (i in form_list) {
+            response = await fetch(form_list[i]);
+            text = await response.text();
+            const parser = new DOMParser();
+            const doc = parser.parseFromString(text, "text/xml");
+            labels = doc.getElementsByTagName("label");
+              for (i in labels) {
+                try {
+                  if ((labels[i].parentNode.getAttribute("appearance") !== "label" && labels[i].parentNode.nodeName != "input")) //label type questions don't ever have answers
+                  {
+                    quest_dict[labels[i].parentNode.getAttribute("ref").split('/').slice(-1)] = labels[i].firstChild.data;
+                  }
+                } catch (e) { }
+              } 
+            }
+          resolve(quest_dict);
+        })
+    };
   </script>
 
   <script type="text/javascript">
@@ -407,7 +430,40 @@
           mode_studied = data.intro.mode_studied
           // Load list of plots corresponding to study/program
           dynamic_labels = data.label_options
-          if (data.intro.program_or_study == 'program') {
+          surveys = data.survey_info.surveys 
+          console.log(data.survey_info['trip-labels'])
+          if (data.survey_info['trip-labels'] === 'ENKETO') { //CASE: SURVEYS
+            survey_list = Object.keys(surveys)
+            survey_list = survey_list.filter(name => name !== 'UserProfileSurvey')
+
+            sheet_list = []
+            for (name in survey_list) {
+              form_path = data.survey_info.surveys[survey_list[name]].formPath;
+              //THIS ASSUMES THE FILENAME IS THE SAME AS THE FORM PATH BUT WITH xml FILE TYPE
+              l_path = form_path.split('.')
+              l_path.splice(l_path.length -1, 1, 'xml');
+              console.log(l_path);
+              sheet_path = l_path.join('.')
+              sheet_list.push(sheet_path)
+            }
+
+            getDictionaryList(sheet_list).then((quest_dict) => {
+              console.log(quest_dict);
+              load_file = "metrics_study_surveys.html"
+              $.get(load_file, function (file) {
+                Object.entries(quest_dict).forEach(([key, value]) => {
+                  var text = '<option ' + 'value="' + key + '" data-sizex="10" data-sizey="4">' + value + '</option>';
+                  file = file.concat('\n', text);
+                });
+                console.log("configuring units");
+                const unitConfigured = file.replaceAll("${data.display_config.use_imperial}", dist_units);
+                $('#metric').append(unitConfigured);
+                addPreconfiguredMetrics(Object.keys(quest_dict).slice(0, 5)); //only adding the first 6 elements
+              });
+            });         
+
+          }
+          else if (data.intro.program_or_study == 'program') { //CASE: PROGRAM
             // Note: We're disabling energy metrics on public dashboard when dynamic labels are available.
             // TODO: Remove the if (data.label_options) in future when energy computation is handled properly.
             if (dynamic_labels) {
@@ -515,7 +571,7 @@
         }
       }).data('gridster');
 
-      $('.js-add-new').on('click', function () {
+      $('.js-add-new').on('click', async function () {
         const metric = $("#metric").val();
         const dateVal = $("#date").val();
         const program = $("#program").val();
@@ -533,10 +589,11 @@
         const htmlFile = "plots/" + metric + "_" + dateVal + program + ".html";
         const altTextFile = "plots/" + metric + "_" + dateVal + program + ".txt";
         const altText = loadFile(altTextFile);
-        const isStackedMetric = ['ntrips_total', 'ntrips_purpose', 'ntrips_under80', 'ntrips_commute_mode_confirm',
-                                 'total_trip_length', 'total_trip_length_land',`ntrips_${mode_studied}_total`,
-                                 `ntrips_${mode_studied}_purpose`,`total_trip_length_${mode_studied}_replaced_mode`]
-                                 .includes(metric);
+        const quest_dict = await getDictionaryList(sheet_list);
+        const stackedMetrics = ['ntrips_total', 'ntrips_total_survey', 'ntrips_purpose', 'ntrips_under80', 'ntrips_under80_survey', 'ntrips_commute_mode_confirm',
+                                 'total_trip_length', 'total_trip_length_land', 'total_trip_length_land_survey',`ntrips_${mode_studied}_total`,
+                                 `ntrips_${mode_studied}_purpose`,`total_trip_length_${mode_studied}_replaced_mode`].concat(Object.keys(quest_dict));
+        const isStackedMetric = stackedMetrics.includes(metric);
         const jsonData = { metric, dateVal, program, metricLabel, dateLabel, programLabel, sizex, sizey };
 
         if (isStackedMetric){

frontend/metrics_study_surveys.html

@@ -0,0 +1,10 @@
+<!-- htmnl options should be 1 per chart question -->
+<!-- <option value="question key" data-sizex="4" data-sizey="4">translated question</option> -->
+
+<option value="ntrips_total_survey" data-sizex="10" data-sizey="4">Number of trips</option>
+<option value="ntrips_under80_survey" data-sizex="10" data-sizey="4">Number of trips (under 80th percentile of total trips)</option>
+<option value="total_trip_length_land_survey" data-sizex="10" data-sizey="4">Total trip length (${data.display_config.use_imperial}) covered by mode in land</option>
+<option value="average_miles_sensed_mode" data-sizex="6" data-sizey="4">Average trip length (${data.display_config.use_imperial}) (sensed)</option>
+<option value="ntrips_sensed_per_day" data-sizex="6" data-sizey="4">Trip frequency (sensed)</option>
+<option value="ntrips_sensed_per_weekday" data-sizex="6" data-sizey="4">Trip frequency (weekday, sensed)</option>
+<option value="ts_users" data-sizex="8" data-sizey="2">Timeseries of active users</option>

viz_scripts/Dockerfile

@@ -1,5 +1,5 @@
 # python 3
-FROM shankari/e-mission-server:master_2024-04-15--53-23
+FROM shankari/e-mission-server:master_2024-05-06--36-33
 
 VOLUME /plots
 

viz_scripts/bin/generate_plots.py

@@ -90,7 +90,8 @@ def compute_for_date(month, year):
         include_test_users=dynamic_config.get('metrics', {}).get('include_test_users', False),
         dynamic_labels = dynamic_labels,
         use_imperial = dynamic_config.get('display_config', {}).get('use_imperial', True),
-        sensed_algo_prefix=dynamic_config.get('metrics', {}).get('sensed_algo_prefix', "cleaned"))
+        sensed_algo_prefix=dynamic_config.get('metrics', {}).get('sensed_algo_prefix', "cleaned"),
+        survey_info = dynamic_config.get('survey_info', {}))
 
     print(f"Running at {arrow.get()} with params {params}")
 

viz_scripts/docker/Dockerfile.dev

@@ -1,5 +1,5 @@
 # python 3
-FROM shankari/e-mission-server:master_2024-04-15--53-23
+FROM shankari/e-mission-server:master_2024-05-06--36-33
 
 VOLUME /plots
 

viz_scripts/docker/crontab

@@ -5,5 +5,7 @@
 0 8 * * * python bin/generate_plots.py mode_specific_metrics.ipynb default >> /var/log/intake.stdinout 2>&1
 0 8 * * * python bin/generate_plots.py mode_specific_timeseries.ipynb default >> /var/log/intake.stdinout 2>&1
 0 8 * * * python bin/generate_plots.py energy_calculations.ipynb default >> /var/log/intake.stdinout 2>&1
+0 8 * * * python bin/generate_plots.py survey_responses.ipynb default >> /var/log/intake.stdinout 2>&1
+0 8 * * * python bin/generate_plots.py survey_metrics.ipynb default >> /var/log/intake.stdinout 2>&1
 # For testing only
 # */5 * * * * python bin/generate_plots.py mode_purpose_share.ipynb default >> /var/log/intake.stdinout 2>&1

viz_scripts/plots.py

@@ -48,13 +48,15 @@ def merge_small_entries(labels, values):
     # We could have already had a non-zero other, and it could be small or large
     if "Other" not in v2l_df.index:
         # zero other will end up with misc_count
-        v2l_df.loc["Other"] = misc_count
+        if misc_count.vals > 0:
+            v2l_df.loc["Other"] = misc_count
     elif "Other" in small_chunk.index:
         # non-zero small other will already be in misc_count
         v2l_df.loc["Other"] = misc_count
     else:
         # non-zero large other, will not already be in misc_count
         v2l_df.loc["Other"] = v2l_df.loc["Other"] + misc_count
+
     disp.display(v2l_df)
 
     return (v2l_df.index.to_list(),v2l_df.vals.to_list(), v2l_df.pct.to_list())
@@ -128,8 +130,13 @@ def plot_and_text_stacked_bar_chart(df, bar_label, ax, text_result, colors, debu
         ax.tick_params(axis='y', labelsize=18)
         ax.tick_params(axis='x', labelsize=18, rotation=90)
         ncols = len(df_only_small)//5 if len(df_only_small) % 5 == 0 else len(df_only_small)//5 + 1
-        ax.legend(bbox_to_anchor=(1, 0), loc='lower left', fancybox=True, shadow=True, fontsize=15)
-        # ax.legend(bbox_to_anchor=(1, 1), loc='upper left', fancybox=True, shadow=True, fontsize=15, ncols=ncols)
+
+        if len(pd.unique(df_only_small['Label'])[0]) > 15:
+            ax.legend(bbox_to_anchor=(0.5, -0.5), loc='upper center', fancybox=True, shadow=True, fontsize=15)
+        else:
+            ax.legend(bbox_to_anchor=(1, 0), loc='lower left', fancybox=True, shadow=True, fontsize=15)
+            # ax.legend(bbox_to_anchor=(1, 1), loc='upper left', fancybox=True, shadow=True, fontsize=15, ncols=ncols)
+
         # Fix for the error: RuntimeError("Unknown return type"), adding the below line to address as mentioned here https://github.com/matplotlib/matplotlib/issues/25625/
         ax.set_xlim(right=ax.get_xlim()[1] + 1.0, auto=True)
         text_result[0], text_result[1] = store_alt_text_and_html_stacked_bar_chart(df_all_entries, bar_label)

viz_scripts/scaffolding.py

@@ -69,6 +69,16 @@ def load_all_participant_trips(program, tq, load_test_users):
     disp.display(participant_ct_df.head())
     return participant_ct_df
 
+def filter_composite_trips(all_comp_trips, program, load_test_users):
+    participant_list = get_participant_uuids(program, load_test_users)
+    # CASE 1 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867
+    if len(all_comp_trips) == 0:
+        return all_comp_trips
+    participant_ct_df = all_comp_trips[all_comp_trips.user_id.isin(participant_list)]
+    print("After filtering, found %s participant trips " % len(participant_ct_df))
+    disp.display(participant_ct_df.head())
+    return participant_ct_df
+
 def filter_labeled_trips(mixed_trip_df):
     # CASE 1 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867
     if len(mixed_trip_df) == 0:
@@ -214,6 +224,20 @@ def mapping_color_labels(dynamic_labels, dic_re, dic_pur):
 
     return colors_mode, colors_purpose, colors_sensed
 
+# Function: Maps survey answers to colors.
+# Input: dictionary of raw and translated survey answers
+# Output: Map for color with survey answers
+def mapping_color_surveys(dic_options):
+    dictionary_values = (list(OrderedDict.fromkeys(dic_options.values())))
+
+    colors = {}
+    for i in range(len(dictionary_values)):
+        colors[dictionary_values[i]] = plt.cm.tab10.colors[i%10]
+
+    colors['Other'] = plt.cm.tab10.colors[(i+1)%10]
+
+    return colors
+
 def load_viz_notebook_sensor_inference_data(year, month, program, include_test_users=False, sensed_algo_prefix="cleaned"):
     """ Inputs:
     year/month/program = parameters from the visualization notebook