Update by adding the individual level AUC calculation

feature_importance/01_run_importance_local_simulations.py

@@ -115,33 +115,66 @@ def compare_estimators(estimators: List[ModelConfig],
 
                 metric_results['fi_scores'] = pd.merge(local_fi_score_summary, support_df, on="var", how="left")
 
-
                 if np.max(support_group1) != np.min(support_group1):
+                    # Compute metrics using the average prediction and the true support
                     for i, (met_name, met) in enumerate(metrics):
                         if met is not None:
                             imp_vals = local_fi_score_group1_mean
                             imp_vals[imp_vals == float("-inf")] = -sys.maxsize - 1
                             imp_vals[imp_vals == float("inf")] = sys.maxsize - 1
                             if fi_est.ascending:
                                 imp_vals[np.isnan(imp_vals)] = -sys.maxsize - 1
-                                metric_results[met_name + "_group1"] = met(support_group1, imp_vals)
+                                metric_results[met_name + "_group1_avg_prediction"] = met(support_group1, imp_vals)
                             else:
                                 imp_vals[np.isnan(imp_vals)] = sys.maxsize - 1
-                                metric_results[met_name+ "_group1"] = met(support_group1, -imp_vals)
-
+                                metric_results[met_name+ "_group1_avg_prediction"] = met(support_group1, -imp_vals)
+
+                    # Compute metrics using the each prediction and the true support then average
+                    for i, (met_name, met) in enumerate(metrics):
+                        if met is not None:
+                            results_group1 = 0
+                            for j in range(n_local_fi_score // 2):
+                                imp_vals = local_fi_score_group1[j]
+                                imp_vals[imp_vals == float("-inf")] = -sys.maxsize - 1
+                                imp_vals[imp_vals == float("inf")] = sys.maxsize - 1
+                                if fi_est.ascending:
+                                    imp_vals[np.isnan(imp_vals)] = -sys.maxsize - 1
+                                    results_group1 += met(support_group1, imp_vals)
+                                else:
+                                    imp_vals[np.isnan(imp_vals)] = sys.maxsize - 1
+                                    results_group1 += met(support_group1, -imp_vals)
+                            metric_results[met_name + "_group1_avg_metric"] = results_group1 / (n_local_fi_score // 2)
 
                 if np.max(support_group2) != np.min(support_group2):
+                    # Compute metrics using the average prediction and the true support
                     for i, (met_name, met) in enumerate(metrics):
                         if met is not None:
                             imp_vals = local_fi_score_group2_mean
                             imp_vals[imp_vals == float("-inf")] = -sys.maxsize - 1
                             imp_vals[imp_vals == float("inf")] = sys.maxsize - 1
                             if fi_est.ascending:
                                 imp_vals[np.isnan(imp_vals)] = -sys.maxsize - 1
-                                metric_results[met_name+ "_group2"] = met(support_group2, imp_vals)
+                                metric_results[met_name+ "_group2_avg_prediction"] = met(support_group2, imp_vals)
                             else:
                                 imp_vals[np.isnan(imp_vals)] = sys.maxsize - 1
-                                metric_results[met_name+ "_group2"] = met(support_group2, -imp_vals)
+                                metric_results[met_name+ "_group2_avg_prediction"] = met(support_group2, -imp_vals)
+
+                    # Compute metrics using the each prediction and the true support then average
+                    for i, (met_name, met) in enumerate(metrics):
+                        if met is not None:
+                            results_group2 = 0
+                            for j in range(n_local_fi_score // 2):
+                                imp_vals = local_fi_score_group2[j]
+                                imp_vals[imp_vals == float("-inf")] = -sys.maxsize - 1
+                                imp_vals[imp_vals == float("inf")] = sys.maxsize - 1
+                                if fi_est.ascending:
+                                    imp_vals[np.isnan(imp_vals)] = -sys.maxsize - 1
+                                    results_group2 += met(support_group2, imp_vals)
+                                else:
+                                    imp_vals[np.isnan(imp_vals)] = sys.maxsize - 1
+                                    results_group2 += met(support_group2, -imp_vals)
+                            metric_results[met_name + "_group2_avg_metric"] = results_group2 / (n_local_fi_score - n_local_fi_score // 2)
+
                 metric_results['time'] = end - start
 
                 # initialize results with metadata and metric results

feature_importance/fi_config/mdi_local/two_subgroups_linear_sims/models.py

@@ -12,8 +12,8 @@
 FI_ESTIMATORS = [
     [FIModelConfig('MDI_all_stumps_with_raw', MDI_local_all_stumps, ascending = False, model_type='tree')],
     [FIModelConfig('MDI_sub_stumps_with_raw', MDI_local_sub_stumps, ascending = False, model_type='tree')],
-    [FIModelConfig('MDI_all_stumps_without_raw', MDI_local_all_stumps, ascending = False, model_type='tree', include_raw=False)],
-    [FIModelConfig('MDI_sub_stumps_without_raw', MDI_local_sub_stumps, ascending = False, model_type='tree', include_raw=False)],
+    [FIModelConfig('MDI_all_stumps_without_raw', MDI_local_all_stumps, ascending = False, model_type='tree', other_params={"include_raw": False})],
+    [FIModelConfig('MDI_sub_stumps_without_raw', MDI_local_sub_stumps, ascending = False, model_type='tree', other_params={"include_raw": False})],
     [FIModelConfig('TreeSHAP', tree_shap_local, model_type='tree')],
     [FIModelConfig('LIME', lime_local, model_type='tree')],
 ]