Add optimal subset results

TODO: MultiPref Experts and Normal
allenai · Sep 25, 2024 · a71bfbd · a71bfbd
1 parent 980c647
commit a71bfbd
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 23 deletions.
diff --git a/results/rewardbench_multipref_feats_best.json b/results/rewardbench_multipref_feats_best.json
@@ -1,9 +1,9 @@
 {
     "MultiPref (Majority)": {
         "optimal": {
-            "score": null,
+            "score": 0.706378476,
             "std": null,
-            "swaps": null
+            "swaps": 1.49
         },
         "human_100": {
             "score": 0.6036541233,
@@ -172,9 +172,9 @@
     },
     "HelpSteer2": {
         "optimal": {
-            "score": null,
+            "score": 0.7970620446,
             "std": null,
-            "swaps": null
+            "swaps": 2.78
         },
         "human_100": {
             "score": 0.7244862062,
@@ -229,9 +229,9 @@
     },
     "ChatArena": {
         "optimal": {
-            "score": null,
+            "score": 0.7227572617,
             "std": null,
-            "swaps": null
+            "swaps": 0.92 
         },
         "human_100": {
             "score": 0.5900274236,
@@ -286,9 +286,9 @@
     },
     "AlpacaFarm": {
         "optimal": {
-            "score": null,
+            "score": 0.6681374886,
             "std": null,
-            "swaps": null
+            "swaps": 2.69 
         },
         "human_100": {
             "score": 0.5506994896,

diff --git a/scripts/plot_results.py b/scripts/plot_results.py
@@ -157,21 +157,22 @@ def plot(ax, dataset: str):
         )
 
         # Plot optimal scores
-        x_opt = data["dataset"]["optimal"].get(
-            "swaps", 0
-        )  # you need to precompute this: swaps / total_counts then interpolate
-        y_opt = data[dataset]["optimal"].get("score", 0) * 100
-
-        # Plot optimal scores
-        ax.plot(
-            x_opt,
-            y_opt,
-            "*",
-            markersize=20,
-            color=COLORS.get("green"),
-            # markeredgecolor=COLORS.get("dark_teal"),
-            label="Optimal subset",
-        )
+        # you need to precompute this: swaps / total_counts then interpolate
+        x_opt = data[dataset]["optimal"].get("swaps")
+        y_opt = data[dataset]["optimal"].get("score")
+
+        if x_opt and y_opt:
+            y_opt = y_opt * 100
+            # Plot optimal scores
+            ax.plot(
+                x_opt,
+                y_opt,
+                "*",
+                markersize=20,
+                color=COLORS.get("green"),
+                # markeredgecolor=COLORS.get("dark_teal"),
+                label="Optimal subset",
+            )
 
         ax.set_xticks(x)
         ax.set_xticklabels(x_levels)