Hugging Face's logo

chanind
/
sae-training-snapshots

Model card Files
xet
 Community
chanind commited on
Commit
97f2193
·
verified ·
1 Parent(s): 44dd9c1

Add files using upload-large-folder tool

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. gemma-2-2b-btk-mat/k-100/seed-0/snapshots/step-317382-tokens-1300000000/tpp/saebench_tpp_custom_sae_eval_results.json +263 -139
  2. gemma-2-2b-btk-mat/k-100/seed-0/snapshots/step-48828-tokens-200000000/tpp/saebench_tpp_custom_sae_eval_results.json +265 -141
  3. gemma-2-2b-btk-mat/k-100/seed-0/snapshots/step-61035-tokens-250000000/tpp/saebench_tpp_custom_sae_eval_results.json +266 -142
  4. gemma-2-2b-btk/k-100/seed-0/snapshots/step-1-tokens-0/tpp/saebench_tpp_custom_sae_eval_results.json +270 -146
  5. gemma-2-2b-btk/k-100/seed-0/snapshots/step-109863-tokens-450000000/scr/saebench_scr_custom_sae_eval_results.json +115 -111
  6. gemma-2-2b-btk/k-100/seed-0/snapshots/step-109863-tokens-450000000/tpp/saebench_tpp_custom_sae_eval_results.json +266 -142
  7. gemma-2-2b-btk/k-100/seed-0/snapshots/step-12207-tokens-50000000/scr/saebench_scr_custom_sae_eval_results.json +115 -111
  8. gemma-2-2b-btk/k-100/seed-0/snapshots/step-12207-tokens-50000000/tpp/saebench_tpp_custom_sae_eval_results.json +269 -145
  9. gemma-2-2b-btk/k-100/seed-0/snapshots/step-122070-tokens-500000000/scr/saebench_scr_custom_sae_eval_results.json +115 -111
  10. gemma-2-2b-btk/k-100/seed-0/snapshots/step-122070-tokens-500000000/tpp/saebench_tpp_custom_sae_eval_results.json +265 -141
  11. gemma-2-2b-btk/k-100/seed-0/snapshots/step-134277-tokens-550000000/scr/saebench_scr_custom_sae_eval_results.json +119 -115
  12. gemma-2-2b-btk/k-100/seed-0/snapshots/step-134277-tokens-550000000/tpp/saebench_tpp_custom_sae_eval_results.json +263 -139
  13. gemma-2-2b-btk/k-100/seed-0/snapshots/step-146484-tokens-600000000/scr/saebench_scr_custom_sae_eval_results.json +115 -111
  14. gemma-2-2b-btk/k-100/seed-0/snapshots/step-146484-tokens-600000000/tpp/saebench_tpp_custom_sae_eval_results.json +269 -145
  15. gemma-2-2b-btk/k-100/seed-0/snapshots/step-158691-tokens-650000000/scr/saebench_scr_custom_sae_eval_results.json +115 -111
  16. gemma-2-2b-btk/k-100/seed-0/snapshots/step-158691-tokens-650000000/tpp/saebench_tpp_custom_sae_eval_results.json +266 -142
  17. gemma-2-2b-btk/k-100/seed-0/snapshots/step-170898-tokens-700000000/scr/saebench_scr_custom_sae_eval_results.json +115 -111
  18. gemma-2-2b-btk/k-100/seed-0/snapshots/step-170898-tokens-700000000/tpp/saebench_tpp_custom_sae_eval_results.json +267 -143
  19. gemma-2-2b-btk/k-100/seed-0/snapshots/step-183105-tokens-750000000/scr/saebench_scr_custom_sae_eval_results.json +115 -111
  20. gemma-2-2b-btk/k-100/seed-0/snapshots/step-183105-tokens-750000000/tpp/saebench_tpp_custom_sae_eval_results.json +266 -142
  21. gemma-2-2b-btk/k-100/seed-0/snapshots/step-195312-tokens-800000000/scr/saebench_scr_custom_sae_eval_results.json +115 -111
  22. gemma-2-2b-btk/k-100/seed-0/snapshots/step-195312-tokens-800000000/tpp/saebench_tpp_custom_sae_eval_results.json +266 -142
  23. gemma-2-2b-btk/k-100/seed-0/snapshots/step-207519-tokens-850000000/scr/saebench_scr_custom_sae_eval_results.json +115 -111
  24. gemma-2-2b-btk/k-100/seed-0/snapshots/step-207519-tokens-850000000/tpp/saebench_tpp_custom_sae_eval_results.json +266 -142
  25. gemma-2-2b-btk/k-100/seed-0/snapshots/step-219726-tokens-900000000/scr/saebench_scr_custom_sae_eval_results.json +115 -111
  26. gemma-2-2b-btk/k-100/seed-0/snapshots/step-219726-tokens-900000000/tpp/saebench_tpp_custom_sae_eval_results.json +265 -141
  27. gemma-2-2b-btk/k-100/seed-0/snapshots/step-231933-tokens-950000000/scr/saebench_scr_custom_sae_eval_results.json +115 -111
  28. gemma-2-2b-btk/k-100/seed-0/snapshots/step-231933-tokens-950000000/tpp/saebench_tpp_custom_sae_eval_results.json +264 -140
  29. gemma-2-2b-btk/k-100/seed-0/snapshots/step-2441-tokens-10000000/scr/saebench_scr_custom_sae_eval_results.json +115 -111
  30. gemma-2-2b-btk/k-100/seed-0/snapshots/step-2441-tokens-10000000/tpp/saebench_tpp_custom_sae_eval_results.json +269 -145
  31. gemma-2-2b-btk/k-100/seed-0/snapshots/step-24414-tokens-100000000/scr/saebench_scr_custom_sae_eval_results.json +115 -111
  32. gemma-2-2b-btk/k-100/seed-0/snapshots/step-24414-tokens-100000000/tpp/saebench_tpp_custom_sae_eval_results.json +271 -147
  33. gemma-2-2b-btk/k-100/seed-0/snapshots/step-244140-tokens-1000000000/scr/saebench_scr_custom_sae_eval_results.json +115 -111
  34. gemma-2-2b-btk/k-100/seed-0/snapshots/step-244140-tokens-1000000000/tpp/saebench_tpp_custom_sae_eval_results.json +265 -141
  35. gemma-2-2b-btk/k-100/seed-0/snapshots/step-268554-tokens-1100000000/scr/saebench_scr_custom_sae_eval_results.json +115 -111
  36. gemma-2-2b-btk/k-100/seed-0/snapshots/step-268554-tokens-1100000000/tpp/saebench_tpp_custom_sae_eval_results.json +265 -141
  37. gemma-2-2b-btk/k-100/seed-0/snapshots/step-292968-tokens-1200000000/scr/saebench_scr_custom_sae_eval_results.json +115 -111
  38. gemma-2-2b-btk/k-100/seed-0/snapshots/step-292968-tokens-1200000000/tpp/saebench_tpp_custom_sae_eval_results.json +264 -140
  39. gemma-2-2b-btk/k-100/seed-0/snapshots/step-317382-tokens-1300000000/scr/saebench_scr_custom_sae_eval_results.json +115 -111
  40. gemma-2-2b-btk/k-100/seed-0/snapshots/step-317382-tokens-1300000000/tpp/saebench_tpp_custom_sae_eval_results.json +267 -143
  41. gemma-2-2b-btk/k-100/seed-0/snapshots/step-341796-tokens-1400000000/scr/saebench_scr_custom_sae_eval_results.json +115 -111
  42. gemma-2-2b-btk/k-100/seed-0/snapshots/step-341796-tokens-1400000000/tpp/saebench_tpp_custom_sae_eval_results.json +263 -139
  43. gemma-2-2b-btk/k-100/seed-0/snapshots/step-36621-tokens-150000000/scr/saebench_scr_custom_sae_eval_results.json +115 -111
  44. gemma-2-2b-btk/k-100/seed-0/snapshots/step-36621-tokens-150000000/tpp/saebench_tpp_custom_sae_eval_results.json +270 -146
  45. gemma-2-2b-btk/k-100/seed-0/snapshots/step-366210-tokens-1500000000/scr/saebench_scr_custom_sae_eval_results.json +115 -111
  46. gemma-2-2b-btk/k-100/seed-0/snapshots/step-366210-tokens-1500000000/tpp/saebench_tpp_custom_sae_eval_results.json +264 -140
  47. gemma-2-2b-btk/k-100/seed-0/snapshots/step-48828-tokens-200000000/scr/saebench_scr_custom_sae_eval_results.json +115 -111
  48. gemma-2-2b-btk/k-100/seed-0/snapshots/step-48828-tokens-200000000/tpp/saebench_tpp_custom_sae_eval_results.json +270 -146
  49. gemma-2-2b-btk/k-100/seed-0/snapshots/step-6103-tokens-25000000/scr/saebench_scr_custom_sae_eval_results.json +115 -111
  50. gemma-2-2b-btk/k-100/seed-0/snapshots/step-6103-tokens-25000000/tpp/saebench_tpp_custom_sae_eval_results.json +269 -145
gemma-2-2b-btk-mat/k-100/seed-0/snapshots/step-317382-tokens-1300000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "52cd9c67-1bb8-4261-8090-dac372146614",
69
- "datetime_epoch_millis": 1776881466884,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.04565000385046005,
73
- "tpp_threshold_2_intended_diff_only": 0.057700008153915405,
74
- "tpp_threshold_2_unintended_diff_only": 0.012050004303455352,
75
- "tpp_threshold_5_total_metric": 0.10462500154972076,
76
- "tpp_threshold_5_intended_diff_only": 0.13940000534057617,
77
- "tpp_threshold_5_unintended_diff_only": 0.03477500379085541,
78
- "tpp_threshold_10_total_metric": 0.1483999952673912,
79
- "tpp_threshold_10_intended_diff_only": 0.21450000405311584,
80
- "tpp_threshold_10_unintended_diff_only": 0.06610000878572464,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.07639999985694886,
99
- "tpp_threshold_2_intended_diff_only": 0.09900001287460328,
100
- "tpp_threshold_2_unintended_diff_only": 0.022600013017654418,
101
- "tpp_threshold_5_total_metric": 0.17915000319480895,
102
- "tpp_threshold_5_intended_diff_only": 0.24400001764297485,
103
- "tpp_threshold_5_unintended_diff_only": 0.06485001444816589,
104
- "tpp_threshold_10_total_metric": 0.2423500031232834,
105
  "tpp_threshold_10_intended_diff_only": 0.3666000247001648,
106
- "tpp_threshold_10_unintended_diff_only": 0.1242500215768814,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.014900007843971252,
123
- "tpp_threshold_2_intended_diff_only": 0.01640000343322754,
124
- "tpp_threshold_2_unintended_diff_only": 0.0014999955892562866,
125
- "tpp_threshold_5_total_metric": 0.03009999990463257,
126
- "tpp_threshold_5_intended_diff_only": 0.03479999303817749,
127
- "tpp_threshold_5_unintended_diff_only": 0.004699993133544922,
128
- "tpp_threshold_10_total_metric": 0.05444998741149902,
129
- "tpp_threshold_10_intended_diff_only": 0.062399983406066895,
130
- "tpp_threshold_10_unintended_diff_only": 0.00794999599456787,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.10175003111362457,
190
  "tpp_threshold_10_total_metric": 0.21299993991851807,
191
  "tpp_threshold_10_intended_diff_only": 0.3489999771118164,
192
- "tpp_threshold_10_unintended_diff_only": 0.13600003719329834
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": 0.12025000154972076,
196
- "tpp_threshold_2_intended_diff_only": 0.1600000262260437,
197
- "tpp_threshold_2_unintended_diff_only": 0.03975002467632294,
198
- "tpp_threshold_5_total_metric": 0.1772499680519104,
199
  "tpp_threshold_5_intended_diff_only": 0.22699999809265137,
200
- "tpp_threshold_5_unintended_diff_only": 0.04975003004074097,
201
- "tpp_threshold_10_total_metric": 0.2952499985694885,
202
  "tpp_threshold_10_intended_diff_only": 0.35100001096725464,
203
- "tpp_threshold_10_unintended_diff_only": 0.05575001239776611
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.02124999463558197,
207
- "tpp_threshold_2_intended_diff_only": 0.018000006675720215,
208
- "tpp_threshold_2_unintended_diff_only": -0.0032499879598617554,
209
- "tpp_threshold_5_total_metric": 0.1820000559091568,
210
  "tpp_threshold_5_intended_diff_only": 0.29600006341934204,
211
- "tpp_threshold_5_unintended_diff_only": 0.11400000751018524,
212
- "tpp_threshold_10_total_metric": 0.16375002264976501,
213
  "tpp_threshold_10_intended_diff_only": 0.3400000333786011,
214
- "tpp_threshold_10_unintended_diff_only": 0.17625001072883606
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.04650002717971802,
218
- "tpp_threshold_2_intended_diff_only": 0.04900002479553223,
219
- "tpp_threshold_2_unintended_diff_only": 0.002499997615814209,
220
- "tpp_threshold_5_total_metric": 0.1340000033378601,
221
- "tpp_threshold_5_intended_diff_only": 0.15299999713897705,
222
- "tpp_threshold_5_unintended_diff_only": 0.018999993801116943,
223
- "tpp_threshold_10_total_metric": 0.343500018119812,
224
  "tpp_threshold_10_intended_diff_only": 0.437000036239624,
225
- "tpp_threshold_10_unintended_diff_only": 0.09350001811981201
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.009750008583068848,
229
- "tpp_threshold_2_intended_diff_only": 0.012000024318695068,
230
- "tpp_threshold_2_unintended_diff_only": 0.0022500157356262207,
231
- "tpp_threshold_5_total_metric": 0.19025000929832458,
232
  "tpp_threshold_5_intended_diff_only": 0.23000001907348633,
233
- "tpp_threshold_5_unintended_diff_only": 0.03975000977516174,
234
- "tpp_threshold_10_total_metric": 0.1962500363588333,
235
  "tpp_threshold_10_intended_diff_only": 0.35600006580352783,
236
- "tpp_threshold_10_unintended_diff_only": 0.15975002944469452
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.008000001311302185,
242
- "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
243
- "tpp_threshold_2_unintended_diff_only": -0.003999993205070496,
244
- "tpp_threshold_5_total_metric": 0.012750014662742615,
245
- "tpp_threshold_5_intended_diff_only": 0.013000011444091797,
246
- "tpp_threshold_5_unintended_diff_only": 0.00024999678134918213,
247
- "tpp_threshold_10_total_metric": 0.011250004172325134,
248
  "tpp_threshold_10_intended_diff_only": 0.018999993801116943,
249
- "tpp_threshold_10_unintended_diff_only": 0.007749989628791809
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": 0.006500020623207092,
253
- "tpp_threshold_2_intended_diff_only": 0.013000011444091797,
254
- "tpp_threshold_2_unintended_diff_only": 0.006499990820884705,
255
- "tpp_threshold_5_total_metric": 0.02000005543231964,
256
- "tpp_threshold_5_intended_diff_only": 0.030000030994415283,
257
- "tpp_threshold_5_unintended_diff_only": 0.009999975562095642,
258
- "tpp_threshold_10_total_metric": 0.07975001633167267,
259
- "tpp_threshold_10_intended_diff_only": 0.0910000205039978,
260
- "tpp_threshold_10_unintended_diff_only": 0.011250004172325134
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.022750020027160645,
264
- "tpp_threshold_2_intended_diff_only": 0.018000006675720215,
265
- "tpp_threshold_2_unintended_diff_only": -0.00475001335144043,
266
- "tpp_threshold_5_total_metric": 0.02399998903274536,
267
- "tpp_threshold_5_intended_diff_only": 0.02499997615814209,
268
- "tpp_threshold_5_unintended_diff_only": 0.0009999871253967285,
269
- "tpp_threshold_10_total_metric": 0.02724999189376831,
270
- "tpp_threshold_10_intended_diff_only": 0.0339999794960022,
271
- "tpp_threshold_10_unintended_diff_only": 0.006749987602233887
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": 0.0014999955892562866,
275
- "tpp_threshold_2_intended_diff_only": 0.009999990463256836,
276
- "tpp_threshold_2_unintended_diff_only": 0.00849999487400055,
277
- "tpp_threshold_5_total_metric": 0.01349996030330658,
278
- "tpp_threshold_5_intended_diff_only": 0.01699995994567871,
279
- "tpp_threshold_5_unintended_diff_only": 0.0034999996423721313,
280
- "tpp_threshold_10_total_metric": 0.030249953269958496,
281
- "tpp_threshold_10_intended_diff_only": 0.035999953746795654,
282
- "tpp_threshold_10_unintended_diff_only": 0.005750000476837158
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.035750001668930054,
286
- "tpp_threshold_2_intended_diff_only": 0.03700000047683716,
287
- "tpp_threshold_2_unintended_diff_only": 0.0012499988079071045,
288
- "tpp_threshold_5_total_metric": 0.08024998009204865,
289
- "tpp_threshold_5_intended_diff_only": 0.08899998664855957,
290
- "tpp_threshold_5_unintended_diff_only": 0.008750006556510925,
291
- "tpp_threshold_10_total_metric": 0.12374997138977051,
292
- "tpp_threshold_10_intended_diff_only": 0.13199996948242188,
293
- "tpp_threshold_10_unintended_diff_only": 0.008249998092651367
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "8fce6eea-45ea-4c59-8c5a-6af79f7a8d8d",
73
+ "datetime_epoch_millis": 1777314582341,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.04417499750852585,
77
+ "tpp_threshold_2_intended_diff_only": 0.05649999976158142,
78
+ "tpp_threshold_2_unintended_diff_only": 0.012325002253055573,
79
+ "tpp_threshold_5_total_metric": 0.10682500898838043,
80
+ "tpp_threshold_5_intended_diff_only": 0.13920001387596131,
81
+ "tpp_threshold_5_unintended_diff_only": 0.03237500488758087,
82
+ "tpp_threshold_10_total_metric": 0.15090001076459886,
83
+ "tpp_threshold_10_intended_diff_only": 0.21670001745224,
84
+ "tpp_threshold_10_unintended_diff_only": 0.06580000668764115,
85
+ "tpp_threshold_20_total_metric": 0.1770750045776367,
86
+ "tpp_threshold_20_intended_diff_only": 0.2630000114440918,
87
+ "tpp_threshold_20_unintended_diff_only": 0.08592500686645507,
88
+ "tpp_threshold_50_total_metric": 0.21272501796483995,
89
+ "tpp_threshold_50_intended_diff_only": 0.3382000267505646,
90
+ "tpp_threshold_50_unintended_diff_only": 0.12547500878572465,
91
+ "tpp_threshold_100_total_metric": 0.2501500189304352,
92
+ "tpp_threshold_100_intended_diff_only": 0.40690003037452693,
93
+ "tpp_threshold_100_unintended_diff_only": 0.15675001144409179,
94
+ "tpp_threshold_500_total_metric": 0.23062503784894944,
95
+ "tpp_threshold_500_intended_diff_only": 0.4453000485897064,
96
+ "tpp_threshold_500_unintended_diff_only": 0.21467501074075698
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.07420000433921814,
103
+ "tpp_threshold_2_intended_diff_only": 0.0974000096321106,
104
+ "tpp_threshold_2_unintended_diff_only": 0.023200005292892456,
105
+ "tpp_threshold_5_total_metric": 0.1817000150680542,
106
+ "tpp_threshold_5_intended_diff_only": 0.24420002698898316,
107
+ "tpp_threshold_5_unintended_diff_only": 0.06250001192092895,
108
+ "tpp_threshold_10_total_metric": 0.24305000603199006,
109
  "tpp_threshold_10_intended_diff_only": 0.3666000247001648,
110
+ "tpp_threshold_10_unintended_diff_only": 0.12355001866817475,
111
+ "tpp_threshold_20_total_metric": 0.2477500170469284,
112
+ "tpp_threshold_20_intended_diff_only": 0.41080002784729003,
113
+ "tpp_threshold_20_unintended_diff_only": 0.16305001080036163,
114
+ "tpp_threshold_50_total_metric": 0.2190000206232071,
115
+ "tpp_threshold_50_intended_diff_only": 0.4552000403404236,
116
+ "tpp_threshold_50_unintended_diff_only": 0.2362000197172165,
117
+ "tpp_threshold_100_total_metric": 0.18680003583431243,
118
+ "tpp_threshold_100_intended_diff_only": 0.4652000546455383,
119
+ "tpp_threshold_100_unintended_diff_only": 0.2784000188112259,
120
+ "tpp_threshold_500_total_metric": 0.09815002977848053,
121
+ "tpp_threshold_500_intended_diff_only": 0.46780005693435667,
122
+ "tpp_threshold_500_unintended_diff_only": 0.36965002715587614
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.014149990677833558,
127
+ "tpp_threshold_2_intended_diff_only": 0.015599989891052246,
128
+ "tpp_threshold_2_unintended_diff_only": 0.001449999213218689,
129
+ "tpp_threshold_5_total_metric": 0.031950002908706664,
130
+ "tpp_threshold_5_intended_diff_only": 0.034200000762939456,
131
+ "tpp_threshold_5_unintended_diff_only": 0.002249997854232788,
132
+ "tpp_threshold_10_total_metric": 0.05875001549720764,
133
+ "tpp_threshold_10_intended_diff_only": 0.06680001020431518,
134
+ "tpp_threshold_10_unintended_diff_only": 0.008049994707107544,
135
+ "tpp_threshold_20_total_metric": 0.10639999210834503,
136
+ "tpp_threshold_20_intended_diff_only": 0.11519999504089355,
137
+ "tpp_threshold_20_unintended_diff_only": 0.008800002932548522,
138
+ "tpp_threshold_50_total_metric": 0.20645001530647278,
139
+ "tpp_threshold_50_intended_diff_only": 0.22120001316070556,
140
+ "tpp_threshold_50_unintended_diff_only": 0.014749997854232788,
141
+ "tpp_threshold_100_total_metric": 0.3135000020265579,
142
+ "tpp_threshold_100_intended_diff_only": 0.3486000061035156,
143
+ "tpp_threshold_100_unintended_diff_only": 0.0351000040769577,
144
+ "tpp_threshold_500_total_metric": 0.36310004591941836,
145
+ "tpp_threshold_500_intended_diff_only": 0.42280004024505613,
146
+ "tpp_threshold_500_unintended_diff_only": 0.059699994325637815
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.10175003111362457,
194
  "tpp_threshold_10_total_metric": 0.21299993991851807,
195
  "tpp_threshold_10_intended_diff_only": 0.3489999771118164,
196
+ "tpp_threshold_10_unintended_diff_only": 0.13600003719329834,
197
+ "tpp_threshold_20_total_metric": 0.20374999940395355,
198
+ "tpp_threshold_20_intended_diff_only": 0.3880000114440918,
199
+ "tpp_threshold_20_unintended_diff_only": 0.18425001204013824,
200
+ "tpp_threshold_50_total_metric": 0.17900000512599945,
201
+ "tpp_threshold_50_intended_diff_only": 0.4520000219345093,
202
+ "tpp_threshold_50_unintended_diff_only": 0.2730000168085098,
203
+ "tpp_threshold_100_total_metric": 0.14824999868869781,
204
+ "tpp_threshold_100_intended_diff_only": 0.45600003004074097,
205
+ "tpp_threshold_100_unintended_diff_only": 0.30775003135204315,
206
+ "tpp_threshold_500_total_metric": 0.07350000739097595,
207
+ "tpp_threshold_500_intended_diff_only": 0.45600003004074097,
208
+ "tpp_threshold_500_unintended_diff_only": 0.382500022649765
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.12274999916553497,
212
+ "tpp_threshold_2_intended_diff_only": 0.15700000524520874,
213
+ "tpp_threshold_2_unintended_diff_only": 0.03425000607967377,
214
+ "tpp_threshold_5_total_metric": 0.17999999225139618,
215
  "tpp_threshold_5_intended_diff_only": 0.22699999809265137,
216
+ "tpp_threshold_5_unintended_diff_only": 0.04700000584125519,
217
+ "tpp_threshold_10_total_metric": 0.296999990940094,
218
  "tpp_threshold_10_intended_diff_only": 0.35100001096725464,
219
+ "tpp_threshold_10_unintended_diff_only": 0.054000020027160645,
220
+ "tpp_threshold_20_total_metric": 0.32349999248981476,
221
+ "tpp_threshold_20_intended_diff_only": 0.42100000381469727,
222
+ "tpp_threshold_20_unintended_diff_only": 0.09750001132488251,
223
+ "tpp_threshold_50_total_metric": 0.2900000214576721,
224
+ "tpp_threshold_50_intended_diff_only": 0.4500000476837158,
225
+ "tpp_threshold_50_unintended_diff_only": 0.1600000262260437,
226
+ "tpp_threshold_100_total_metric": 0.25050003826618195,
227
+ "tpp_threshold_100_intended_diff_only": 0.4590000510215759,
228
+ "tpp_threshold_100_unintended_diff_only": 0.20850001275539398,
229
+ "tpp_threshold_500_total_metric": 0.08950002491474152,
230
+ "tpp_threshold_500_intended_diff_only": 0.4590000510215759,
231
+ "tpp_threshold_500_unintended_diff_only": 0.3695000261068344
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.010250017046928406,
235
+ "tpp_threshold_2_intended_diff_only": 0.013000011444091797,
236
+ "tpp_threshold_2_unintended_diff_only": 0.002749994397163391,
237
+ "tpp_threshold_5_total_metric": 0.19400005042552948,
238
  "tpp_threshold_5_intended_diff_only": 0.29600006341934204,
239
+ "tpp_threshold_5_unintended_diff_only": 0.10200001299381256,
240
+ "tpp_threshold_10_total_metric": 0.1627500206232071,
241
  "tpp_threshold_10_intended_diff_only": 0.3400000333786011,
242
+ "tpp_threshold_10_unintended_diff_only": 0.17725001275539398,
243
+ "tpp_threshold_20_total_metric": 0.16425004601478577,
244
+ "tpp_threshold_20_intended_diff_only": 0.3800000548362732,
245
+ "tpp_threshold_20_unintended_diff_only": 0.21575000882148743,
246
+ "tpp_threshold_50_total_metric": 0.148499995470047,
247
+ "tpp_threshold_50_intended_diff_only": 0.42900002002716064,
248
+ "tpp_threshold_50_unintended_diff_only": 0.28050002455711365,
249
+ "tpp_threshold_100_total_metric": 0.15000003576278687,
250
+ "tpp_threshold_100_intended_diff_only": 0.4490000605583191,
251
+ "tpp_threshold_100_unintended_diff_only": 0.2990000247955322,
252
+ "tpp_threshold_500_total_metric": 0.058750033378601074,
253
+ "tpp_threshold_500_intended_diff_only": 0.4530000686645508,
254
+ "tpp_threshold_500_unintended_diff_only": 0.3942500352859497
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.042000025510787964,
258
+ "tpp_threshold_2_intended_diff_only": 0.0480000376701355,
259
+ "tpp_threshold_2_unintended_diff_only": 0.006000012159347534,
260
+ "tpp_threshold_5_total_metric": 0.137750044465065,
261
+ "tpp_threshold_5_intended_diff_only": 0.15400004386901855,
262
+ "tpp_threshold_5_unintended_diff_only": 0.016249999403953552,
263
+ "tpp_threshold_10_total_metric": 0.34550002217292786,
264
  "tpp_threshold_10_intended_diff_only": 0.437000036239624,
265
+ "tpp_threshold_10_unintended_diff_only": 0.09150001406669617,
266
+ "tpp_threshold_20_total_metric": 0.3540000170469284,
267
+ "tpp_threshold_20_intended_diff_only": 0.4610000252723694,
268
+ "tpp_threshold_20_unintended_diff_only": 0.10700000822544098,
269
+ "tpp_threshold_50_total_metric": 0.2565000355243683,
270
+ "tpp_threshold_50_intended_diff_only": 0.48200005292892456,
271
+ "tpp_threshold_50_unintended_diff_only": 0.22550001740455627,
272
+ "tpp_threshold_100_total_metric": 0.2092500478029251,
273
+ "tpp_threshold_100_intended_diff_only": 0.48600006103515625,
274
+ "tpp_threshold_100_unintended_diff_only": 0.27675001323223114,
275
+ "tpp_threshold_500_total_metric": 0.16175003349781036,
276
+ "tpp_threshold_500_intended_diff_only": 0.49500006437301636,
277
+ "tpp_threshold_500_unintended_diff_only": 0.333250030875206
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.011750012636184692,
281
+ "tpp_threshold_2_intended_diff_only": 0.013000011444091797,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0012499988079071045,
283
+ "tpp_threshold_5_total_metric": 0.18450000882148743,
284
  "tpp_threshold_5_intended_diff_only": 0.23000001907348633,
285
+ "tpp_threshold_5_unintended_diff_only": 0.0455000102519989,
286
+ "tpp_threshold_10_total_metric": 0.19700005650520325,
287
  "tpp_threshold_10_intended_diff_only": 0.35600006580352783,
288
+ "tpp_threshold_10_unintended_diff_only": 0.15900000929832458,
289
+ "tpp_threshold_20_total_metric": 0.19325003027915955,
290
+ "tpp_threshold_20_intended_diff_only": 0.40400004386901855,
291
+ "tpp_threshold_20_unintended_diff_only": 0.210750013589859,
292
+ "tpp_threshold_50_total_metric": 0.2210000455379486,
293
+ "tpp_threshold_50_intended_diff_only": 0.4630000591278076,
294
+ "tpp_threshold_50_unintended_diff_only": 0.242000013589859,
295
+ "tpp_threshold_100_total_metric": 0.17600005865097046,
296
+ "tpp_threshold_100_intended_diff_only": 0.4760000705718994,
297
+ "tpp_threshold_100_unintended_diff_only": 0.30000001192092896,
298
+ "tpp_threshold_500_total_metric": 0.10725004971027374,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.36875002086162567
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.005750000476837158,
306
+ "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
307
+ "tpp_threshold_2_unintended_diff_only": 0.0032500028610229492,
308
+ "tpp_threshold_5_total_metric": 0.008000031113624573,
309
+ "tpp_threshold_5_intended_diff_only": 0.012000024318695068,
310
+ "tpp_threshold_5_unintended_diff_only": 0.003999993205070496,
311
+ "tpp_threshold_10_total_metric": 0.012999996542930603,
312
  "tpp_threshold_10_intended_diff_only": 0.018999993801116943,
313
+ "tpp_threshold_10_unintended_diff_only": 0.00599999725818634,
314
+ "tpp_threshold_20_total_metric": 0.07224997878074646,
315
+ "tpp_threshold_20_intended_diff_only": 0.07599997520446777,
316
+ "tpp_threshold_20_unintended_diff_only": 0.0037499964237213135,
317
+ "tpp_threshold_50_total_metric": 0.19425000250339508,
318
+ "tpp_threshold_50_intended_diff_only": 0.19999998807907104,
319
+ "tpp_threshold_50_unintended_diff_only": 0.005749985575675964,
320
+ "tpp_threshold_100_total_metric": 0.3400000035762787,
321
+ "tpp_threshold_100_intended_diff_only": 0.35600000619888306,
322
+ "tpp_threshold_100_unintended_diff_only": 0.01600000262260437,
323
+ "tpp_threshold_500_total_metric": 0.42875005304813385,
324
+ "tpp_threshold_500_intended_diff_only": 0.45500004291534424,
325
+ "tpp_threshold_500_unintended_diff_only": 0.026249989867210388
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.016500040888786316,
329
+ "tpp_threshold_2_intended_diff_only": 0.01500004529953003,
330
+ "tpp_threshold_2_unintended_diff_only": -0.0014999955892562866,
331
+ "tpp_threshold_5_total_metric": 0.024500057101249695,
332
+ "tpp_threshold_5_intended_diff_only": 0.033000051975250244,
333
+ "tpp_threshold_5_unintended_diff_only": 0.00849999487400055,
334
+ "tpp_threshold_10_total_metric": 0.07975004613399506,
335
+ "tpp_threshold_10_intended_diff_only": 0.09400004148483276,
336
+ "tpp_threshold_10_unintended_diff_only": 0.014249995350837708,
337
+ "tpp_threshold_20_total_metric": 0.11650000512599945,
338
+ "tpp_threshold_20_intended_diff_only": 0.1290000081062317,
339
+ "tpp_threshold_20_unintended_diff_only": 0.012500002980232239,
340
+ "tpp_threshold_50_total_metric": 0.2482500672340393,
341
+ "tpp_threshold_50_intended_diff_only": 0.2640000581741333,
342
+ "tpp_threshold_50_unintended_diff_only": 0.015749990940093994,
343
+ "tpp_threshold_100_total_metric": 0.36225005984306335,
344
+ "tpp_threshold_100_intended_diff_only": 0.3850000500679016,
345
+ "tpp_threshold_100_unintended_diff_only": 0.022749990224838257,
346
+ "tpp_threshold_500_total_metric": 0.3890000879764557,
347
+ "tpp_threshold_500_intended_diff_only": 0.43900007009506226,
348
+ "tpp_threshold_500_unintended_diff_only": 0.04999998211860657
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.014249980449676514,
352
+ "tpp_threshold_2_intended_diff_only": 0.015999972820281982,
353
+ "tpp_threshold_2_unintended_diff_only": 0.0017499923706054688,
354
+ "tpp_threshold_5_total_metric": 0.01449999213218689,
355
+ "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
356
+ "tpp_threshold_5_unintended_diff_only": -0.0004999935626983643,
357
+ "tpp_threshold_10_total_metric": 0.03275004029273987,
358
+ "tpp_threshold_10_intended_diff_only": 0.0350000262260437,
359
+ "tpp_threshold_10_unintended_diff_only": 0.002249985933303833,
360
+ "tpp_threshold_20_total_metric": 0.05674998462200165,
361
+ "tpp_threshold_20_intended_diff_only": 0.06599998474121094,
362
+ "tpp_threshold_20_unintended_diff_only": 0.00925000011920929,
363
+ "tpp_threshold_50_total_metric": 0.14900000393390656,
364
+ "tpp_threshold_50_intended_diff_only": 0.16600000858306885,
365
+ "tpp_threshold_50_unintended_diff_only": 0.017000004649162292,
366
+ "tpp_threshold_100_total_metric": 0.2670000344514847,
367
+ "tpp_threshold_100_intended_diff_only": 0.3490000367164612,
368
+ "tpp_threshold_100_unintended_diff_only": 0.0820000022649765,
369
+ "tpp_threshold_500_total_metric": 0.3265000283718109,
370
+ "tpp_threshold_500_intended_diff_only": 0.4270000457763672,
371
+ "tpp_threshold_500_unintended_diff_only": 0.10050001740455627
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.008999943733215332,
375
+ "tpp_threshold_2_intended_diff_only": 0.008999943733215332,
376
+ "tpp_threshold_2_unintended_diff_only": 0.0,
377
+ "tpp_threshold_5_total_metric": 0.027249962091445923,
378
+ "tpp_threshold_5_intended_diff_only": 0.026999950408935547,
379
+ "tpp_threshold_5_unintended_diff_only": -0.000250011682510376,
380
+ "tpp_threshold_10_total_metric": 0.03299999237060547,
381
+ "tpp_threshold_10_intended_diff_only": 0.041999995708465576,
382
+ "tpp_threshold_10_unintended_diff_only": 0.009000003337860107,
383
+ "tpp_threshold_20_total_metric": 0.052249982953071594,
384
+ "tpp_threshold_20_intended_diff_only": 0.05699998140335083,
385
+ "tpp_threshold_20_unintended_diff_only": 0.004749998450279236,
386
+ "tpp_threshold_50_total_metric": 0.11225000023841858,
387
+ "tpp_threshold_50_intended_diff_only": 0.12999999523162842,
388
+ "tpp_threshold_50_unintended_diff_only": 0.01774999499320984,
389
+ "tpp_threshold_100_total_metric": 0.2544999420642853,
390
+ "tpp_threshold_100_intended_diff_only": 0.28099995851516724,
391
+ "tpp_threshold_100_unintended_diff_only": 0.026500016450881958,
392
+ "tpp_threshold_500_total_metric": 0.35075002908706665,
393
+ "tpp_threshold_500_intended_diff_only": 0.42000001668930054,
394
+ "tpp_threshold_500_unintended_diff_only": 0.06924998760223389
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.025249987840652466,
398
+ "tpp_threshold_2_intended_diff_only": 0.02899998426437378,
399
+ "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135,
400
+ "tpp_threshold_5_total_metric": 0.08549997210502625,
401
+ "tpp_threshold_5_intended_diff_only": 0.08499997854232788,
402
+ "tpp_threshold_5_unintended_diff_only": -0.0004999935626983643,
403
+ "tpp_threshold_10_total_metric": 0.1352500021457672,
404
+ "tpp_threshold_10_intended_diff_only": 0.14399999380111694,
405
+ "tpp_threshold_10_unintended_diff_only": 0.008749991655349731,
406
+ "tpp_threshold_20_total_metric": 0.234250009059906,
407
+ "tpp_threshold_20_intended_diff_only": 0.24800002574920654,
408
+ "tpp_threshold_20_unintended_diff_only": 0.013750016689300537,
409
+ "tpp_threshold_50_total_metric": 0.32850000262260437,
410
+ "tpp_threshold_50_intended_diff_only": 0.3460000157356262,
411
+ "tpp_threshold_50_unintended_diff_only": 0.01750001311302185,
412
+ "tpp_threshold_100_total_metric": 0.3437499701976776,
413
+ "tpp_threshold_100_intended_diff_only": 0.37199997901916504,
414
+ "tpp_threshold_100_unintended_diff_only": 0.028250008821487427,
415
+ "tpp_threshold_500_total_metric": 0.3205000311136246,
416
+ "tpp_threshold_500_intended_diff_only": 0.37300002574920654,
417
+ "tpp_threshold_500_unintended_diff_only": 0.05249999463558197
418
  }
419
  }
420
  }
gemma-2-2b-btk-mat/k-100/seed-0/snapshots/step-48828-tokens-200000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "03e860f7-ff02-427b-b6dc-682a08f89af5",
69
- "datetime_epoch_millis": 1776882661358,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.013974991440773011,
73
- "tpp_threshold_2_intended_diff_only": 0.014899992942810058,
74
- "tpp_threshold_2_unintended_diff_only": 0.0009250015020370483,
75
- "tpp_threshold_5_total_metric": 0.04457501173019409,
76
- "tpp_threshold_5_intended_diff_only": 0.05280001759529114,
77
- "tpp_threshold_5_unintended_diff_only": 0.008225005865097047,
78
- "tpp_threshold_10_total_metric": 0.1080999955534935,
79
- "tpp_threshold_10_intended_diff_only": 0.1331000030040741,
80
- "tpp_threshold_10_unintended_diff_only": 0.025000007450580595,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.00914999544620514,
99
- "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
100
- "tpp_threshold_2_unintended_diff_only": -0.00014999210834503173,
101
- "tpp_threshold_5_total_metric": 0.05970000922679901,
102
- "tpp_threshold_5_intended_diff_only": 0.07140002250671387,
103
- "tpp_threshold_5_unintended_diff_only": 0.011700013279914856,
104
- "tpp_threshold_10_total_metric": 0.16279999315738677,
105
- "tpp_threshold_10_intended_diff_only": 0.20660001039505005,
106
- "tpp_threshold_10_unintended_diff_only": 0.04380001723766327,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.018799987435340882,
123
- "tpp_threshold_2_intended_diff_only": 0.02079998254776001,
124
- "tpp_threshold_2_unintended_diff_only": 0.0019999951124191283,
125
- "tpp_threshold_5_total_metric": 0.029450014233589172,
126
- "tpp_threshold_5_intended_diff_only": 0.03420001268386841,
127
- "tpp_threshold_5_unintended_diff_only": 0.004749998450279236,
128
- "tpp_threshold_10_total_metric": 0.05339999794960022,
129
- "tpp_threshold_10_intended_diff_only": 0.059599995613098145,
130
- "tpp_threshold_10_unintended_diff_only": 0.006199997663497925,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.0020000189542770386,
190
  "tpp_threshold_10_total_metric": 0.03774997591972351,
191
  "tpp_threshold_10_intended_diff_only": 0.042999982833862305,
192
- "tpp_threshold_10_unintended_diff_only": 0.005250006914138794
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": 0.0037499815225601196,
196
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
197
- "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983,
198
- "tpp_threshold_5_total_metric": 0.058500006794929504,
199
- "tpp_threshold_5_intended_diff_only": 0.07700002193450928,
200
- "tpp_threshold_5_unintended_diff_only": 0.018500015139579773,
201
- "tpp_threshold_10_total_metric": 0.10574996471405029,
202
- "tpp_threshold_10_intended_diff_only": 0.12599998712539673,
203
- "tpp_threshold_10_unintended_diff_only": 0.020250022411346436
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.008750036358833313,
207
- "tpp_threshold_2_intended_diff_only": 0.006000041961669922,
208
- "tpp_threshold_2_unintended_diff_only": -0.002749994397163391,
209
- "tpp_threshold_5_total_metric": 0.06275001168251038,
210
  "tpp_threshold_5_intended_diff_only": 0.07300001382827759,
211
- "tpp_threshold_5_unintended_diff_only": 0.010250002145767212,
212
- "tpp_threshold_10_total_metric": 0.2070000171661377,
213
- "tpp_threshold_10_intended_diff_only": 0.2890000343322754,
214
- "tpp_threshold_10_unintended_diff_only": 0.0820000171661377
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.015000015497207642,
218
  "tpp_threshold_2_intended_diff_only": 0.013000011444091797,
219
- "tpp_threshold_2_unintended_diff_only": -0.0020000040531158447,
220
- "tpp_threshold_5_total_metric": 0.05674999952316284,
221
- "tpp_threshold_5_intended_diff_only": 0.07300001382827759,
222
- "tpp_threshold_5_unintended_diff_only": 0.016250014305114746,
223
- "tpp_threshold_10_total_metric": 0.24774999916553497,
224
  "tpp_threshold_10_intended_diff_only": 0.281000018119812,
225
- "tpp_threshold_10_unintended_diff_only": 0.03325001895427704
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.01150001585483551,
229
- "tpp_threshold_2_intended_diff_only": 0.013000011444091797,
230
- "tpp_threshold_2_unintended_diff_only": 0.0014999955892562866,
231
- "tpp_threshold_5_total_metric": 0.10050003230571747,
232
  "tpp_threshold_5_intended_diff_only": 0.11200004816055298,
233
- "tpp_threshold_5_unintended_diff_only": 0.01150001585483551,
234
- "tpp_threshold_10_total_metric": 0.21575000882148743,
235
- "tpp_threshold_10_intended_diff_only": 0.2940000295639038,
236
- "tpp_threshold_10_unintended_diff_only": 0.07825002074241638
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.0104999840259552,
242
- "tpp_threshold_2_intended_diff_only": 0.0059999823570251465,
243
- "tpp_threshold_2_unintended_diff_only": -0.004500001668930054,
244
- "tpp_threshold_5_total_metric": 0.012750014662742615,
245
- "tpp_threshold_5_intended_diff_only": 0.013000011444091797,
246
- "tpp_threshold_5_unintended_diff_only": 0.00024999678134918213,
247
- "tpp_threshold_10_total_metric": 0.02049998939037323,
248
  "tpp_threshold_10_intended_diff_only": 0.02799999713897705,
249
- "tpp_threshold_10_unintended_diff_only": 0.007500007748603821
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": 0.005250036716461182,
253
- "tpp_threshold_2_intended_diff_only": 0.012000024318695068,
254
- "tpp_threshold_2_unintended_diff_only": 0.006749987602233887,
255
- "tpp_threshold_5_total_metric": 0.01175004243850708,
256
- "tpp_threshold_5_intended_diff_only": 0.020000040531158447,
257
- "tpp_threshold_5_unintended_diff_only": 0.008249998092651367,
258
- "tpp_threshold_10_total_metric": 0.06925004720687866,
259
- "tpp_threshold_10_intended_diff_only": 0.07600003480911255,
260
- "tpp_threshold_10_unintended_diff_only": 0.006749987602233887
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.018749982118606567,
264
  "tpp_threshold_2_intended_diff_only": 0.015999972820281982,
265
- "tpp_threshold_2_unintended_diff_only": -0.002750009298324585,
266
- "tpp_threshold_5_total_metric": 0.02050003409385681,
267
- "tpp_threshold_5_intended_diff_only": 0.021000027656555176,
268
- "tpp_threshold_5_unintended_diff_only": 0.0004999935626983643,
269
- "tpp_threshold_10_total_metric": 0.013999998569488525,
270
- "tpp_threshold_10_intended_diff_only": 0.019999980926513672,
271
- "tpp_threshold_10_unintended_diff_only": 0.0059999823570251465
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": 0.0002499520778656006,
275
  "tpp_threshold_2_intended_diff_only": 0.008999943733215332,
276
- "tpp_threshold_2_unintended_diff_only": 0.008749991655349731,
277
- "tpp_threshold_5_total_metric": 0.009249970316886902,
278
- "tpp_threshold_5_intended_diff_only": 0.011999964714050293,
279
- "tpp_threshold_5_unintended_diff_only": 0.002749994397163391,
280
- "tpp_threshold_10_total_metric": 0.02349993586540222,
281
- "tpp_threshold_10_intended_diff_only": 0.026999950408935547,
282
- "tpp_threshold_10_unintended_diff_only": 0.003500014543533325
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.05924998223781586,
286
- "tpp_threshold_2_intended_diff_only": 0.06099998950958252,
287
- "tpp_threshold_2_unintended_diff_only": 0.0017500072717666626,
288
- "tpp_threshold_5_total_metric": 0.09300000965595245,
289
- "tpp_threshold_5_intended_diff_only": 0.10500001907348633,
290
- "tpp_threshold_5_unintended_diff_only": 0.012000009417533875,
291
- "tpp_threshold_10_total_metric": 0.13975001871585846,
292
- "tpp_threshold_10_intended_diff_only": 0.1470000147819519,
293
- "tpp_threshold_10_unintended_diff_only": 0.007249996066093445
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "7997e007-9853-4f5e-9dab-29873d03948e",
73
+ "datetime_epoch_millis": 1777314742336,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.013749976456165313,
77
+ "tpp_threshold_2_intended_diff_only": 0.015399986505508423,
78
+ "tpp_threshold_2_unintended_diff_only": 0.0016500100493431092,
79
+ "tpp_threshold_5_total_metric": 0.04444999247789383,
80
+ "tpp_threshold_5_intended_diff_only": 0.0515999972820282,
81
+ "tpp_threshold_5_unintended_diff_only": 0.007150004804134368,
82
+ "tpp_threshold_10_total_metric": 0.10890000611543656,
83
+ "tpp_threshold_10_intended_diff_only": 0.1340000033378601,
84
+ "tpp_threshold_10_unintended_diff_only": 0.025099997222423554,
85
+ "tpp_threshold_20_total_metric": 0.189675010740757,
86
+ "tpp_threshold_20_intended_diff_only": 0.22810001969337462,
87
+ "tpp_threshold_20_unintended_diff_only": 0.03842500895261765,
88
+ "tpp_threshold_50_total_metric": 0.2488000199198723,
89
+ "tpp_threshold_50_intended_diff_only": 0.3137000262737274,
90
+ "tpp_threshold_50_unintended_diff_only": 0.06490000635385514,
91
+ "tpp_threshold_100_total_metric": 0.28297500461339953,
92
+ "tpp_threshold_100_intended_diff_only": 0.37920001745223997,
93
+ "tpp_threshold_100_unintended_diff_only": 0.09622501283884048,
94
+ "tpp_threshold_500_total_metric": 0.26877502501010897,
95
+ "tpp_threshold_500_intended_diff_only": 0.44490004181861875,
96
+ "tpp_threshold_500_unintended_diff_only": 0.1761250168085098
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.008649978041648864,
103
+ "tpp_threshold_2_intended_diff_only": 0.009399998188018798,
104
+ "tpp_threshold_2_unintended_diff_only": 0.0007500201463699341,
105
+ "tpp_threshold_5_total_metric": 0.05999999940395355,
106
+ "tpp_threshold_5_intended_diff_only": 0.07120001316070557,
107
+ "tpp_threshold_5_unintended_diff_only": 0.011200013756752013,
108
+ "tpp_threshold_10_total_metric": 0.16315000951290132,
109
+ "tpp_threshold_10_intended_diff_only": 0.2064000129699707,
110
+ "tpp_threshold_10_unintended_diff_only": 0.0432500034570694,
111
+ "tpp_threshold_20_total_metric": 0.28975001275539397,
112
+ "tpp_threshold_20_intended_diff_only": 0.3580000281333923,
113
+ "tpp_threshold_20_unintended_diff_only": 0.06825001537799835,
114
+ "tpp_threshold_50_total_metric": 0.3225500226020813,
115
+ "tpp_threshold_50_intended_diff_only": 0.4402000427246094,
116
+ "tpp_threshold_50_unintended_diff_only": 0.11765002012252808,
117
+ "tpp_threshold_100_total_metric": 0.28970001339912416,
118
+ "tpp_threshold_100_intended_diff_only": 0.4628000378608704,
119
+ "tpp_threshold_100_unintended_diff_only": 0.17310002446174622,
120
+ "tpp_threshold_500_total_metric": 0.15300002992153167,
121
+ "tpp_threshold_500_intended_diff_only": 0.46780005693435667,
122
+ "tpp_threshold_500_unintended_diff_only": 0.314800027012825
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.01884997487068176,
127
+ "tpp_threshold_2_intended_diff_only": 0.021399974822998047,
128
+ "tpp_threshold_2_unintended_diff_only": 0.0025499999523162843,
129
+ "tpp_threshold_5_total_metric": 0.028899985551834106,
130
+ "tpp_threshold_5_intended_diff_only": 0.03199998140335083,
131
+ "tpp_threshold_5_unintended_diff_only": 0.003099995851516724,
132
+ "tpp_threshold_10_total_metric": 0.0546500027179718,
133
+ "tpp_threshold_10_intended_diff_only": 0.06159999370574951,
134
+ "tpp_threshold_10_unintended_diff_only": 0.00694999098777771,
135
+ "tpp_threshold_20_total_metric": 0.08960000872611999,
136
+ "tpp_threshold_20_intended_diff_only": 0.09820001125335694,
137
+ "tpp_threshold_20_unintended_diff_only": 0.008600002527236939,
138
+ "tpp_threshold_50_total_metric": 0.17505001723766328,
139
+ "tpp_threshold_50_intended_diff_only": 0.18720000982284546,
140
+ "tpp_threshold_50_unintended_diff_only": 0.01214999258518219,
141
+ "tpp_threshold_100_total_metric": 0.27624999582767484,
142
+ "tpp_threshold_100_intended_diff_only": 0.2955999970436096,
143
+ "tpp_threshold_100_unintended_diff_only": 0.019350001215934755,
144
+ "tpp_threshold_500_total_metric": 0.3845500200986862,
145
+ "tpp_threshold_500_intended_diff_only": 0.42200002670288084,
146
+ "tpp_threshold_500_unintended_diff_only": 0.03745000660419464
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.0020000189542770386,
194
  "tpp_threshold_10_total_metric": 0.03774997591972351,
195
  "tpp_threshold_10_intended_diff_only": 0.042999982833862305,
196
+ "tpp_threshold_10_unintended_diff_only": 0.005250006914138794,
197
+ "tpp_threshold_20_total_metric": 0.231999933719635,
198
+ "tpp_threshold_20_intended_diff_only": 0.2709999680519104,
199
+ "tpp_threshold_20_unintended_diff_only": 0.03900003433227539,
200
+ "tpp_threshold_50_total_metric": 0.312749981880188,
201
+ "tpp_threshold_50_intended_diff_only": 0.42000001668930054,
202
+ "tpp_threshold_50_unintended_diff_only": 0.10725003480911255,
203
+ "tpp_threshold_100_total_metric": 0.3267499953508377,
204
+ "tpp_threshold_100_intended_diff_only": 0.4520000219345093,
205
+ "tpp_threshold_100_unintended_diff_only": 0.12525002658367157,
206
+ "tpp_threshold_500_total_metric": 0.1525000035762787,
207
+ "tpp_threshold_500_intended_diff_only": 0.45600003004074097,
208
+ "tpp_threshold_500_unintended_diff_only": 0.3035000264644623
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.007749989628791809,
212
+ "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
213
+ "tpp_threshold_2_unintended_diff_only": 0.0002500265836715698,
214
+ "tpp_threshold_5_total_metric": 0.06049996614456177,
215
+ "tpp_threshold_5_intended_diff_only": 0.07499998807907104,
216
+ "tpp_threshold_5_unintended_diff_only": 0.014500021934509277,
217
+ "tpp_threshold_10_total_metric": 0.10999999940395355,
218
+ "tpp_threshold_10_intended_diff_only": 0.125,
219
+ "tpp_threshold_10_unintended_diff_only": 0.015000000596046448,
220
+ "tpp_threshold_20_total_metric": 0.31074999272823334,
221
+ "tpp_threshold_20_intended_diff_only": 0.36000001430511475,
222
+ "tpp_threshold_20_unintended_diff_only": 0.04925002157688141,
223
+ "tpp_threshold_50_total_metric": 0.34825000166893005,
224
+ "tpp_threshold_50_intended_diff_only": 0.4390000104904175,
225
+ "tpp_threshold_50_unintended_diff_only": 0.09075000882148743,
226
+ "tpp_threshold_100_total_metric": 0.2982500195503235,
227
+ "tpp_threshold_100_intended_diff_only": 0.4540000557899475,
228
+ "tpp_threshold_100_unintended_diff_only": 0.15575003623962402,
229
+ "tpp_threshold_500_total_metric": 0.14525002241134644,
230
+ "tpp_threshold_500_intended_diff_only": 0.4590000510215759,
231
+ "tpp_threshold_500_unintended_diff_only": 0.3137500286102295
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.0037499964237213135,
235
+ "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
236
+ "tpp_threshold_2_unintended_diff_only": 0.000250011682510376,
237
+ "tpp_threshold_5_total_metric": 0.06024999916553497,
238
  "tpp_threshold_5_intended_diff_only": 0.07300001382827759,
239
+ "tpp_threshold_5_unintended_diff_only": 0.012750014662742615,
240
+ "tpp_threshold_10_total_metric": 0.20650000870227814,
241
+ "tpp_threshold_10_intended_diff_only": 0.2900000214576721,
242
+ "tpp_threshold_10_unintended_diff_only": 0.08350001275539398,
243
+ "tpp_threshold_20_total_metric": 0.21900004148483276,
244
+ "tpp_threshold_20_intended_diff_only": 0.3380000591278076,
245
+ "tpp_threshold_20_unintended_diff_only": 0.11900001764297485,
246
+ "tpp_threshold_50_total_metric": 0.2367500513792038,
247
+ "tpp_threshold_50_intended_diff_only": 0.4070000648498535,
248
+ "tpp_threshold_50_unintended_diff_only": 0.17025001347064972,
249
+ "tpp_threshold_100_total_metric": 0.23674999177455902,
250
+ "tpp_threshold_100_intended_diff_only": 0.43800002336502075,
251
+ "tpp_threshold_100_unintended_diff_only": 0.20125003159046173,
252
+ "tpp_threshold_500_total_metric": 0.10350003838539124,
253
+ "tpp_threshold_500_intended_diff_only": 0.4530000686645508,
254
+ "tpp_threshold_500_unintended_diff_only": 0.34950003027915955
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.012249991297721863,
258
  "tpp_threshold_2_intended_diff_only": 0.013000011444091797,
259
+ "tpp_threshold_2_unintended_diff_only": 0.0007500201463699341,
260
+ "tpp_threshold_5_total_metric": 0.062499985098838806,
261
+ "tpp_threshold_5_intended_diff_only": 0.07400000095367432,
262
+ "tpp_threshold_5_unintended_diff_only": 0.01150001585483551,
263
+ "tpp_threshold_10_total_metric": 0.24575001001358032,
264
  "tpp_threshold_10_intended_diff_only": 0.281000018119812,
265
+ "tpp_threshold_10_unintended_diff_only": 0.03525000810623169,
266
+ "tpp_threshold_20_total_metric": 0.4050000458955765,
267
+ "tpp_threshold_20_intended_diff_only": 0.4500000476837158,
268
+ "tpp_threshold_20_unintended_diff_only": 0.04500000178813934,
269
+ "tpp_threshold_50_total_metric": 0.39475004374980927,
270
+ "tpp_threshold_50_intended_diff_only": 0.48100006580352783,
271
+ "tpp_threshold_50_unintended_diff_only": 0.08625002205371857,
272
+ "tpp_threshold_100_total_metric": 0.3282500058412552,
273
+ "tpp_threshold_100_intended_diff_only": 0.49400001764297485,
274
+ "tpp_threshold_100_unintended_diff_only": 0.16575001180171967,
275
+ "tpp_threshold_500_total_metric": 0.23000004887580872,
276
+ "tpp_threshold_500_intended_diff_only": 0.49500006437301636,
277
+ "tpp_threshold_500_unintended_diff_only": 0.26500001549720764
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.012749984860420227,
281
+ "tpp_threshold_2_intended_diff_only": 0.013999998569488525,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983,
283
+ "tpp_threshold_5_total_metric": 0.09675005078315735,
284
  "tpp_threshold_5_intended_diff_only": 0.11200004816055298,
285
+ "tpp_threshold_5_unintended_diff_only": 0.01524999737739563,
286
+ "tpp_threshold_10_total_metric": 0.215750053524971,
287
+ "tpp_threshold_10_intended_diff_only": 0.2930000424385071,
288
+ "tpp_threshold_10_unintended_diff_only": 0.07724998891353607,
289
+ "tpp_threshold_20_total_metric": 0.2820000499486923,
290
+ "tpp_threshold_20_intended_diff_only": 0.3710000514984131,
291
+ "tpp_threshold_20_unintended_diff_only": 0.08900000154972076,
292
+ "tpp_threshold_50_total_metric": 0.3202500343322754,
293
+ "tpp_threshold_50_intended_diff_only": 0.4540000557899475,
294
+ "tpp_threshold_50_unintended_diff_only": 0.13375002145767212,
295
+ "tpp_threshold_100_total_metric": 0.2585000544786453,
296
+ "tpp_threshold_100_intended_diff_only": 0.4760000705718994,
297
+ "tpp_threshold_100_unintended_diff_only": 0.2175000160932541,
298
+ "tpp_threshold_500_total_metric": 0.1337500363588333,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.3422500342130661
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.007249981164932251,
306
+ "tpp_threshold_2_intended_diff_only": 0.009999990463256836,
307
+ "tpp_threshold_2_unintended_diff_only": 0.002750009298324585,
308
+ "tpp_threshold_5_total_metric": 0.0059999823570251465,
309
+ "tpp_threshold_5_intended_diff_only": 0.010999977588653564,
310
+ "tpp_threshold_5_unintended_diff_only": 0.004999995231628418,
311
+ "tpp_threshold_10_total_metric": 0.02175000309944153,
312
  "tpp_threshold_10_intended_diff_only": 0.02799999713897705,
313
+ "tpp_threshold_10_unintended_diff_only": 0.0062499940395355225,
314
+ "tpp_threshold_20_total_metric": 0.0532500296831131,
315
+ "tpp_threshold_20_intended_diff_only": 0.058000028133392334,
316
+ "tpp_threshold_20_unintended_diff_only": 0.004749998450279236,
317
+ "tpp_threshold_50_total_metric": 0.16500000655651093,
318
+ "tpp_threshold_50_intended_diff_only": 0.17100000381469727,
319
+ "tpp_threshold_50_unintended_diff_only": 0.00599999725818634,
320
+ "tpp_threshold_100_total_metric": 0.3009999990463257,
321
+ "tpp_threshold_100_intended_diff_only": 0.3149999976158142,
322
+ "tpp_threshold_100_unintended_diff_only": 0.013999998569488525,
323
+ "tpp_threshold_500_total_metric": 0.42924998700618744,
324
+ "tpp_threshold_500_intended_diff_only": 0.45399999618530273,
325
+ "tpp_threshold_500_unintended_diff_only": 0.024750009179115295
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.01524999737739563,
329
+ "tpp_threshold_2_intended_diff_only": 0.013999998569488525,
330
+ "tpp_threshold_2_unintended_diff_only": -0.0012499988079071045,
331
+ "tpp_threshold_5_total_metric": 0.008500024676322937,
332
+ "tpp_threshold_5_intended_diff_only": 0.017000019550323486,
333
+ "tpp_threshold_5_unintended_diff_only": 0.00849999487400055,
334
+ "tpp_threshold_10_total_metric": 0.0675000250339508,
335
+ "tpp_threshold_10_intended_diff_only": 0.078000009059906,
336
+ "tpp_threshold_10_unintended_diff_only": 0.0104999840259552,
337
+ "tpp_threshold_20_total_metric": 0.09450003504753113,
338
+ "tpp_threshold_20_intended_diff_only": 0.10800004005432129,
339
+ "tpp_threshold_20_unintended_diff_only": 0.013500005006790161,
340
+ "tpp_threshold_50_total_metric": 0.1817500740289688,
341
+ "tpp_threshold_50_intended_diff_only": 0.1990000605583191,
342
+ "tpp_threshold_50_unintended_diff_only": 0.01724998652935028,
343
+ "tpp_threshold_100_total_metric": 0.3047500252723694,
344
+ "tpp_threshold_100_intended_diff_only": 0.3370000123977661,
345
+ "tpp_threshold_100_unintended_diff_only": 0.03224998712539673,
346
+ "tpp_threshold_500_total_metric": 0.39600005745887756,
347
+ "tpp_threshold_500_intended_diff_only": 0.43900007009506226,
348
+ "tpp_threshold_500_unintended_diff_only": 0.04300001263618469
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.012749984860420227,
352
  "tpp_threshold_2_intended_diff_only": 0.015999972820281982,
353
+ "tpp_threshold_2_unintended_diff_only": 0.0032499879598617554,
354
+ "tpp_threshold_5_total_metric": 0.015000000596046448,
355
+ "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
356
+ "tpp_threshold_5_unintended_diff_only": -0.0010000020265579224,
357
+ "tpp_threshold_10_total_metric": 0.022749990224838257,
358
+ "tpp_threshold_10_intended_diff_only": 0.02499997615814209,
359
+ "tpp_threshold_10_unintended_diff_only": 0.002249985933303833,
360
+ "tpp_threshold_20_total_metric": 0.02800002694129944,
361
+ "tpp_threshold_20_intended_diff_only": 0.0350000262260437,
362
+ "tpp_threshold_20_unintended_diff_only": 0.006999999284744263,
363
+ "tpp_threshold_50_total_metric": 0.13324999809265137,
364
+ "tpp_threshold_50_intended_diff_only": 0.14399999380111694,
365
+ "tpp_threshold_50_unintended_diff_only": 0.010749995708465576,
366
+ "tpp_threshold_100_total_metric": 0.23224999010562897,
367
+ "tpp_threshold_100_intended_diff_only": 0.25,
368
+ "tpp_threshold_100_unintended_diff_only": 0.017750009894371033,
369
+ "tpp_threshold_500_total_metric": 0.38700003921985626,
370
+ "tpp_threshold_500_intended_diff_only": 0.4270000457763672,
371
+ "tpp_threshold_500_unintended_diff_only": 0.040000006556510925
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.008249938488006592,
375
  "tpp_threshold_2_intended_diff_only": 0.008999943733215332,
376
+ "tpp_threshold_2_unintended_diff_only": 0.0007500052452087402,
377
+ "tpp_threshold_5_total_metric": 0.018499955534934998,
378
+ "tpp_threshold_5_intended_diff_only": 0.01799994707107544,
379
+ "tpp_threshold_5_unintended_diff_only": -0.0005000084638595581,
380
+ "tpp_threshold_10_total_metric": 0.01774996519088745,
381
+ "tpp_threshold_10_intended_diff_only": 0.02499997615814209,
382
+ "tpp_threshold_10_unintended_diff_only": 0.007250010967254639,
383
+ "tpp_threshold_20_total_metric": 0.03374996781349182,
384
+ "tpp_threshold_20_intended_diff_only": 0.038999974727630615,
385
+ "tpp_threshold_20_unintended_diff_only": 0.005250006914138794,
386
+ "tpp_threshold_50_total_metric": 0.08224999904632568,
387
+ "tpp_threshold_50_intended_diff_only": 0.09299999475479126,
388
+ "tpp_threshold_50_unintended_diff_only": 0.010749995708465576,
389
+ "tpp_threshold_100_total_metric": 0.20024996995925903,
390
+ "tpp_threshold_100_intended_diff_only": 0.20999997854232788,
391
+ "tpp_threshold_100_unintended_diff_only": 0.009750008583068848,
392
+ "tpp_threshold_500_total_metric": 0.3814999908208847,
393
+ "tpp_threshold_500_intended_diff_only": 0.4169999957084656,
394
+ "tpp_threshold_500_unintended_diff_only": 0.03550000488758087
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.050749972462654114,
398
+ "tpp_threshold_2_intended_diff_only": 0.05799996852874756,
399
+ "tpp_threshold_2_unintended_diff_only": 0.007249996066093445,
400
+ "tpp_threshold_5_total_metric": 0.096499964594841,
401
+ "tpp_threshold_5_intended_diff_only": 0.09999996423721313,
402
+ "tpp_threshold_5_unintended_diff_only": 0.0034999996423721313,
403
+ "tpp_threshold_10_total_metric": 0.14350003004074097,
404
+ "tpp_threshold_10_intended_diff_only": 0.15200001001358032,
405
+ "tpp_threshold_10_unintended_diff_only": 0.008499979972839355,
406
+ "tpp_threshold_20_total_metric": 0.2384999841451645,
407
+ "tpp_threshold_20_intended_diff_only": 0.25099998712539673,
408
+ "tpp_threshold_20_unintended_diff_only": 0.012500002980232239,
409
+ "tpp_threshold_50_total_metric": 0.31300000846385956,
410
+ "tpp_threshold_50_intended_diff_only": 0.32899999618530273,
411
+ "tpp_threshold_50_unintended_diff_only": 0.015999987721443176,
412
+ "tpp_threshold_100_total_metric": 0.34299999475479126,
413
+ "tpp_threshold_100_intended_diff_only": 0.3659999966621399,
414
+ "tpp_threshold_100_unintended_diff_only": 0.023000001907348633,
415
+ "tpp_threshold_500_total_metric": 0.3290000259876251,
416
+ "tpp_threshold_500_intended_diff_only": 0.37300002574920654,
417
+ "tpp_threshold_500_unintended_diff_only": 0.04399999976158142
418
  }
419
  }
420
  }
gemma-2-2b-btk-mat/k-100/seed-0/snapshots/step-61035-tokens-250000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "258fd16b-d44d-4fbd-ba02-c9735bc19385",
69
- "datetime_epoch_millis": 1776882735186,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.01367499530315399,
73
- "tpp_threshold_2_intended_diff_only": 0.014899998903274536,
74
- "tpp_threshold_2_unintended_diff_only": 0.0012250036001205443,
75
- "tpp_threshold_5_total_metric": 0.04029998928308487,
76
- "tpp_threshold_5_intended_diff_only": 0.04739999771118164,
77
- "tpp_threshold_5_unintended_diff_only": 0.007100008428096771,
78
- "tpp_threshold_10_total_metric": 0.11207500398159026,
79
- "tpp_threshold_10_intended_diff_only": 0.13380001187324525,
80
- "tpp_threshold_10_unintended_diff_only": 0.021725007891654966,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.011549988389015197,
99
- "tpp_threshold_2_intended_diff_only": 0.012000000476837159,
100
- "tpp_threshold_2_unintended_diff_only": 0.00045001208782196044,
101
- "tpp_threshold_5_total_metric": 0.0490999847650528,
102
- "tpp_threshold_5_intended_diff_only": 0.05839999914169312,
103
- "tpp_threshold_5_unintended_diff_only": 0.00930001437664032,
104
- "tpp_threshold_10_total_metric": 0.17535000443458557,
105
  "tpp_threshold_10_intended_diff_only": 0.21220002174377442,
106
- "tpp_threshold_10_unintended_diff_only": 0.03685001730918884,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.015800002217292785,
123
- "tpp_threshold_2_intended_diff_only": 0.017799997329711915,
124
- "tpp_threshold_2_unintended_diff_only": 0.0019999951124191283,
125
- "tpp_threshold_5_total_metric": 0.03149999380111694,
126
- "tpp_threshold_5_intended_diff_only": 0.036399996280670165,
127
- "tpp_threshold_5_unintended_diff_only": 0.004900002479553222,
128
- "tpp_threshold_10_total_metric": 0.04880000352859497,
129
- "tpp_threshold_10_intended_diff_only": 0.05540000200271607,
130
- "tpp_threshold_10_unintended_diff_only": 0.006599998474121094,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.0022500306367874146,
190
  "tpp_threshold_10_total_metric": 0.05924996733665466,
191
  "tpp_threshold_10_intended_diff_only": 0.06499999761581421,
192
- "tpp_threshold_10_unintended_diff_only": 0.005750030279159546
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": 0.00024996697902679443,
196
- "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
197
- "tpp_threshold_2_unintended_diff_only": 0.0007500201463699341,
198
- "tpp_threshold_5_total_metric": 0.044249966740608215,
199
- "tpp_threshold_5_intended_diff_only": 0.05699998140335083,
200
- "tpp_threshold_5_unintended_diff_only": 0.012750014662742615,
201
- "tpp_threshold_10_total_metric": 0.06199999153614044,
202
  "tpp_threshold_10_intended_diff_only": 0.078000009059906,
203
- "tpp_threshold_10_unintended_diff_only": 0.016000017523765564
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.020999997854232788,
207
- "tpp_threshold_2_intended_diff_only": 0.022000014781951904,
208
- "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162,
209
- "tpp_threshold_5_total_metric": 0.048250019550323486,
210
  "tpp_threshold_5_intended_diff_only": 0.054000020027160645,
211
- "tpp_threshold_5_unintended_diff_only": 0.005750000476837158,
212
- "tpp_threshold_10_total_metric": 0.216000035405159,
213
- "tpp_threshold_10_intended_diff_only": 0.27400004863739014,
214
- "tpp_threshold_10_unintended_diff_only": 0.05800001323223114
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.01600000262260437,
218
  "tpp_threshold_2_intended_diff_only": 0.013999998569488525,
219
- "tpp_threshold_2_unintended_diff_only": -0.0020000040531158447,
220
- "tpp_threshold_5_total_metric": 0.02374999225139618,
221
- "tpp_threshold_5_intended_diff_only": 0.03700000047683716,
222
- "tpp_threshold_5_unintended_diff_only": 0.013250008225440979,
223
- "tpp_threshold_10_total_metric": 0.30124999582767487,
224
  "tpp_threshold_10_intended_diff_only": 0.328000009059906,
225
- "tpp_threshold_10_unintended_diff_only": 0.02675001323223114
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.011750012636184692,
229
- "tpp_threshold_2_intended_diff_only": 0.013000011444091797,
230
- "tpp_threshold_2_unintended_diff_only": 0.0012499988079071045,
231
- "tpp_threshold_5_total_metric": 0.10749998688697815,
232
  "tpp_threshold_5_intended_diff_only": 0.12000000476837158,
233
- "tpp_threshold_5_unintended_diff_only": 0.012500017881393433,
234
- "tpp_threshold_10_total_metric": 0.2382500320672989,
235
- "tpp_threshold_10_intended_diff_only": 0.3160000443458557,
236
- "tpp_threshold_10_unintended_diff_only": 0.07775001227855682
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.012500002980232239,
242
- "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
243
- "tpp_threshold_2_unintended_diff_only": -0.0034999996423721313,
244
- "tpp_threshold_5_total_metric": 0.014749974012374878,
245
- "tpp_threshold_5_intended_diff_only": 0.015999972820281982,
246
- "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045,
247
- "tpp_threshold_10_total_metric": 0.021500006318092346,
248
- "tpp_threshold_10_intended_diff_only": 0.03100001811981201,
249
- "tpp_threshold_10_unintended_diff_only": 0.009500011801719666
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": 0.004750043153762817,
253
- "tpp_threshold_2_intended_diff_only": 0.01100003719329834,
254
- "tpp_threshold_2_unintended_diff_only": 0.0062499940395355225,
255
- "tpp_threshold_5_total_metric": 0.02850005030632019,
256
- "tpp_threshold_5_intended_diff_only": 0.03800004720687866,
257
- "tpp_threshold_5_unintended_diff_only": 0.009499996900558472,
258
- "tpp_threshold_10_total_metric": 0.04950001835823059,
259
- "tpp_threshold_10_intended_diff_only": 0.05500000715255737,
260
- "tpp_threshold_10_unintended_diff_only": 0.005499988794326782
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.015750005841255188,
264
- "tpp_threshold_2_intended_diff_only": 0.013999998569488525,
265
- "tpp_threshold_2_unintended_diff_only": -0.0017500072717666626,
266
- "tpp_threshold_5_total_metric": 0.021750018000602722,
267
- "tpp_threshold_5_intended_diff_only": 0.023000001907348633,
268
- "tpp_threshold_5_unintended_diff_only": 0.0012499839067459106,
269
- "tpp_threshold_10_total_metric": 0.01875002682209015,
270
- "tpp_threshold_10_intended_diff_only": 0.026000022888183594,
271
- "tpp_threshold_10_unintended_diff_only": 0.007249996066093445
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.003000006079673767,
275
- "tpp_threshold_2_intended_diff_only": 0.0059999823570251465,
276
- "tpp_threshold_2_unintended_diff_only": 0.008999988436698914,
277
- "tpp_threshold_5_total_metric": 0.016249969601631165,
278
- "tpp_threshold_5_intended_diff_only": 0.019999980926513672,
279
- "tpp_threshold_5_unintended_diff_only": 0.0037500113248825073,
280
- "tpp_threshold_10_total_metric": 0.02449999749660492,
281
- "tpp_threshold_10_intended_diff_only": 0.02799999713897705,
282
- "tpp_threshold_10_unintended_diff_only": 0.0034999996423721313
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.04899996519088745,
286
- "tpp_threshold_2_intended_diff_only": 0.04899996519088745,
287
- "tpp_threshold_2_unintended_diff_only": 0.0,
288
- "tpp_threshold_5_total_metric": 0.07624995708465576,
289
- "tpp_threshold_5_intended_diff_only": 0.08499997854232788,
290
- "tpp_threshold_5_unintended_diff_only": 0.00875002145767212,
291
- "tpp_threshold_10_total_metric": 0.12974996864795685,
292
- "tpp_threshold_10_intended_diff_only": 0.1369999647140503,
293
- "tpp_threshold_10_unintended_diff_only": 0.007249996066093445
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "c9af90f5-0d76-4153-b35d-e71d8540b778",
73
+ "datetime_epoch_millis": 1777315179100,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.012924993038177492,
77
+ "tpp_threshold_2_intended_diff_only": 0.01510000228881836,
78
+ "tpp_threshold_2_unintended_diff_only": 0.0021750092506408695,
79
+ "tpp_threshold_5_total_metric": 0.040825004875659945,
80
+ "tpp_threshold_5_intended_diff_only": 0.04670000672340393,
81
+ "tpp_threshold_5_unintended_diff_only": 0.005875001847743988,
82
+ "tpp_threshold_10_total_metric": 0.11460000276565552,
83
+ "tpp_threshold_10_intended_diff_only": 0.13590000867843627,
84
+ "tpp_threshold_10_unintended_diff_only": 0.02130000591278076,
85
+ "tpp_threshold_20_total_metric": 0.18427502512931823,
86
+ "tpp_threshold_20_intended_diff_only": 0.231900030374527,
87
+ "tpp_threshold_20_unintended_diff_only": 0.04762500524520874,
88
+ "tpp_threshold_50_total_metric": 0.23530001342296603,
89
+ "tpp_threshold_50_intended_diff_only": 0.3132000207901001,
90
+ "tpp_threshold_50_unintended_diff_only": 0.0779000073671341,
91
+ "tpp_threshold_100_total_metric": 0.27075001746416094,
92
+ "tpp_threshold_100_intended_diff_only": 0.3764000296592712,
93
+ "tpp_threshold_100_unintended_diff_only": 0.10565001219511033,
94
+ "tpp_threshold_500_total_metric": 0.25757503062486653,
95
+ "tpp_threshold_500_intended_diff_only": 0.4451000392436981,
96
+ "tpp_threshold_500_unintended_diff_only": 0.18752500861883165
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.01004999279975891,
103
+ "tpp_threshold_2_intended_diff_only": 0.01220000982284546,
104
+ "tpp_threshold_2_unintended_diff_only": 0.002150017023086548,
105
+ "tpp_threshold_5_total_metric": 0.049700000882148744,
106
+ "tpp_threshold_5_intended_diff_only": 0.05840001106262207,
107
+ "tpp_threshold_5_unintended_diff_only": 0.008700010180473328,
108
+ "tpp_threshold_10_total_metric": 0.17580000758171083,
109
  "tpp_threshold_10_intended_diff_only": 0.21220002174377442,
110
+ "tpp_threshold_10_unintended_diff_only": 0.0364000141620636,
111
+ "tpp_threshold_20_total_metric": 0.2883000373840332,
112
+ "tpp_threshold_20_intended_diff_only": 0.3756000518798828,
113
+ "tpp_threshold_20_unintended_diff_only": 0.0873000144958496,
114
+ "tpp_threshold_50_total_metric": 0.29470000565052035,
115
+ "tpp_threshold_50_intended_diff_only": 0.4412000298500061,
116
+ "tpp_threshold_50_unintended_diff_only": 0.14650002419948577,
117
+ "tpp_threshold_100_total_metric": 0.26705001294612885,
118
+ "tpp_threshold_100_intended_diff_only": 0.46100003719329835,
119
+ "tpp_threshold_100_unintended_diff_only": 0.1939500242471695,
120
+ "tpp_threshold_500_total_metric": 0.13205003440380098,
121
+ "tpp_threshold_500_intended_diff_only": 0.46780005693435667,
122
+ "tpp_threshold_500_unintended_diff_only": 0.33575002253055575
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.01579999327659607,
127
+ "tpp_threshold_2_intended_diff_only": 0.01799999475479126,
128
+ "tpp_threshold_2_unintended_diff_only": 0.0022000014781951905,
129
+ "tpp_threshold_5_total_metric": 0.03195000886917114,
130
+ "tpp_threshold_5_intended_diff_only": 0.03500000238418579,
131
+ "tpp_threshold_5_unintended_diff_only": 0.0030499935150146485,
132
+ "tpp_threshold_10_total_metric": 0.05339999794960022,
133
+ "tpp_threshold_10_intended_diff_only": 0.059599995613098145,
134
+ "tpp_threshold_10_unintended_diff_only": 0.006199997663497925,
135
+ "tpp_threshold_20_total_metric": 0.08025001287460327,
136
+ "tpp_threshold_20_intended_diff_only": 0.08820000886917115,
137
+ "tpp_threshold_20_unintended_diff_only": 0.00794999599456787,
138
+ "tpp_threshold_50_total_metric": 0.17590002119541168,
139
+ "tpp_threshold_50_intended_diff_only": 0.1852000117301941,
140
+ "tpp_threshold_50_unintended_diff_only": 0.00929999053478241,
141
+ "tpp_threshold_100_total_metric": 0.274450021982193,
142
+ "tpp_threshold_100_intended_diff_only": 0.29180002212524414,
143
+ "tpp_threshold_100_unintended_diff_only": 0.017350000143051148,
144
+ "tpp_threshold_500_total_metric": 0.38310002684593203,
145
+ "tpp_threshold_500_intended_diff_only": 0.4224000215530396,
146
+ "tpp_threshold_500_unintended_diff_only": 0.039299994707107544
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.0022500306367874146,
194
  "tpp_threshold_10_total_metric": 0.05924996733665466,
195
  "tpp_threshold_10_intended_diff_only": 0.06499999761581421,
196
+ "tpp_threshold_10_unintended_diff_only": 0.005750030279159546,
197
+ "tpp_threshold_20_total_metric": 0.24324999749660492,
198
+ "tpp_threshold_20_intended_diff_only": 0.3270000219345093,
199
+ "tpp_threshold_20_unintended_diff_only": 0.08375002443790436,
200
+ "tpp_threshold_50_total_metric": 0.2789999693632126,
201
+ "tpp_threshold_50_intended_diff_only": 0.4350000023841858,
202
+ "tpp_threshold_50_unintended_diff_only": 0.1560000330209732,
203
+ "tpp_threshold_100_total_metric": 0.2512499839067459,
204
+ "tpp_threshold_100_intended_diff_only": 0.4520000219345093,
205
+ "tpp_threshold_100_unintended_diff_only": 0.20075003802776337,
206
+ "tpp_threshold_500_total_metric": 0.1184999942779541,
207
+ "tpp_threshold_500_intended_diff_only": 0.45600003004074097,
208
+ "tpp_threshold_500_unintended_diff_only": 0.33750003576278687
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.0029999911785125732,
212
+ "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
213
+ "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162,
214
+ "tpp_threshold_5_total_metric": 0.046499982476234436,
215
+ "tpp_threshold_5_intended_diff_only": 0.0559999942779541,
216
+ "tpp_threshold_5_unintended_diff_only": 0.009500011801719666,
217
+ "tpp_threshold_10_total_metric": 0.06549999117851257,
218
  "tpp_threshold_10_intended_diff_only": 0.078000009059906,
219
+ "tpp_threshold_10_unintended_diff_only": 0.012500017881393433,
220
+ "tpp_threshold_20_total_metric": 0.29750002920627594,
221
+ "tpp_threshold_20_intended_diff_only": 0.34400004148483276,
222
+ "tpp_threshold_20_unintended_diff_only": 0.046500012278556824,
223
+ "tpp_threshold_50_total_metric": 0.31174999475479126,
224
+ "tpp_threshold_50_intended_diff_only": 0.42500001192092896,
225
+ "tpp_threshold_50_unintended_diff_only": 0.1132500171661377,
226
+ "tpp_threshold_100_total_metric": 0.28724998235702515,
227
+ "tpp_threshold_100_intended_diff_only": 0.4520000219345093,
228
+ "tpp_threshold_100_unintended_diff_only": 0.16475003957748413,
229
+ "tpp_threshold_500_total_metric": 0.12150003015995026,
230
+ "tpp_threshold_500_intended_diff_only": 0.4590000510215759,
231
+ "tpp_threshold_500_unintended_diff_only": 0.33750002086162567
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.010499998927116394,
235
+ "tpp_threshold_2_intended_diff_only": 0.018000006675720215,
236
+ "tpp_threshold_2_unintended_diff_only": 0.007500007748603821,
237
+ "tpp_threshold_5_total_metric": 0.047500014305114746,
238
  "tpp_threshold_5_intended_diff_only": 0.054000020027160645,
239
+ "tpp_threshold_5_unintended_diff_only": 0.0065000057220458984,
240
+ "tpp_threshold_10_total_metric": 0.2172500342130661,
241
+ "tpp_threshold_10_intended_diff_only": 0.27500003576278687,
242
+ "tpp_threshold_10_unintended_diff_only": 0.057750001549720764,
243
+ "tpp_threshold_20_total_metric": 0.19175004959106445,
244
+ "tpp_threshold_20_intended_diff_only": 0.36100006103515625,
245
+ "tpp_threshold_20_unintended_diff_only": 0.1692500114440918,
246
+ "tpp_threshold_50_total_metric": 0.19675001502037048,
247
+ "tpp_threshold_50_intended_diff_only": 0.40400004386901855,
248
+ "tpp_threshold_50_unintended_diff_only": 0.20725002884864807,
249
+ "tpp_threshold_100_total_metric": 0.19050003588199615,
250
+ "tpp_threshold_100_intended_diff_only": 0.4360000491142273,
251
+ "tpp_threshold_100_unintended_diff_only": 0.24550001323223114,
252
+ "tpp_threshold_500_total_metric": 0.08900004625320435,
253
+ "tpp_threshold_500_intended_diff_only": 0.4530000686645508,
254
+ "tpp_threshold_500_unintended_diff_only": 0.36400002241134644
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.013999983668327332,
258
  "tpp_threshold_2_intended_diff_only": 0.013999998569488525,
259
+ "tpp_threshold_2_unintended_diff_only": 1.4901161193847656e-08,
260
+ "tpp_threshold_5_total_metric": 0.028000041842460632,
261
+ "tpp_threshold_5_intended_diff_only": 0.03800004720687866,
262
+ "tpp_threshold_5_unintended_diff_only": 0.01000000536441803,
263
+ "tpp_threshold_10_total_metric": 0.3007500022649765,
264
  "tpp_threshold_10_intended_diff_only": 0.328000009059906,
265
+ "tpp_threshold_10_unintended_diff_only": 0.027250006794929504,
266
+ "tpp_threshold_20_total_metric": 0.41175004839897156,
267
+ "tpp_threshold_20_intended_diff_only": 0.4580000638961792,
268
+ "tpp_threshold_20_unintended_diff_only": 0.04625001549720764,
269
+ "tpp_threshold_50_total_metric": 0.36375002562999725,
270
+ "tpp_threshold_50_intended_diff_only": 0.48200005292892456,
271
+ "tpp_threshold_50_unintended_diff_only": 0.11825002729892731,
272
+ "tpp_threshold_100_total_metric": 0.33800001442432404,
273
+ "tpp_threshold_100_intended_diff_only": 0.48900002241134644,
274
+ "tpp_threshold_100_unintended_diff_only": 0.1510000079870224,
275
+ "tpp_threshold_500_total_metric": 0.2055000513792038,
276
+ "tpp_threshold_500_intended_diff_only": 0.49500006437301636,
277
+ "tpp_threshold_500_unintended_diff_only": 0.28950001299381256
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.014000028371810913,
281
+ "tpp_threshold_2_intended_diff_only": 0.01500004529953003,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162,
283
+ "tpp_threshold_5_total_metric": 0.10475000739097595,
284
  "tpp_threshold_5_intended_diff_only": 0.12000000476837158,
285
+ "tpp_threshold_5_unintended_diff_only": 0.01524999737739563,
286
+ "tpp_threshold_10_total_metric": 0.23625004291534424,
287
+ "tpp_threshold_10_intended_diff_only": 0.315000057220459,
288
+ "tpp_threshold_10_unintended_diff_only": 0.07875001430511475,
289
+ "tpp_threshold_20_total_metric": 0.29725006222724915,
290
+ "tpp_threshold_20_intended_diff_only": 0.3880000710487366,
291
+ "tpp_threshold_20_unintended_diff_only": 0.09075000882148743,
292
+ "tpp_threshold_50_total_metric": 0.32225002348423004,
293
+ "tpp_threshold_50_intended_diff_only": 0.46000003814697266,
294
+ "tpp_threshold_50_unintended_diff_only": 0.13775001466274261,
295
+ "tpp_threshold_100_total_metric": 0.268250048160553,
296
+ "tpp_threshold_100_intended_diff_only": 0.4760000705718994,
297
+ "tpp_threshold_100_unintended_diff_only": 0.20775002241134644,
298
+ "tpp_threshold_500_total_metric": 0.12575004994869232,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.3502500206232071
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.009499996900558472,
306
+ "tpp_threshold_2_intended_diff_only": 0.013999998569488525,
307
+ "tpp_threshold_2_unintended_diff_only": 0.004500001668930054,
308
+ "tpp_threshold_5_total_metric": 0.009749993681907654,
309
+ "tpp_threshold_5_intended_diff_only": 0.014999985694885254,
310
+ "tpp_threshold_5_unintended_diff_only": 0.0052499920129776,
311
+ "tpp_threshold_10_total_metric": 0.02649998664855957,
312
+ "tpp_threshold_10_intended_diff_only": 0.03299999237060547,
313
+ "tpp_threshold_10_unintended_diff_only": 0.0065000057220458984,
314
+ "tpp_threshold_20_total_metric": 0.06700003147125244,
315
+ "tpp_threshold_20_intended_diff_only": 0.07200002670288086,
316
+ "tpp_threshold_20_unintended_diff_only": 0.004999995231628418,
317
+ "tpp_threshold_50_total_metric": 0.19675002992153168,
318
+ "tpp_threshold_50_intended_diff_only": 0.1980000138282776,
319
+ "tpp_threshold_50_unintended_diff_only": 0.0012499839067459106,
320
+ "tpp_threshold_100_total_metric": 0.30775003135204315,
321
+ "tpp_threshold_100_intended_diff_only": 0.32200002670288086,
322
+ "tpp_threshold_100_unintended_diff_only": 0.014249995350837708,
323
+ "tpp_threshold_500_total_metric": 0.4332500547170639,
324
+ "tpp_threshold_500_intended_diff_only": 0.45500004291534424,
325
+ "tpp_threshold_500_unintended_diff_only": 0.021749988198280334
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.01475001871585846,
329
+ "tpp_threshold_2_intended_diff_only": 0.013000011444091797,
330
+ "tpp_threshold_2_unintended_diff_only": -0.0017500072717666626,
331
+ "tpp_threshold_5_total_metric": 0.026750028133392334,
332
+ "tpp_threshold_5_intended_diff_only": 0.0350000262260437,
333
+ "tpp_threshold_5_unintended_diff_only": 0.008249998092651367,
334
+ "tpp_threshold_10_total_metric": 0.0492500364780426,
335
+ "tpp_threshold_10_intended_diff_only": 0.058000028133392334,
336
+ "tpp_threshold_10_unintended_diff_only": 0.008749991655349731,
337
+ "tpp_threshold_20_total_metric": 0.07200005650520325,
338
+ "tpp_threshold_20_intended_diff_only": 0.08000004291534424,
339
+ "tpp_threshold_20_unintended_diff_only": 0.007999986410140991,
340
+ "tpp_threshold_50_total_metric": 0.16300006210803986,
341
+ "tpp_threshold_50_intended_diff_only": 0.1770000457763672,
342
+ "tpp_threshold_50_unintended_diff_only": 0.013999983668327332,
343
+ "tpp_threshold_100_total_metric": 0.28700006008148193,
344
+ "tpp_threshold_100_intended_diff_only": 0.3060000538825989,
345
+ "tpp_threshold_100_unintended_diff_only": 0.018999993801116943,
346
+ "tpp_threshold_500_total_metric": 0.40050002932548523,
347
+ "tpp_threshold_500_intended_diff_only": 0.43800002336502075,
348
+ "tpp_threshold_500_unintended_diff_only": 0.03749999403953552
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.010500013828277588,
352
+ "tpp_threshold_2_intended_diff_only": 0.013000011444091797,
353
+ "tpp_threshold_2_unintended_diff_only": 0.002499997615814209,
354
+ "tpp_threshold_5_total_metric": 0.01175004243850708,
355
+ "tpp_threshold_5_intended_diff_only": 0.012000024318695068,
356
+ "tpp_threshold_5_unintended_diff_only": 0.0002499818801879883,
357
+ "tpp_threshold_10_total_metric": 0.026749998331069946,
358
+ "tpp_threshold_10_intended_diff_only": 0.02899998426437378,
359
+ "tpp_threshold_10_unintended_diff_only": 0.002249985933303833,
360
+ "tpp_threshold_20_total_metric": 0.026249989867210388,
361
+ "tpp_threshold_20_intended_diff_only": 0.03299999237060547,
362
+ "tpp_threshold_20_unintended_diff_only": 0.006750002503395081,
363
+ "tpp_threshold_50_total_metric": 0.1250000298023224,
364
+ "tpp_threshold_50_intended_diff_only": 0.13300001621246338,
365
+ "tpp_threshold_50_unintended_diff_only": 0.007999986410140991,
366
+ "tpp_threshold_100_total_metric": 0.22925002872943878,
367
+ "tpp_threshold_100_intended_diff_only": 0.24700003862380981,
368
+ "tpp_threshold_100_unintended_diff_only": 0.017750009894371033,
369
+ "tpp_threshold_500_total_metric": 0.3862500488758087,
370
+ "tpp_threshold_500_intended_diff_only": 0.4270000457763672,
371
+ "tpp_threshold_500_unintended_diff_only": 0.04074999690055847
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.007249936461448669,
375
+ "tpp_threshold_2_intended_diff_only": 0.007999956607818604,
376
+ "tpp_threshold_2_unintended_diff_only": 0.0007500201463699341,
377
+ "tpp_threshold_5_total_metric": 0.030749961733818054,
378
+ "tpp_threshold_5_intended_diff_only": 0.030999958515167236,
379
+ "tpp_threshold_5_unintended_diff_only": 0.00024999678134918213,
380
+ "tpp_threshold_10_total_metric": 0.025249958038330078,
381
+ "tpp_threshold_10_intended_diff_only": 0.030999958515167236,
382
+ "tpp_threshold_10_unintended_diff_only": 0.005750000476837158,
383
+ "tpp_threshold_20_total_metric": 0.04999998211860657,
384
+ "tpp_threshold_20_intended_diff_only": 0.05299997329711914,
385
+ "tpp_threshold_20_unintended_diff_only": 0.0029999911785125732,
386
+ "tpp_threshold_50_total_metric": 0.10349997878074646,
387
+ "tpp_threshold_50_intended_diff_only": 0.1119999885559082,
388
+ "tpp_threshold_50_unintended_diff_only": 0.008500009775161743,
389
+ "tpp_threshold_100_total_metric": 0.21199999749660492,
390
+ "tpp_threshold_100_intended_diff_only": 0.22200000286102295,
391
+ "tpp_threshold_100_unintended_diff_only": 0.01000000536441803,
392
+ "tpp_threshold_500_total_metric": 0.37424996495246887,
393
+ "tpp_threshold_500_intended_diff_only": 0.41899996995925903,
394
+ "tpp_threshold_500_unintended_diff_only": 0.04475000500679016
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.03700000047683716,
398
+ "tpp_threshold_2_intended_diff_only": 0.041999995708465576,
399
+ "tpp_threshold_2_unintended_diff_only": 0.004999995231628418,
400
+ "tpp_threshold_5_total_metric": 0.08075001835823059,
401
+ "tpp_threshold_5_intended_diff_only": 0.0820000171661377,
402
+ "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045,
403
+ "tpp_threshold_10_total_metric": 0.1392500102519989,
404
+ "tpp_threshold_10_intended_diff_only": 0.1470000147819519,
405
+ "tpp_threshold_10_unintended_diff_only": 0.007750004529953003,
406
+ "tpp_threshold_20_total_metric": 0.1860000044107437,
407
+ "tpp_threshold_20_intended_diff_only": 0.203000009059906,
408
+ "tpp_threshold_20_unintended_diff_only": 0.017000004649162292,
409
+ "tpp_threshold_50_total_metric": 0.29125000536441803,
410
+ "tpp_threshold_50_intended_diff_only": 0.3059999942779541,
411
+ "tpp_threshold_50_unintended_diff_only": 0.014749988913536072,
412
+ "tpp_threshold_100_total_metric": 0.3362499922513962,
413
+ "tpp_threshold_100_intended_diff_only": 0.3619999885559082,
414
+ "tpp_threshold_100_unintended_diff_only": 0.025749996304512024,
415
+ "tpp_threshold_500_total_metric": 0.3212500363588333,
416
+ "tpp_threshold_500_intended_diff_only": 0.37300002574920654,
417
+ "tpp_threshold_500_unintended_diff_only": 0.05174998939037323
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-1-tokens-0/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "88ce73c6-9294-4e50-9eca-f41dc29bbd93",
69
- "datetime_epoch_millis": 1776883332532,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.002375009655952454,
73
- "tpp_threshold_2_intended_diff_only": 0.0010000109672546388,
74
- "tpp_threshold_2_unintended_diff_only": -0.001374998688697815,
75
- "tpp_threshold_5_total_metric": -0.0002499982714653015,
76
- "tpp_threshold_5_intended_diff_only": 0.0005000054836273194,
77
- "tpp_threshold_5_unintended_diff_only": 0.0007500037550926209,
78
- "tpp_threshold_10_total_metric": -0.0014749899506568908,
79
- "tpp_threshold_10_intended_diff_only": -0.0011999845504760743,
80
- "tpp_threshold_10_unintended_diff_only": 0.0002750054001808167,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.002300018072128296,
99
- "tpp_threshold_2_intended_diff_only": 0.0008000254631042481,
100
- "tpp_threshold_2_unintended_diff_only": -0.001499992609024048,
101
- "tpp_threshold_5_total_metric": -0.0015500009059906006,
102
  "tpp_threshold_5_intended_diff_only": -0.0007999897003173828,
103
- "tpp_threshold_5_unintended_diff_only": 0.0007500112056732178,
104
- "tpp_threshold_10_total_metric": -0.0018499940633773804,
105
- "tpp_threshold_10_intended_diff_only": -0.0015999794006347657,
106
- "tpp_threshold_10_unintended_diff_only": 0.00025001466274261476,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.002450001239776611,
123
- "tpp_threshold_2_intended_diff_only": 0.0011999964714050292,
124
- "tpp_threshold_2_unintended_diff_only": -0.001250004768371582,
125
- "tpp_threshold_5_total_metric": 0.0010500043630599976,
126
- "tpp_threshold_5_intended_diff_only": 0.0018000006675720215,
127
- "tpp_threshold_5_unintended_diff_only": 0.000749996304512024,
128
- "tpp_threshold_10_total_metric": -0.0010999858379364014,
129
- "tpp_threshold_10_intended_diff_only": -0.0007999897003173828,
130
- "tpp_threshold_10_unintended_diff_only": 0.0002999961376190186,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.000250011682510376,
190
  "tpp_threshold_10_total_metric": -0.004500031471252441,
191
  "tpp_threshold_10_intended_diff_only": -0.0040000081062316895,
192
- "tpp_threshold_10_unintended_diff_only": 0.000500023365020752
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": -0.001749977469444275,
196
- "tpp_threshold_2_intended_diff_only": -0.001999974250793457,
197
- "tpp_threshold_2_unintended_diff_only": -0.00024999678134918213,
198
- "tpp_threshold_5_total_metric": -0.0010000169277191162,
199
- "tpp_threshold_5_intended_diff_only": 0.0,
200
- "tpp_threshold_5_unintended_diff_only": 0.0010000169277191162,
201
- "tpp_threshold_10_total_metric": -0.007499992847442627,
202
- "tpp_threshold_10_intended_diff_only": -0.0059999823570251465,
203
- "tpp_threshold_10_unintended_diff_only": 0.0015000104904174805
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.005750030279159546,
207
- "tpp_threshold_2_intended_diff_only": 0.0020000338554382324,
208
- "tpp_threshold_2_unintended_diff_only": -0.0037499964237213135,
209
- "tpp_threshold_5_total_metric": 0.0007499903440475464,
210
- "tpp_threshold_5_intended_diff_only": 0.0,
211
- "tpp_threshold_5_unintended_diff_only": -0.0007499903440475464,
212
- "tpp_threshold_10_total_metric": 0.0045000165700912476,
213
- "tpp_threshold_10_intended_diff_only": 0.0020000338554382324,
214
- "tpp_threshold_10_unintended_diff_only": -0.002499982714653015
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.004000052809715271,
218
  "tpp_threshold_2_intended_diff_only": 0.001000046730041504,
219
- "tpp_threshold_2_unintended_diff_only": -0.003000006079673767,
220
- "tpp_threshold_5_total_metric": -0.005249977111816406,
221
- "tpp_threshold_5_intended_diff_only": -0.0029999613761901855,
222
- "tpp_threshold_5_unintended_diff_only": 0.0022500157356262207,
223
- "tpp_threshold_10_total_metric": 2.9802322387695312e-08,
224
- "tpp_threshold_10_intended_diff_only": 0.001000046730041504,
225
- "tpp_threshold_10_unintended_diff_only": 0.0010000169277191162
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": -0.0010000020265579224,
229
- "tpp_threshold_2_intended_diff_only": -0.0009999871253967285,
230
- "tpp_threshold_2_unintended_diff_only": 1.4901161193847656e-08,
231
- "tpp_threshold_5_total_metric": -0.00599999725818634,
232
- "tpp_threshold_5_intended_diff_only": -0.004999995231628418,
233
- "tpp_threshold_5_unintended_diff_only": 0.0010000020265579224,
234
- "tpp_threshold_10_total_metric": -0.0017499923706054688,
235
- "tpp_threshold_10_intended_diff_only": -0.0009999871253967285,
236
- "tpp_threshold_10_unintended_diff_only": 0.0007500052452087402
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.006999984383583069,
242
- "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
243
- "tpp_threshold_2_unintended_diff_only": -0.005000010132789612,
244
- "tpp_threshold_5_total_metric": 0.00899997353553772,
245
- "tpp_threshold_5_intended_diff_only": 0.006999969482421875,
246
- "tpp_threshold_5_unintended_diff_only": -0.0020000040531158447,
247
- "tpp_threshold_10_total_metric": -0.007749974727630615,
248
- "tpp_threshold_10_intended_diff_only": -0.004999995231628418,
249
- "tpp_threshold_10_unintended_diff_only": 0.0027499794960021973
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": -0.0057499706745147705,
253
- "tpp_threshold_2_intended_diff_only": -0.001999974250793457,
254
- "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135,
255
- "tpp_threshold_5_total_metric": -0.007999926805496216,
256
- "tpp_threshold_5_intended_diff_only": -0.003999948501586914,
257
- "tpp_threshold_5_unintended_diff_only": 0.003999978303909302,
258
- "tpp_threshold_10_total_metric": -0.0009999573230743408,
259
- "tpp_threshold_10_intended_diff_only": -0.0029999613761901855,
260
- "tpp_threshold_10_unintended_diff_only": -0.0020000040531158447
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.010000035166740417,
264
- "tpp_threshold_2_intended_diff_only": 0.003000020980834961,
265
- "tpp_threshold_2_unintended_diff_only": -0.0070000141859054565,
266
- "tpp_threshold_5_total_metric": 0.014250025153160095,
267
- "tpp_threshold_5_intended_diff_only": 0.012000024318695068,
268
- "tpp_threshold_5_unintended_diff_only": -0.002250000834465027,
269
- "tpp_threshold_10_total_metric": 0.011250019073486328,
270
- "tpp_threshold_10_intended_diff_only": 0.012000024318695068,
271
- "tpp_threshold_10_unintended_diff_only": 0.0007500052452087402
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.006250053644180298,
275
- "tpp_threshold_2_intended_diff_only": -0.001000046730041504,
276
- "tpp_threshold_2_unintended_diff_only": 0.005250006914138794,
277
- "tpp_threshold_5_total_metric": -0.005250066518783569,
278
- "tpp_threshold_5_intended_diff_only": -0.005000054836273193,
279
- "tpp_threshold_5_unintended_diff_only": 0.000250011682510376,
280
- "tpp_threshold_10_total_metric": -0.005000010132789612,
281
- "tpp_threshold_10_intended_diff_only": -0.0040000081062316895,
282
- "tpp_threshold_10_unintended_diff_only": 0.0010000020265579224
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.007250010967254639,
286
- "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
287
- "tpp_threshold_2_unintended_diff_only": -0.0032500028610229492,
288
- "tpp_threshold_5_total_metric": -0.004749983549118042,
289
- "tpp_threshold_5_intended_diff_only": -0.0009999871253967285,
290
- "tpp_threshold_5_unintended_diff_only": 0.0037499964237213135,
291
- "tpp_threshold_10_total_metric": -0.003000006079673767,
292
- "tpp_threshold_10_intended_diff_only": -0.0040000081062316895,
293
- "tpp_threshold_10_unintended_diff_only": -0.0010000020265579224
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "e1af4f20-5d92-44a9-9bac-6f65ac766765",
73
+ "datetime_epoch_millis": 1777315836333,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.0010000094771385193,
77
+ "tpp_threshold_2_intended_diff_only": 0.000800013542175293,
78
+ "tpp_threshold_2_unintended_diff_only": -0.00019999593496322632,
79
+ "tpp_threshold_5_total_metric": -2.9802322387803733e-09,
80
+ "tpp_threshold_5_intended_diff_only": -0.0007999956607818604,
81
+ "tpp_threshold_5_unintended_diff_only": -0.0007999926805496216,
82
+ "tpp_threshold_10_total_metric": 0.0002999991178512574,
83
+ "tpp_threshold_10_intended_diff_only": 0.0008000016212463378,
84
+ "tpp_threshold_10_unintended_diff_only": 0.0005000025033950805,
85
+ "tpp_threshold_20_total_metric": 0.0010500028729438782,
86
+ "tpp_threshold_20_intended_diff_only": 0.0012000083923339843,
87
+ "tpp_threshold_20_unintended_diff_only": 0.0001500055193901062,
88
+ "tpp_threshold_50_total_metric": 0.003299999237060547,
89
+ "tpp_threshold_50_intended_diff_only": 0.0024000048637390133,
90
+ "tpp_threshold_50_unintended_diff_only": -0.0008999943733215332,
91
+ "tpp_threshold_100_total_metric": 0.0006500110030174255,
92
+ "tpp_threshold_100_intended_diff_only": 0.0023000121116638185,
93
+ "tpp_threshold_100_unintended_diff_only": 0.0016500011086463928,
94
+ "tpp_threshold_500_total_metric": 0.027400009334087372,
95
+ "tpp_threshold_500_intended_diff_only": 0.028900015354156493,
96
+ "tpp_threshold_500_unintended_diff_only": 0.0015000060200691223
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.0014000117778778076,
103
+ "tpp_threshold_2_intended_diff_only": 0.0012000203132629395,
104
+ "tpp_threshold_2_unintended_diff_only": -0.00019999146461486817,
105
+ "tpp_threshold_5_total_metric": -0.00030000805854797364,
106
  "tpp_threshold_5_intended_diff_only": -0.0007999897003173828,
107
+ "tpp_threshold_5_unintended_diff_only": -0.0004999816417694091,
108
+ "tpp_threshold_10_total_metric": -0.0014500021934509277,
109
+ "tpp_threshold_10_intended_diff_only": -0.001399993896484375,
110
+ "tpp_threshold_10_unintended_diff_only": 5.0008296966552734e-05,
111
+ "tpp_threshold_20_total_metric": 0.0032000094652175903,
112
+ "tpp_threshold_20_intended_diff_only": 0.002800023555755615,
113
+ "tpp_threshold_20_unintended_diff_only": -0.0003999859094619751,
114
+ "tpp_threshold_50_total_metric": 0.002300000190734863,
115
+ "tpp_threshold_50_intended_diff_only": 0.002000010013580322,
116
+ "tpp_threshold_50_unintended_diff_only": -0.000299990177154541,
117
+ "tpp_threshold_100_total_metric": -0.0005499899387359619,
118
+ "tpp_threshold_100_intended_diff_only": 0.00020002126693725587,
119
+ "tpp_threshold_100_unintended_diff_only": 0.0007500112056732178,
120
+ "tpp_threshold_500_total_metric": 0.011700013279914856,
121
+ "tpp_threshold_500_intended_diff_only": 0.011800026893615723,
122
+ "tpp_threshold_500_unintended_diff_only": 0.0001000136137008667
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.000600007176399231,
127
+ "tpp_threshold_2_intended_diff_only": 0.0004000067710876465,
128
+ "tpp_threshold_2_unintended_diff_only": -0.00020000040531158446,
129
+ "tpp_threshold_5_total_metric": 0.0003000020980834961,
130
+ "tpp_threshold_5_intended_diff_only": -0.0008000016212463378,
131
+ "tpp_threshold_5_unintended_diff_only": -0.001100003719329834,
132
+ "tpp_threshold_10_total_metric": 0.0020500004291534426,
133
+ "tpp_threshold_10_intended_diff_only": 0.0029999971389770507,
134
+ "tpp_threshold_10_unintended_diff_only": 0.0009499967098236084,
135
+ "tpp_threshold_20_total_metric": -0.001100003719329834,
136
+ "tpp_threshold_20_intended_diff_only": -0.0004000067710876465,
137
+ "tpp_threshold_20_unintended_diff_only": 0.0006999969482421875,
138
+ "tpp_threshold_50_total_metric": 0.0042999982833862305,
139
+ "tpp_threshold_50_intended_diff_only": 0.002799999713897705,
140
+ "tpp_threshold_50_unintended_diff_only": -0.0014999985694885253,
141
+ "tpp_threshold_100_total_metric": 0.001850011944770813,
142
+ "tpp_threshold_100_intended_diff_only": 0.004400002956390381,
143
+ "tpp_threshold_100_unintended_diff_only": 0.002549991011619568,
144
+ "tpp_threshold_500_total_metric": 0.043100005388259886,
145
+ "tpp_threshold_500_intended_diff_only": 0.046000003814697266,
146
+ "tpp_threshold_500_unintended_diff_only": 0.002899998426437378
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.000250011682510376,
194
  "tpp_threshold_10_total_metric": -0.004500031471252441,
195
  "tpp_threshold_10_intended_diff_only": -0.0040000081062316895,
196
+ "tpp_threshold_10_unintended_diff_only": 0.000500023365020752,
197
+ "tpp_threshold_20_total_metric": 0.0034999698400497437,
198
+ "tpp_threshold_20_intended_diff_only": 0.001999974250793457,
199
+ "tpp_threshold_20_unintended_diff_only": -0.0014999955892562866,
200
+ "tpp_threshold_50_total_metric": -2.9802322387695312e-08,
201
+ "tpp_threshold_50_intended_diff_only": 0.0009999871253967285,
202
+ "tpp_threshold_50_unintended_diff_only": 0.0010000169277191162,
203
+ "tpp_threshold_100_total_metric": -0.0010000020265579224,
204
+ "tpp_threshold_100_intended_diff_only": -0.0009999871253967285,
205
+ "tpp_threshold_100_unintended_diff_only": 1.4901161193847656e-08,
206
+ "tpp_threshold_500_total_metric": 0.019999966025352478,
207
+ "tpp_threshold_500_intended_diff_only": 0.018999993801116943,
208
+ "tpp_threshold_500_unintended_diff_only": -0.0009999722242355347
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.003000020980834961,
212
+ "tpp_threshold_2_intended_diff_only": 0.0020000338554382324,
213
+ "tpp_threshold_2_unintended_diff_only": -0.0009999871253967285,
214
+ "tpp_threshold_5_total_metric": -0.004250019788742065,
215
+ "tpp_threshold_5_intended_diff_only": -0.004999995231628418,
216
+ "tpp_threshold_5_unintended_diff_only": -0.0007499754428863525,
217
+ "tpp_threshold_10_total_metric": -0.0022500157356262207,
218
+ "tpp_threshold_10_intended_diff_only": -0.0040000081062316895,
219
+ "tpp_threshold_10_unintended_diff_only": -0.0017499923706054688,
220
+ "tpp_threshold_20_total_metric": -0.0015000253915786743,
221
+ "tpp_threshold_20_intended_diff_only": 0.0,
222
+ "tpp_threshold_20_unintended_diff_only": 0.0015000253915786743,
223
+ "tpp_threshold_50_total_metric": -0.0002500265836715698,
224
+ "tpp_threshold_50_intended_diff_only": 0.0009999871253967285,
225
+ "tpp_threshold_50_unintended_diff_only": 0.0012500137090682983,
226
+ "tpp_threshold_100_total_metric": -0.006750017404556274,
227
+ "tpp_threshold_100_intended_diff_only": -0.0040000081062316895,
228
+ "tpp_threshold_100_unintended_diff_only": 0.002750009298324585,
229
+ "tpp_threshold_500_total_metric": 0.016750022768974304,
230
+ "tpp_threshold_500_intended_diff_only": 0.016000032424926758,
231
+ "tpp_threshold_500_unintended_diff_only": -0.0007499903440475464
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": -0.0017499923706054688,
235
+ "tpp_threshold_2_intended_diff_only": -0.0009999871253967285,
236
+ "tpp_threshold_2_unintended_diff_only": 0.0007500052452087402,
237
+ "tpp_threshold_5_total_metric": -0.0012499988079071045,
238
+ "tpp_threshold_5_intended_diff_only": -0.0009999871253967285,
239
+ "tpp_threshold_5_unintended_diff_only": 0.000250011682510376,
240
+ "tpp_threshold_10_total_metric": 0.003999993205070496,
241
+ "tpp_threshold_10_intended_diff_only": 0.004999995231628418,
242
+ "tpp_threshold_10_unintended_diff_only": 0.0010000020265579224,
243
+ "tpp_threshold_20_total_metric": 0.010750040411949158,
244
+ "tpp_threshold_20_intended_diff_only": 0.010000050067901611,
245
+ "tpp_threshold_20_unintended_diff_only": -0.0007499903440475464,
246
+ "tpp_threshold_50_total_metric": 0.005250006914138794,
247
+ "tpp_threshold_50_intended_diff_only": 0.003000020980834961,
248
+ "tpp_threshold_50_unintended_diff_only": -0.002249985933303833,
249
+ "tpp_threshold_100_total_metric": 0.004000023007392883,
250
+ "tpp_threshold_100_intended_diff_only": 0.0020000338554382324,
251
+ "tpp_threshold_100_unintended_diff_only": -0.001999989151954651,
252
+ "tpp_threshold_500_total_metric": 0.011250033974647522,
253
+ "tpp_threshold_500_intended_diff_only": 0.010000050067901611,
254
+ "tpp_threshold_500_unintended_diff_only": -0.0012499839067459106
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.0017500370740890503,
258
  "tpp_threshold_2_intended_diff_only": 0.001000046730041504,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0007499903440475464,
260
+ "tpp_threshold_5_total_metric": 0.0007500052452087402,
261
+ "tpp_threshold_5_intended_diff_only": -0.001999974250793457,
262
+ "tpp_threshold_5_unintended_diff_only": -0.0027499794960021973,
263
+ "tpp_threshold_10_total_metric": 0.003000035881996155,
264
+ "tpp_threshold_10_intended_diff_only": 0.0020000338554382324,
265
+ "tpp_threshold_10_unintended_diff_only": -0.0010000020265579224,
266
+ "tpp_threshold_20_total_metric": 0.003500029444694519,
267
+ "tpp_threshold_20_intended_diff_only": 0.001000046730041504,
268
+ "tpp_threshold_20_unintended_diff_only": -0.002499982714653015,
269
+ "tpp_threshold_50_total_metric": 0.0065000057220458984,
270
+ "tpp_threshold_50_intended_diff_only": 0.0040000081062316895,
271
+ "tpp_threshold_50_unintended_diff_only": -0.002499997615814209,
272
+ "tpp_threshold_100_total_metric": 0.0007500201463699341,
273
+ "tpp_threshold_100_intended_diff_only": 0.0020000338554382324,
274
+ "tpp_threshold_100_unintended_diff_only": 0.0012500137090682983,
275
+ "tpp_threshold_500_total_metric": 0.006000012159347534,
276
+ "tpp_threshold_500_intended_diff_only": 0.00700002908706665,
277
+ "tpp_threshold_500_unintended_diff_only": 0.0010000169277191162
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": -0.0004999935626983643,
281
+ "tpp_threshold_2_intended_diff_only": 0.0,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0004999935626983643,
283
+ "tpp_threshold_5_total_metric": -0.000500023365020752,
284
+ "tpp_threshold_5_intended_diff_only": 0.0,
285
+ "tpp_threshold_5_unintended_diff_only": 0.000500023365020752,
286
+ "tpp_threshold_10_total_metric": -0.007499992847442627,
287
+ "tpp_threshold_10_intended_diff_only": -0.0059999823570251465,
288
+ "tpp_threshold_10_unintended_diff_only": 0.0015000104904174805,
289
+ "tpp_threshold_20_total_metric": -0.00024996697902679443,
290
+ "tpp_threshold_20_intended_diff_only": 0.001000046730041504,
291
+ "tpp_threshold_20_unintended_diff_only": 0.0012500137090682983,
292
+ "tpp_threshold_50_total_metric": 4.470348358154297e-08,
293
+ "tpp_threshold_50_intended_diff_only": 0.001000046730041504,
294
+ "tpp_threshold_50_unintended_diff_only": 0.0010000020265579224,
295
+ "tpp_threshold_100_total_metric": 0.0002500265836715698,
296
+ "tpp_threshold_100_intended_diff_only": 0.0020000338554382324,
297
+ "tpp_threshold_100_unintended_diff_only": 0.0017500072717666626,
298
+ "tpp_threshold_500_total_metric": 0.004500031471252441,
299
+ "tpp_threshold_500_intended_diff_only": 0.00700002908706665,
300
+ "tpp_threshold_500_unintended_diff_only": 0.002499997615814209
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.0007500052452087402,
306
+ "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
307
+ "tpp_threshold_2_unintended_diff_only": 0.0032500028610229492,
308
+ "tpp_threshold_5_total_metric": 0.0012500137090682983,
309
+ "tpp_threshold_5_intended_diff_only": 0.003000020980834961,
310
+ "tpp_threshold_5_unintended_diff_only": 0.0017500072717666626,
311
+ "tpp_threshold_10_total_metric": -0.002249971032142639,
312
+ "tpp_threshold_10_intended_diff_only": -0.0009999871253967285,
313
+ "tpp_threshold_10_unintended_diff_only": 0.0012499839067459106,
314
+ "tpp_threshold_20_total_metric": -0.001999974250793457,
315
+ "tpp_threshold_20_intended_diff_only": -0.0009999871253967285,
316
+ "tpp_threshold_20_unintended_diff_only": 0.0009999871253967285,
317
+ "tpp_threshold_50_total_metric": 0.0017499923706054688,
318
+ "tpp_threshold_50_intended_diff_only": -0.0040000081062316895,
319
+ "tpp_threshold_50_unintended_diff_only": -0.005750000476837158,
320
+ "tpp_threshold_100_total_metric": -0.0010000020265579224,
321
+ "tpp_threshold_100_intended_diff_only": 0.0009999871253967285,
322
+ "tpp_threshold_100_unintended_diff_only": 0.001999989151954651,
323
+ "tpp_threshold_500_total_metric": 0.020249992609024048,
324
+ "tpp_threshold_500_intended_diff_only": 0.015999972820281982,
325
+ "tpp_threshold_500_unintended_diff_only": -0.004250019788742065
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.003000020980834961,
329
+ "tpp_threshold_2_intended_diff_only": -0.0009999871253967285,
330
+ "tpp_threshold_2_unintended_diff_only": -0.0040000081062316895,
331
+ "tpp_threshold_5_total_metric": -0.010749951004981995,
332
+ "tpp_threshold_5_intended_diff_only": -0.006999969482421875,
333
+ "tpp_threshold_5_unintended_diff_only": 0.0037499815225601196,
334
+ "tpp_threshold_10_total_metric": 0.003250017762184143,
335
+ "tpp_threshold_10_intended_diff_only": 0.0040000081062316895,
336
+ "tpp_threshold_10_unintended_diff_only": 0.0007499903440475464,
337
+ "tpp_threshold_20_total_metric": -0.0019999444484710693,
338
+ "tpp_threshold_20_intended_diff_only": -0.003999948501586914,
339
+ "tpp_threshold_20_unintended_diff_only": -0.0020000040531158447,
340
+ "tpp_threshold_50_total_metric": -0.0004999637603759766,
341
+ "tpp_threshold_50_intended_diff_only": 0.0020000338554382324,
342
+ "tpp_threshold_50_unintended_diff_only": 0.002499997615814209,
343
+ "tpp_threshold_100_total_metric": -0.003749936819076538,
344
+ "tpp_threshold_100_intended_diff_only": 0.001000046730041504,
345
+ "tpp_threshold_100_unintended_diff_only": 0.004749983549118042,
346
+ "tpp_threshold_500_total_metric": 0.05375005304813385,
347
+ "tpp_threshold_500_intended_diff_only": 0.057000041007995605,
348
+ "tpp_threshold_500_unintended_diff_only": 0.0032499879598617554
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.0007500052452087402,
352
+ "tpp_threshold_2_intended_diff_only": 0.0,
353
+ "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402,
354
+ "tpp_threshold_5_total_metric": 0.006999999284744263,
355
+ "tpp_threshold_5_intended_diff_only": 0.0009999871253967285,
356
+ "tpp_threshold_5_unintended_diff_only": -0.006000012159347534,
357
+ "tpp_threshold_10_total_metric": 0.009000033140182495,
358
+ "tpp_threshold_10_intended_diff_only": 0.00700002908706665,
359
+ "tpp_threshold_10_unintended_diff_only": -0.0020000040531158447,
360
+ "tpp_threshold_20_total_metric": 0.0012499839067459106,
361
+ "tpp_threshold_20_intended_diff_only": 0.001999974250793457,
362
+ "tpp_threshold_20_unintended_diff_only": 0.0007499903440475464,
363
+ "tpp_threshold_50_total_metric": 0.013250023126602173,
364
+ "tpp_threshold_50_intended_diff_only": 0.00700002908706665,
365
+ "tpp_threshold_50_unintended_diff_only": -0.0062499940395355225,
366
+ "tpp_threshold_100_total_metric": 0.020500004291534424,
367
+ "tpp_threshold_100_intended_diff_only": 0.018999993801116943,
368
+ "tpp_threshold_100_unintended_diff_only": -0.0015000104904174805,
369
+ "tpp_threshold_500_total_metric": 0.03399999439716339,
370
+ "tpp_threshold_500_intended_diff_only": 0.03700000047683716,
371
+ "tpp_threshold_500_unintended_diff_only": 0.003000006079673767
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.0012499839067459106,
375
+ "tpp_threshold_2_intended_diff_only": 0.0,
376
+ "tpp_threshold_2_unintended_diff_only": -0.0012499839067459106,
377
+ "tpp_threshold_5_total_metric": 0.002249985933303833,
378
+ "tpp_threshold_5_intended_diff_only": 0.0009999871253967285,
379
+ "tpp_threshold_5_unintended_diff_only": -0.0012499988079071045,
380
+ "tpp_threshold_10_total_metric": -0.010500043630599976,
381
+ "tpp_threshold_10_intended_diff_only": -0.00700002908706665,
382
+ "tpp_threshold_10_unintended_diff_only": 0.003500014543533325,
383
+ "tpp_threshold_20_total_metric": -0.004750058054924011,
384
+ "tpp_threshold_20_intended_diff_only": -0.005000054836273193,
385
+ "tpp_threshold_20_unintended_diff_only": -0.00024999678134918213,
386
+ "tpp_threshold_50_total_metric": 0.0012499690055847168,
387
+ "tpp_threshold_50_intended_diff_only": 0.0029999613761901855,
388
+ "tpp_threshold_50_unintended_diff_only": 0.0017499923706054688,
389
+ "tpp_threshold_100_total_metric": -0.005250021815299988,
390
+ "tpp_threshold_100_intended_diff_only": -0.003000020980834961,
391
+ "tpp_threshold_100_unintended_diff_only": 0.002250000834465027,
392
+ "tpp_threshold_500_total_metric": 0.020999982953071594,
393
+ "tpp_threshold_500_intended_diff_only": 0.02799999713897705,
394
+ "tpp_threshold_500_unintended_diff_only": 0.0070000141859054565
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": -0.0027499794960021973,
398
+ "tpp_threshold_2_intended_diff_only": -0.0009999871253967285,
399
+ "tpp_threshold_2_unintended_diff_only": 0.0017499923706054688,
400
+ "tpp_threshold_5_total_metric": 0.001749962568283081,
401
+ "tpp_threshold_5_intended_diff_only": -0.0020000338554382324,
402
+ "tpp_threshold_5_unintended_diff_only": -0.0037499964237213135,
403
+ "tpp_threshold_10_total_metric": 0.010749965906143188,
404
+ "tpp_threshold_10_intended_diff_only": 0.011999964714050293,
405
+ "tpp_threshold_10_unintended_diff_only": 0.0012499988079071045,
406
+ "tpp_threshold_20_total_metric": 0.001999974250793457,
407
+ "tpp_threshold_20_intended_diff_only": 0.0059999823570251465,
408
+ "tpp_threshold_20_unintended_diff_only": 0.0040000081062316895,
409
+ "tpp_threshold_50_total_metric": 0.0057499706745147705,
410
+ "tpp_threshold_50_intended_diff_only": 0.0059999823570251465,
411
+ "tpp_threshold_50_unintended_diff_only": 0.000250011682510376,
412
+ "tpp_threshold_100_total_metric": -0.0012499839067459106,
413
+ "tpp_threshold_100_intended_diff_only": 0.0040000081062316895,
414
+ "tpp_threshold_100_unintended_diff_only": 0.0052499920129776,
415
+ "tpp_threshold_500_total_metric": 0.08650000393390656,
416
+ "tpp_threshold_500_intended_diff_only": 0.09200000762939453,
417
+ "tpp_threshold_500_unintended_diff_only": 0.005500003695487976
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-109863-tokens-450000000/scr/saebench_scr_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,8 +69,8 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "fe5d3fb3-3b9a-43dc-a36c-a365a99d6d04",
69
- "datetime_epoch_millis": 1776883225997,
70
  "eval_result_metrics": {
71
  "scr_metrics": {
72
  "scr_dir1_threshold_2": 0.24931224390936493,
@@ -78,18 +82,18 @@
78
  "scr_dir1_threshold_10": 0.22760336536778464,
79
  "scr_metric_threshold_10": 0.22359674172327682,
80
  "scr_dir2_threshold_10": 0.22317059201096162,
81
- "scr_dir1_threshold_20": null,
82
- "scr_metric_threshold_20": null,
83
- "scr_dir2_threshold_20": null,
84
- "scr_dir1_threshold_50": null,
85
- "scr_metric_threshold_50": null,
86
- "scr_dir2_threshold_50": null,
87
- "scr_dir1_threshold_100": null,
88
- "scr_metric_threshold_100": null,
89
- "scr_dir2_threshold_100": null,
90
- "scr_dir1_threshold_500": null,
91
- "scr_metric_threshold_500": null,
92
- "scr_dir2_threshold_500": null
93
  }
94
  },
95
  "eval_result_details": [
@@ -104,18 +108,18 @@
104
  "scr_dir1_threshold_10": 0.4230775402847115,
105
  "scr_metric_threshold_10": 0.10103619422217953,
106
  "scr_dir2_threshold_10": 0.10103619422217953,
107
- "scr_dir1_threshold_20": null,
108
- "scr_metric_threshold_20": null,
109
- "scr_dir2_threshold_20": null,
110
- "scr_dir1_threshold_50": null,
111
- "scr_metric_threshold_50": null,
112
- "scr_dir2_threshold_50": null,
113
- "scr_dir1_threshold_100": null,
114
- "scr_metric_threshold_100": null,
115
- "scr_dir2_threshold_100": null,
116
- "scr_dir1_threshold_500": null,
117
- "scr_metric_threshold_500": null,
118
- "scr_dir2_threshold_500": null
119
  },
120
  {
121
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
@@ -128,18 +132,18 @@
128
  "scr_dir1_threshold_10": 0.3484848005931564,
129
  "scr_metric_threshold_10": 0.19877673778514987,
130
  "scr_dir2_threshold_10": 0.19877673778514987,
131
- "scr_dir1_threshold_20": null,
132
- "scr_metric_threshold_20": null,
133
- "scr_dir2_threshold_20": null,
134
- "scr_dir1_threshold_50": null,
135
- "scr_metric_threshold_50": null,
136
- "scr_dir2_threshold_50": null,
137
- "scr_dir1_threshold_100": null,
138
- "scr_metric_threshold_100": null,
139
- "scr_dir2_threshold_100": null,
140
- "scr_dir1_threshold_500": null,
141
- "scr_metric_threshold_500": null,
142
- "scr_dir2_threshold_500": null
143
  },
144
  {
145
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
@@ -152,18 +156,18 @@
152
  "scr_dir1_threshold_10": 0.28125005820765137,
153
  "scr_metric_threshold_10": 0.11055282101401862,
154
  "scr_dir2_threshold_10": 0.11055282101401862,
155
- "scr_dir1_threshold_20": null,
156
- "scr_metric_threshold_20": null,
157
- "scr_dir2_threshold_20": null,
158
- "scr_dir1_threshold_50": null,
159
- "scr_metric_threshold_50": null,
160
- "scr_dir2_threshold_50": null,
161
- "scr_dir1_threshold_100": null,
162
- "scr_metric_threshold_100": null,
163
- "scr_dir2_threshold_100": null,
164
- "scr_dir1_threshold_500": null,
165
- "scr_metric_threshold_500": null,
166
- "scr_dir2_threshold_500": null
167
  },
168
  {
169
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
@@ -176,18 +180,18 @@
176
  "scr_dir1_threshold_10": 0.04026834093059259,
177
  "scr_metric_threshold_10": 0.27388542165349355,
178
  "scr_dir2_threshold_10": 0.27388542165349355,
179
- "scr_dir1_threshold_20": null,
180
- "scr_metric_threshold_20": null,
181
- "scr_dir2_threshold_20": null,
182
- "scr_dir1_threshold_50": null,
183
- "scr_metric_threshold_50": null,
184
- "scr_dir2_threshold_50": null,
185
- "scr_dir1_threshold_100": null,
186
- "scr_metric_threshold_100": null,
187
- "scr_dir2_threshold_100": null,
188
- "scr_dir1_threshold_500": null,
189
- "scr_metric_threshold_500": null,
190
- "scr_dir2_threshold_500": null
191
  },
192
  {
193
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
@@ -200,18 +204,18 @@
200
  "scr_dir1_threshold_10": 0.07894729412163301,
201
  "scr_metric_threshold_10": 0.5396826147702655,
202
  "scr_dir2_threshold_10": 0.5396826147702655,
203
- "scr_dir1_threshold_20": null,
204
- "scr_metric_threshold_20": null,
205
- "scr_dir2_threshold_20": null,
206
- "scr_dir1_threshold_50": null,
207
- "scr_metric_threshold_50": null,
208
- "scr_dir2_threshold_50": null,
209
- "scr_dir1_threshold_100": null,
210
- "scr_metric_threshold_100": null,
211
- "scr_dir2_threshold_100": null,
212
- "scr_dir1_threshold_500": null,
213
- "scr_metric_threshold_500": null,
214
- "scr_dir2_threshold_500": null
215
  },
216
  {
217
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
@@ -224,18 +228,18 @@
224
  "scr_dir1_threshold_10": 0.3278689218723027,
225
  "scr_metric_threshold_10": 0.12595415766246074,
226
  "scr_dir2_threshold_10": 0.12595415766246074,
227
- "scr_dir1_threshold_20": null,
228
- "scr_metric_threshold_20": null,
229
- "scr_dir2_threshold_20": null,
230
- "scr_dir1_threshold_50": null,
231
- "scr_metric_threshold_50": null,
232
- "scr_dir2_threshold_50": null,
233
- "scr_dir1_threshold_100": null,
234
- "scr_metric_threshold_100": null,
235
- "scr_dir2_threshold_100": null,
236
- "scr_dir1_threshold_500": null,
237
- "scr_metric_threshold_500": null,
238
- "scr_dir2_threshold_500": null
239
  },
240
  {
241
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
@@ -248,18 +252,18 @@
248
  "scr_dir1_threshold_10": 0.19534878692091323,
249
  "scr_metric_threshold_10": 0.31330480666733007,
250
  "scr_dir2_threshold_10": 0.31330480666733007,
251
- "scr_dir1_threshold_20": null,
252
- "scr_metric_threshold_20": null,
253
- "scr_dir2_threshold_20": null,
254
- "scr_dir1_threshold_50": null,
255
- "scr_metric_threshold_50": null,
256
- "scr_dir2_threshold_50": null,
257
- "scr_dir1_threshold_100": null,
258
- "scr_metric_threshold_100": null,
259
- "scr_dir2_threshold_100": null,
260
- "scr_dir1_threshold_500": null,
261
- "scr_metric_threshold_500": null,
262
- "scr_dir2_threshold_500": null
263
  },
264
  {
265
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
@@ -272,18 +276,18 @@
272
  "scr_dir1_threshold_10": 0.12558118001131657,
273
  "scr_metric_threshold_10": 0.12558118001131657,
274
  "scr_dir2_threshold_10": 0.1221719823127949,
275
- "scr_dir1_threshold_20": null,
276
- "scr_metric_threshold_20": null,
277
- "scr_dir2_threshold_20": null,
278
- "scr_dir1_threshold_50": null,
279
- "scr_metric_threshold_50": null,
280
- "scr_dir2_threshold_50": null,
281
- "scr_dir1_threshold_100": null,
282
- "scr_metric_threshold_100": null,
283
- "scr_dir2_threshold_100": null,
284
- "scr_dir1_threshold_500": null,
285
- "scr_metric_threshold_500": null,
286
- "scr_dir2_threshold_500": null
287
  }
288
  ],
289
  "sae_bench_commit_hash": "Unknown",
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "89ef4954-029b-48d0-9a7a-282b42224e3d",
73
+ "datetime_epoch_millis": 1777315454905,
74
  "eval_result_metrics": {
75
  "scr_metrics": {
76
  "scr_dir1_threshold_2": 0.24931224390936493,
 
82
  "scr_dir1_threshold_10": 0.22760336536778464,
83
  "scr_metric_threshold_10": 0.22359674172327682,
84
  "scr_dir2_threshold_10": 0.22317059201096162,
85
+ "scr_dir1_threshold_20": 0.06267052552110919,
86
+ "scr_metric_threshold_20": 0.26391427202288076,
87
+ "scr_dir2_threshold_20": 0.2605021904039865,
88
+ "scr_dir1_threshold_50": 0.14509570183438503,
89
+ "scr_metric_threshold_50": 0.35441780922749,
90
+ "scr_dir2_threshold_50": 0.3490721145045288,
91
+ "scr_dir1_threshold_100": 0.1175602804273406,
92
+ "scr_metric_threshold_100": 0.3854469697379585,
93
+ "scr_dir2_threshold_100": 0.38118519632470216,
94
+ "scr_dir1_threshold_500": -0.7360395318955923,
95
+ "scr_metric_threshold_500": 0.20030516015854738,
96
+ "scr_dir2_threshold_500": 0.19811638530758433
97
  }
98
  },
99
  "eval_result_details": [
 
108
  "scr_dir1_threshold_10": 0.4230775402847115,
109
  "scr_metric_threshold_10": 0.10103619422217953,
110
  "scr_dir2_threshold_10": 0.10103619422217953,
111
+ "scr_dir1_threshold_20": 0.47435892537422925,
112
+ "scr_metric_threshold_20": 0.173575039924299,
113
+ "scr_dir2_threshold_20": 0.173575039924299,
114
+ "scr_dir1_threshold_50": 0.5641028776049328,
115
+ "scr_metric_threshold_50": 0.27461138856264183,
116
+ "scr_dir2_threshold_50": 0.27461138856264183,
117
+ "scr_dir1_threshold_100": 0.39743608357792887,
118
+ "scr_metric_threshold_100": 0.3834196571158211,
119
+ "scr_dir2_threshold_100": 0.3834196571158211,
120
+ "scr_dir1_threshold_500": -2.730768907469913,
121
+ "scr_metric_threshold_500": 0.07253884570211949,
122
+ "scr_dir2_threshold_500": 0.07253884570211949
123
  },
124
  {
125
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
 
132
  "scr_dir1_threshold_10": 0.3484848005931564,
133
  "scr_metric_threshold_10": 0.19877673778514987,
134
  "scr_dir2_threshold_10": 0.19877673778514987,
135
+ "scr_dir1_threshold_20": 0.21212114370450913,
136
+ "scr_metric_threshold_20": 0.14373084950283896,
137
+ "scr_dir2_threshold_20": 0.14373084950283896,
138
+ "scr_dir1_threshold_50": 0.41666662903748003,
139
+ "scr_metric_threshold_50": 0.3944954111717689,
140
+ "scr_dir2_threshold_50": 0.3944954111717689,
141
+ "scr_dir1_threshold_100": 0.2045454853329709,
142
+ "scr_metric_threshold_100": 0.590214084558388,
143
+ "scr_dir2_threshold_100": 0.590214084558388,
144
+ "scr_dir1_threshold_500": 0.15151542518196345,
145
+ "scr_metric_threshold_500": 0.19571867338661902,
146
+ "scr_dir2_threshold_500": 0.19571867338661902
147
  },
148
  {
149
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
 
156
  "scr_dir1_threshold_10": 0.28125005820765137,
157
  "scr_metric_threshold_10": 0.11055282101401862,
158
  "scr_dir2_threshold_10": 0.11055282101401862,
159
+ "scr_dir1_threshold_20": -0.3906245634426147,
160
+ "scr_metric_threshold_20": 0.16080402160563206,
161
+ "scr_dir2_threshold_20": 0.16080402160563206,
162
+ "scr_dir1_threshold_50": -0.18750034924590825,
163
+ "scr_metric_threshold_50": 0.17085429167603294,
164
+ "scr_dir2_threshold_50": 0.17085429167603294,
165
+ "scr_dir1_threshold_100": 0.1093754365573853,
166
+ "scr_metric_threshold_100": 0.04522614043660845,
167
+ "scr_dir2_threshold_100": 0.04522614043660845,
168
+ "scr_dir1_threshold_500": -2.7812500582076516,
169
+ "scr_metric_threshold_500": -0.09547734102822189,
170
+ "scr_dir2_threshold_500": -0.09547734102822189
171
  },
172
  {
173
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
 
180
  "scr_dir1_threshold_10": 0.04026834093059259,
181
  "scr_metric_threshold_10": 0.27388542165349355,
182
  "scr_dir2_threshold_10": 0.27388542165349355,
183
+ "scr_dir1_threshold_20": -0.6644298255263911,
184
+ "scr_metric_threshold_20": 0.32165607513677746,
185
+ "scr_dir2_threshold_20": 0.32165607513677746,
186
+ "scr_dir1_threshold_50": -0.38926196243306976,
187
+ "scr_metric_threshold_50": 0.5063695343586597,
188
+ "scr_dir2_threshold_50": 0.5063695343586597,
189
+ "scr_dir1_threshold_100": -0.6174497610813644,
190
+ "scr_metric_threshold_100": 0.47452243203647043,
191
+ "scr_dir2_threshold_100": 0.47452243203647043,
192
+ "scr_dir1_threshold_500": -1.5302017557369474,
193
+ "scr_metric_threshold_500": -0.4522293465167161,
194
+ "scr_dir2_threshold_500": -0.4522293465167161
195
  },
196
  {
197
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
 
204
  "scr_dir1_threshold_10": 0.07894729412163301,
205
  "scr_metric_threshold_10": 0.5396826147702655,
206
  "scr_dir2_threshold_10": 0.5396826147702655,
207
+ "scr_dir1_threshold_20": 0.11578955293692479,
208
+ "scr_metric_threshold_20": 0.5833333136228053,
209
+ "scr_dir2_threshold_20": 0.5833333136228053,
210
+ "scr_dir1_threshold_50": -0.28947372548798167,
211
+ "scr_metric_threshold_50": 0.654761901946115,
212
+ "scr_dir2_threshold_50": 0.654761901946115,
213
+ "scr_dir1_threshold_100": -0.29999990588740183,
214
+ "scr_metric_threshold_100": 0.7023808031440972,
215
+ "scr_dir2_threshold_100": 0.7023808031440972,
216
+ "scr_dir1_threshold_500": -0.19999993725826787,
217
+ "scr_metric_threshold_500": 0.7460317385229734,
218
+ "scr_dir2_threshold_500": 0.7460317385229734
219
  },
220
  {
221
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
 
228
  "scr_dir1_threshold_10": 0.3278689218723027,
229
  "scr_metric_threshold_10": 0.12595415766246074,
230
  "scr_dir2_threshold_10": 0.12595415766246074,
231
+ "scr_dir1_threshold_20": 0.3169397732417441,
232
+ "scr_metric_threshold_20": 0.18702297804342538,
233
+ "scr_dir2_threshold_20": 0.18702297804342538,
234
+ "scr_dir1_threshold_50": 0.4043716594522235,
235
+ "scr_metric_threshold_50": 0.24045808212745792,
236
+ "scr_dir2_threshold_50": 0.24045808212745792,
237
+ "scr_dir1_threshold_100": 0.4535520140184937,
238
+ "scr_metric_threshold_100": 0.3015266750097995,
239
+ "scr_dir2_threshold_100": 0.3015266750097995,
240
+ "scr_dir1_threshold_500": 0.4535520140184937,
241
+ "scr_metric_threshold_500": 0.41221362757701907,
242
+ "scr_dir2_threshold_500": 0.41221362757701907
243
  },
244
  {
245
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
 
252
  "scr_dir1_threshold_10": 0.19534878692091323,
253
  "scr_metric_threshold_10": 0.31330480666733007,
254
  "scr_dir2_threshold_10": 0.31330480666733007,
255
+ "scr_dir1_threshold_20": 0.2651161514146259,
256
+ "scr_metric_threshold_20": 0.36909885188142205,
257
+ "scr_dir2_threshold_20": 0.36909885188142205,
258
+ "scr_dir1_threshold_50": 0.40000005544617256,
259
+ "scr_metric_threshold_50": 0.35193143367601315,
260
+ "scr_dir2_threshold_50": 0.35193143367601315,
261
+ "scr_dir1_threshold_100": 0.4372091501709683,
262
+ "scr_metric_threshold_100": 0.330472224872739,
263
+ "scr_dir2_threshold_100": 0.330472224872739,
264
+ "scr_dir1_threshold_500": 0.4372091501709683,
265
+ "scr_metric_threshold_500": 0.41201726948797046,
266
+ "scr_dir2_threshold_500": 0.41201726948797046
267
  },
268
  {
269
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
 
276
  "scr_dir1_threshold_10": 0.12558118001131657,
277
  "scr_metric_threshold_10": 0.12558118001131657,
278
  "scr_dir2_threshold_10": 0.1221719823127949,
279
+ "scr_dir1_threshold_20": 0.1720930464658461,
280
+ "scr_metric_threshold_20": 0.1720930464658461,
281
+ "scr_dir2_threshold_20": 0.14479639351469145,
282
+ "scr_dir1_threshold_50": 0.24186043030123083,
283
+ "scr_metric_threshold_50": 0.24186043030123083,
284
+ "scr_dir2_threshold_50": 0.19909487251754088,
285
+ "scr_dir1_threshold_100": 0.2558137407297439,
286
+ "scr_metric_threshold_100": 0.2558137407297439,
287
+ "scr_dir2_threshold_100": 0.22171955342369326,
288
+ "scr_dir1_threshold_500": 0.3116278141366155,
289
+ "scr_metric_threshold_500": 0.3116278141366155,
290
+ "scr_dir2_threshold_500": 0.29411761532891106
291
  }
292
  ],
293
  "sae_bench_commit_hash": "Unknown",
gemma-2-2b-btk/k-100/seed-0/snapshots/step-109863-tokens-450000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "ad3db8b5-38a5-4958-9dc1-a61ae0bfdebe",
69
- "datetime_epoch_millis": 1776883393381,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.014100004732608794,
73
- "tpp_threshold_2_intended_diff_only": 0.015100008249282836,
74
- "tpp_threshold_2_unintended_diff_only": 0.0010000035166740418,
75
- "tpp_threshold_5_total_metric": 0.05454999953508377,
76
- "tpp_threshold_5_intended_diff_only": 0.06330000162124634,
77
- "tpp_threshold_5_unintended_diff_only": 0.008750002086162566,
78
- "tpp_threshold_10_total_metric": 0.11117500215768814,
79
- "tpp_threshold_10_intended_diff_only": 0.139000004529953,
80
- "tpp_threshold_10_unintended_diff_only": 0.027825002372264863,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.02024998366832733,
99
- "tpp_threshold_2_intended_diff_only": 0.021599996089935302,
100
- "tpp_threshold_2_unintended_diff_only": 0.0013500124216079712,
101
- "tpp_threshold_5_total_metric": 0.0921000063419342,
102
- "tpp_threshold_5_intended_diff_only": 0.10720001459121704,
103
- "tpp_threshold_5_unintended_diff_only": 0.015100008249282837,
104
- "tpp_threshold_10_total_metric": 0.1965500146150589,
105
- "tpp_threshold_10_intended_diff_only": 0.24900002479553224,
106
  "tpp_threshold_10_unintended_diff_only": 0.05245001018047333,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.007950025796890258,
123
- "tpp_threshold_2_intended_diff_only": 0.00860002040863037,
124
- "tpp_threshold_2_unintended_diff_only": 0.0006499946117401123,
125
- "tpp_threshold_5_total_metric": 0.016999992728233337,
126
- "tpp_threshold_5_intended_diff_only": 0.019399988651275634,
127
- "tpp_threshold_5_unintended_diff_only": 0.0023999959230422974,
128
- "tpp_threshold_10_total_metric": 0.025799989700317383,
129
- "tpp_threshold_10_intended_diff_only": 0.02899998426437378,
130
- "tpp_threshold_10_unintended_diff_only": 0.0031999945640563965,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.0020000189542770386,
190
  "tpp_threshold_10_total_metric": 0.10624998807907104,
191
  "tpp_threshold_10_intended_diff_only": 0.12400001287460327,
192
- "tpp_threshold_10_unintended_diff_only": 0.017750024795532227
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": 0.03899995982646942,
196
  "tpp_threshold_2_intended_diff_only": 0.04799997806549072,
197
- "tpp_threshold_2_unintended_diff_only": 0.009000018239021301,
198
- "tpp_threshold_5_total_metric": 0.18300001323223114,
199
  "tpp_threshold_5_intended_diff_only": 0.23000001907348633,
200
- "tpp_threshold_5_unintended_diff_only": 0.04700000584125519,
201
- "tpp_threshold_10_total_metric": 0.2045000046491623,
202
- "tpp_threshold_10_intended_diff_only": 0.28200000524520874,
203
- "tpp_threshold_10_unintended_diff_only": 0.07750000059604645
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.03800000250339508,
207
- "tpp_threshold_2_intended_diff_only": 0.03600001335144043,
208
- "tpp_threshold_2_unintended_diff_only": -0.001999989151954651,
209
- "tpp_threshold_5_total_metric": 0.2017500251531601,
210
- "tpp_threshold_5_intended_diff_only": 0.21100002527236938,
211
- "tpp_threshold_5_unintended_diff_only": 0.00925000011920929,
212
- "tpp_threshold_10_total_metric": 0.2410000115633011,
213
- "tpp_threshold_10_intended_diff_only": 0.30400002002716064,
214
- "tpp_threshold_10_unintended_diff_only": 0.06300000846385956
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.012749999761581421,
218
  "tpp_threshold_2_intended_diff_only": 0.009999990463256836,
219
- "tpp_threshold_2_unintended_diff_only": -0.002750009298324585,
220
- "tpp_threshold_5_total_metric": 0.04575003683567047,
221
- "tpp_threshold_5_intended_diff_only": 0.057000041007995605,
222
- "tpp_threshold_5_unintended_diff_only": 0.011250004172325134,
223
- "tpp_threshold_10_total_metric": 0.26075005531311035,
224
  "tpp_threshold_10_intended_diff_only": 0.315000057220459,
225
- "tpp_threshold_10_unintended_diff_only": 0.05425000190734863
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.003999978303909302,
229
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
230
- "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162,
231
- "tpp_threshold_5_total_metric": 0.0169999897480011,
232
- "tpp_threshold_5_intended_diff_only": 0.023000001907348633,
233
- "tpp_threshold_5_unintended_diff_only": 0.006000012159347534,
234
- "tpp_threshold_10_total_metric": 0.17025001347064972,
235
- "tpp_threshold_10_intended_diff_only": 0.2200000286102295,
236
- "tpp_threshold_10_unintended_diff_only": 0.04975001513957977
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.00849999487400055,
242
- "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
243
- "tpp_threshold_2_unintended_diff_only": -0.00449998676776886,
244
- "tpp_threshold_5_total_metric": 0.011249974370002747,
245
- "tpp_threshold_5_intended_diff_only": 0.009999990463256836,
246
- "tpp_threshold_5_unintended_diff_only": -0.0012499839067459106,
247
- "tpp_threshold_10_total_metric": 0.00024999678134918213,
248
  "tpp_threshold_10_intended_diff_only": 0.004999995231628418,
249
- "tpp_threshold_10_unintended_diff_only": 0.004749998450279236
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": -0.0014999359846115112,
253
- "tpp_threshold_2_intended_diff_only": 0.005000054836273193,
254
- "tpp_threshold_2_unintended_diff_only": 0.006499990820884705,
255
- "tpp_threshold_5_total_metric": 0.016500025987625122,
256
- "tpp_threshold_5_intended_diff_only": 0.023000001907348633,
257
- "tpp_threshold_5_unintended_diff_only": 0.006499975919723511,
258
- "tpp_threshold_10_total_metric": 0.04575002193450928,
259
- "tpp_threshold_10_intended_diff_only": 0.050999999046325684,
260
- "tpp_threshold_10_unintended_diff_only": 0.005249977111816406
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.014000043272972107,
264
- "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
265
- "tpp_threshold_2_unintended_diff_only": -0.0070000141859054565,
266
- "tpp_threshold_5_total_metric": 0.015750005841255188,
267
- "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
268
- "tpp_threshold_5_unintended_diff_only": -0.0017500072717666626,
269
- "tpp_threshold_10_total_metric": 0.013499990105628967,
270
- "tpp_threshold_10_intended_diff_only": 0.014999985694885254,
271
- "tpp_threshold_10_unintended_diff_only": 0.0014999955892562866
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.007749989628791809,
275
- "tpp_threshold_2_intended_diff_only": 0.0,
276
- "tpp_threshold_2_unintended_diff_only": 0.007749989628791809,
277
- "tpp_threshold_5_total_metric": 0.0037499964237213135,
278
- "tpp_threshold_5_intended_diff_only": 0.004999995231628418,
279
- "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045,
280
- "tpp_threshold_10_total_metric": 0.006749942898750305,
281
- "tpp_threshold_10_intended_diff_only": 0.008999943733215332,
282
- "tpp_threshold_10_unintended_diff_only": 0.002250000834465027
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.026500016450881958,
286
- "tpp_threshold_2_intended_diff_only": 0.027000010013580322,
287
- "tpp_threshold_2_unintended_diff_only": 0.0004999935626983643,
288
- "tpp_threshold_5_total_metric": 0.03774996101856232,
289
- "tpp_threshold_5_intended_diff_only": 0.04499995708465576,
290
- "tpp_threshold_5_unintended_diff_only": 0.007249996066093445,
291
- "tpp_threshold_10_total_metric": 0.06274999678134918,
292
- "tpp_threshold_10_intended_diff_only": 0.06499999761581421,
293
- "tpp_threshold_10_unintended_diff_only": 0.002250000834465027
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "2eb8c3ea-654c-438a-b923-6016563c8d5b",
73
+ "datetime_epoch_millis": 1777315731228,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.012424995005130769,
77
+ "tpp_threshold_2_intended_diff_only": 0.014499998092651368,
78
+ "tpp_threshold_2_unintended_diff_only": 0.0020750030875205995,
79
+ "tpp_threshold_5_total_metric": 0.05477501600980759,
80
+ "tpp_threshold_5_intended_diff_only": 0.0626000165939331,
81
+ "tpp_threshold_5_unintended_diff_only": 0.00782500058412552,
82
+ "tpp_threshold_10_total_metric": 0.11235001981258393,
83
+ "tpp_threshold_10_intended_diff_only": 0.14010002017021178,
84
+ "tpp_threshold_10_unintended_diff_only": 0.02775000035762787,
85
+ "tpp_threshold_20_total_metric": 0.14235000908374787,
86
+ "tpp_threshold_20_intended_diff_only": 0.18160001039505005,
87
+ "tpp_threshold_20_unintended_diff_only": 0.039250001311302185,
88
+ "tpp_threshold_50_total_metric": 0.18815000653266908,
89
+ "tpp_threshold_50_intended_diff_only": 0.25790001153945924,
90
+ "tpp_threshold_50_unintended_diff_only": 0.06975000500679017,
91
+ "tpp_threshold_100_total_metric": 0.20410000830888747,
92
+ "tpp_threshold_100_intended_diff_only": 0.31500001549720763,
93
+ "tpp_threshold_100_unintended_diff_only": 0.11090000718832016,
94
+ "tpp_threshold_500_total_metric": 0.2460000142455101,
95
+ "tpp_threshold_500_intended_diff_only": 0.429500025510788,
96
+ "tpp_threshold_500_unintended_diff_only": 0.18350001126527787
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.01844998896121979,
103
+ "tpp_threshold_2_intended_diff_only": 0.02120000123977661,
104
+ "tpp_threshold_2_unintended_diff_only": 0.002750012278556824,
105
+ "tpp_threshold_5_total_metric": 0.09255001544952393,
106
+ "tpp_threshold_5_intended_diff_only": 0.10760002136230469,
107
+ "tpp_threshold_5_unintended_diff_only": 0.015050005912780762,
108
+ "tpp_threshold_10_total_metric": 0.1963500291109085,
109
+ "tpp_threshold_10_intended_diff_only": 0.24880003929138184,
110
  "tpp_threshold_10_unintended_diff_only": 0.05245001018047333,
111
+ "tpp_threshold_20_total_metric": 0.23575001060962678,
112
+ "tpp_threshold_20_intended_diff_only": 0.3086000204086304,
113
+ "tpp_threshold_20_unintended_diff_only": 0.0728500097990036,
114
+ "tpp_threshold_50_total_metric": 0.25990001261234286,
115
+ "tpp_threshold_50_intended_diff_only": 0.39240002632141113,
116
+ "tpp_threshold_50_unintended_diff_only": 0.1325000137090683,
117
+ "tpp_threshold_100_total_metric": 0.22480001151561738,
118
+ "tpp_threshold_100_intended_diff_only": 0.43480002880096436,
119
+ "tpp_threshold_100_unintended_diff_only": 0.21000001728534698,
120
+ "tpp_threshold_500_total_metric": 0.1297000229358673,
121
+ "tpp_threshold_500_intended_diff_only": 0.46680004596710206,
122
+ "tpp_threshold_500_unintended_diff_only": 0.33710002303123476
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.006400001049041748,
127
+ "tpp_threshold_2_intended_diff_only": 0.007799994945526123,
128
+ "tpp_threshold_2_unintended_diff_only": 0.001399993896484375,
129
+ "tpp_threshold_5_total_metric": 0.017000016570091248,
130
+ "tpp_threshold_5_intended_diff_only": 0.017600011825561524,
131
+ "tpp_threshold_5_unintended_diff_only": 0.0005999952554702759,
132
+ "tpp_threshold_10_total_metric": 0.02835001051425934,
133
+ "tpp_threshold_10_intended_diff_only": 0.03140000104904175,
134
+ "tpp_threshold_10_unintended_diff_only": 0.0030499905347824096,
135
+ "tpp_threshold_20_total_metric": 0.048950007557868956,
136
+ "tpp_threshold_20_intended_diff_only": 0.05460000038146973,
137
+ "tpp_threshold_20_unintended_diff_only": 0.005649992823600769,
138
+ "tpp_threshold_50_total_metric": 0.1164000004529953,
139
+ "tpp_threshold_50_intended_diff_only": 0.12339999675750732,
140
+ "tpp_threshold_50_unintended_diff_only": 0.006999996304512024,
141
+ "tpp_threshold_100_total_metric": 0.1834000051021576,
142
+ "tpp_threshold_100_intended_diff_only": 0.19520000219345093,
143
+ "tpp_threshold_100_unintended_diff_only": 0.011799997091293335,
144
+ "tpp_threshold_500_total_metric": 0.3623000055551529,
145
+ "tpp_threshold_500_intended_diff_only": 0.3922000050544739,
146
+ "tpp_threshold_500_unintended_diff_only": 0.029899999499320984
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.0020000189542770386,
194
  "tpp_threshold_10_total_metric": 0.10624998807907104,
195
  "tpp_threshold_10_intended_diff_only": 0.12400001287460327,
196
+ "tpp_threshold_10_unintended_diff_only": 0.017750024795532227,
197
+ "tpp_threshold_20_total_metric": 0.1942499577999115,
198
+ "tpp_threshold_20_intended_diff_only": 0.22899997234344482,
199
+ "tpp_threshold_20_unintended_diff_only": 0.034750014543533325,
200
+ "tpp_threshold_50_total_metric": 0.2789999842643738,
201
+ "tpp_threshold_50_intended_diff_only": 0.34700000286102295,
202
+ "tpp_threshold_50_unintended_diff_only": 0.06800001859664917,
203
+ "tpp_threshold_100_total_metric": 0.27274997532367706,
204
+ "tpp_threshold_100_intended_diff_only": 0.4179999828338623,
205
+ "tpp_threshold_100_unintended_diff_only": 0.14525000751018524,
206
+ "tpp_threshold_500_total_metric": 0.1419999897480011,
207
+ "tpp_threshold_500_intended_diff_only": 0.45600003004074097,
208
+ "tpp_threshold_500_unintended_diff_only": 0.31400004029273987
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.04074995219707489,
212
  "tpp_threshold_2_intended_diff_only": 0.04799997806549072,
213
+ "tpp_threshold_2_unintended_diff_only": 0.0072500258684158325,
214
+ "tpp_threshold_5_total_metric": 0.18400001525878906,
215
  "tpp_threshold_5_intended_diff_only": 0.23000001907348633,
216
+ "tpp_threshold_5_unintended_diff_only": 0.046000003814697266,
217
+ "tpp_threshold_10_total_metric": 0.20000001788139343,
218
+ "tpp_threshold_10_intended_diff_only": 0.281000018119812,
219
+ "tpp_threshold_10_unintended_diff_only": 0.08100000023841858,
220
+ "tpp_threshold_20_total_metric": 0.2292500138282776,
221
+ "tpp_threshold_20_intended_diff_only": 0.31800001859664917,
222
+ "tpp_threshold_20_unintended_diff_only": 0.08875000476837158,
223
+ "tpp_threshold_50_total_metric": 0.24375002086162567,
224
+ "tpp_threshold_50_intended_diff_only": 0.37300002574920654,
225
+ "tpp_threshold_50_unintended_diff_only": 0.12925000488758087,
226
+ "tpp_threshold_100_total_metric": 0.19874998927116394,
227
+ "tpp_threshold_100_intended_diff_only": 0.42000001668930054,
228
+ "tpp_threshold_100_unintended_diff_only": 0.2212500274181366,
229
+ "tpp_threshold_500_total_metric": 0.11475002765655518,
230
+ "tpp_threshold_500_intended_diff_only": 0.4590000510215759,
231
+ "tpp_threshold_500_unintended_diff_only": 0.34425002336502075
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.026750028133392334,
235
+ "tpp_threshold_2_intended_diff_only": 0.03100001811981201,
236
+ "tpp_threshold_2_unintended_diff_only": 0.004249989986419678,
237
+ "tpp_threshold_5_total_metric": 0.19725003838539124,
238
+ "tpp_threshold_5_intended_diff_only": 0.21000003814697266,
239
+ "tpp_threshold_5_unintended_diff_only": 0.012749999761581421,
240
+ "tpp_threshold_10_total_metric": 0.24250006675720215,
241
+ "tpp_threshold_10_intended_diff_only": 0.30500006675720215,
242
+ "tpp_threshold_10_unintended_diff_only": 0.0625,
243
+ "tpp_threshold_20_total_metric": 0.2447500079870224,
244
+ "tpp_threshold_20_intended_diff_only": 0.35500001907348633,
245
+ "tpp_threshold_20_unintended_diff_only": 0.11025001108646393,
246
+ "tpp_threshold_50_total_metric": 0.19425000250339508,
247
+ "tpp_threshold_50_intended_diff_only": 0.3830000162124634,
248
+ "tpp_threshold_50_unintended_diff_only": 0.1887500137090683,
249
+ "tpp_threshold_100_total_metric": 0.14525005221366882,
250
+ "tpp_threshold_100_intended_diff_only": 0.4160000681877136,
251
+ "tpp_threshold_100_unintended_diff_only": 0.2707500159740448,
252
+ "tpp_threshold_500_total_metric": 0.06650000810623169,
253
+ "tpp_threshold_500_intended_diff_only": 0.4520000219345093,
254
+ "tpp_threshold_500_unintended_diff_only": 0.3855000138282776
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.010499969124794006,
258
  "tpp_threshold_2_intended_diff_only": 0.009999990463256836,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704,
260
+ "tpp_threshold_5_total_metric": 0.05250002443790436,
261
+ "tpp_threshold_5_intended_diff_only": 0.058000028133392334,
262
+ "tpp_threshold_5_unintended_diff_only": 0.005500003695487976,
263
+ "tpp_threshold_10_total_metric": 0.2630000412464142,
264
  "tpp_threshold_10_intended_diff_only": 0.315000057220459,
265
+ "tpp_threshold_10_unintended_diff_only": 0.0520000159740448,
266
+ "tpp_threshold_20_total_metric": 0.2950000464916229,
267
+ "tpp_threshold_20_intended_diff_only": 0.3800000548362732,
268
+ "tpp_threshold_20_unintended_diff_only": 0.08500000834465027,
269
+ "tpp_threshold_50_total_metric": 0.30250002443790436,
270
+ "tpp_threshold_50_intended_diff_only": 0.4740000367164612,
271
+ "tpp_threshold_50_unintended_diff_only": 0.17150001227855682,
272
+ "tpp_threshold_100_total_metric": 0.2340000420808792,
273
+ "tpp_threshold_100_intended_diff_only": 0.48200005292892456,
274
+ "tpp_threshold_100_unintended_diff_only": 0.24800001084804535,
275
+ "tpp_threshold_500_total_metric": 0.16825003921985626,
276
+ "tpp_threshold_500_intended_diff_only": 0.49100005626678467,
277
+ "tpp_threshold_500_unintended_diff_only": 0.3227500170469284
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.006750017404556274,
281
+ "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0012499988079071045,
283
+ "tpp_threshold_5_total_metric": 0.016000032424926758,
284
+ "tpp_threshold_5_intended_diff_only": 0.025000035762786865,
285
+ "tpp_threshold_5_unintended_diff_only": 0.009000003337860107,
286
+ "tpp_threshold_10_total_metric": 0.17000003159046173,
287
+ "tpp_threshold_10_intended_diff_only": 0.21900004148483276,
288
+ "tpp_threshold_10_unintended_diff_only": 0.04900000989437103,
289
+ "tpp_threshold_20_total_metric": 0.21550002694129944,
290
+ "tpp_threshold_20_intended_diff_only": 0.26100003719329834,
291
+ "tpp_threshold_20_unintended_diff_only": 0.0455000102519989,
292
+ "tpp_threshold_50_total_metric": 0.2800000309944153,
293
+ "tpp_threshold_50_intended_diff_only": 0.3850000500679016,
294
+ "tpp_threshold_50_unintended_diff_only": 0.10500001907348633,
295
+ "tpp_threshold_100_total_metric": 0.2732499986886978,
296
+ "tpp_threshold_100_intended_diff_only": 0.43800002336502075,
297
+ "tpp_threshold_100_unintended_diff_only": 0.16475002467632294,
298
+ "tpp_threshold_500_total_metric": 0.15700004994869232,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.3190000206232071
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.0029999762773513794,
306
+ "tpp_threshold_2_intended_diff_only": 0.006999969482421875,
307
+ "tpp_threshold_2_unintended_diff_only": 0.003999993205070496,
308
+ "tpp_threshold_5_total_metric": 0.006000027060508728,
309
+ "tpp_threshold_5_intended_diff_only": 0.008000016212463379,
310
+ "tpp_threshold_5_unintended_diff_only": 0.001999989151954651,
311
+ "tpp_threshold_10_total_metric": 0.002500012516975403,
312
  "tpp_threshold_10_intended_diff_only": 0.004999995231628418,
313
+ "tpp_threshold_10_unintended_diff_only": 0.002499982714653015,
314
+ "tpp_threshold_20_total_metric": 0.009000033140182495,
315
+ "tpp_threshold_20_intended_diff_only": 0.012000024318695068,
316
+ "tpp_threshold_20_unintended_diff_only": 0.0029999911785125732,
317
+ "tpp_threshold_50_total_metric": 0.04874996840953827,
318
+ "tpp_threshold_50_intended_diff_only": 0.04799997806549072,
319
+ "tpp_threshold_50_unintended_diff_only": -0.0007499903440475464,
320
+ "tpp_threshold_100_total_metric": 0.09875001013278961,
321
+ "tpp_threshold_100_intended_diff_only": 0.10600000619888306,
322
+ "tpp_threshold_100_unintended_diff_only": 0.007249996066093445,
323
+ "tpp_threshold_500_total_metric": 0.3802500069141388,
324
+ "tpp_threshold_500_intended_diff_only": 0.3889999985694885,
325
+ "tpp_threshold_500_unintended_diff_only": 0.008749991655349731
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.010000035166740417,
329
+ "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
330
+ "tpp_threshold_2_unintended_diff_only": -0.0020000189542770386,
331
+ "tpp_threshold_5_total_metric": 0.011250078678131104,
332
+ "tpp_threshold_5_intended_diff_only": 0.01900005340576172,
333
+ "tpp_threshold_5_unintended_diff_only": 0.007749974727630615,
334
+ "tpp_threshold_10_total_metric": 0.04675005376338959,
335
+ "tpp_threshold_10_intended_diff_only": 0.053000032901763916,
336
+ "tpp_threshold_10_unintended_diff_only": 0.006249979138374329,
337
+ "tpp_threshold_20_total_metric": 0.08725002408027649,
338
+ "tpp_threshold_20_intended_diff_only": 0.09600001573562622,
339
+ "tpp_threshold_20_unintended_diff_only": 0.008749991655349731,
340
+ "tpp_threshold_50_total_metric": 0.18550004065036774,
341
+ "tpp_threshold_50_intended_diff_only": 0.20200002193450928,
342
+ "tpp_threshold_50_unintended_diff_only": 0.01649998128414154,
343
+ "tpp_threshold_100_total_metric": 0.30000004172325134,
344
+ "tpp_threshold_100_intended_diff_only": 0.32600003480911255,
345
+ "tpp_threshold_100_unintended_diff_only": 0.025999993085861206,
346
+ "tpp_threshold_500_total_metric": 0.40525004267692566,
347
+ "tpp_threshold_500_intended_diff_only": 0.437000036239624,
348
+ "tpp_threshold_500_unintended_diff_only": 0.031749993562698364
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.00475001335144043,
352
+ "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
353
+ "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402,
354
+ "tpp_threshold_5_total_metric": 0.01300002634525299,
355
+ "tpp_threshold_5_intended_diff_only": 0.00700002908706665,
356
+ "tpp_threshold_5_unintended_diff_only": -0.00599999725818634,
357
+ "tpp_threshold_10_total_metric": 0.015250027179718018,
358
+ "tpp_threshold_10_intended_diff_only": 0.012000024318695068,
359
+ "tpp_threshold_10_unintended_diff_only": -0.0032500028610229492,
360
+ "tpp_threshold_20_total_metric": 0.01400001347064972,
361
+ "tpp_threshold_20_intended_diff_only": 0.018000006675720215,
362
+ "tpp_threshold_20_unintended_diff_only": 0.003999993205070496,
363
+ "tpp_threshold_50_total_metric": 0.05400000512599945,
364
+ "tpp_threshold_50_intended_diff_only": 0.0559999942779541,
365
+ "tpp_threshold_50_unintended_diff_only": 0.001999989151954651,
366
+ "tpp_threshold_100_total_metric": 0.10975001752376556,
367
+ "tpp_threshold_100_intended_diff_only": 0.11500000953674316,
368
+ "tpp_threshold_100_unintended_diff_only": 0.0052499920129776,
369
+ "tpp_threshold_500_total_metric": 0.33400000631809235,
370
+ "tpp_threshold_500_intended_diff_only": 0.35199999809265137,
371
+ "tpp_threshold_500_unintended_diff_only": 0.01799999177455902
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.0004999786615371704,
375
+ "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
376
+ "tpp_threshold_2_unintended_diff_only": 0.0005000084638595581,
377
+ "tpp_threshold_5_total_metric": 0.012999966740608215,
378
+ "tpp_threshold_5_intended_diff_only": 0.011999964714050293,
379
+ "tpp_threshold_5_unintended_diff_only": -0.0010000020265579224,
380
+ "tpp_threshold_10_total_metric": 0.0032499581575393677,
381
+ "tpp_threshold_10_intended_diff_only": 0.007999956607818604,
382
+ "tpp_threshold_10_unintended_diff_only": 0.004749998450279236,
383
+ "tpp_threshold_20_total_metric": 0.018749982118606567,
384
+ "tpp_threshold_20_intended_diff_only": 0.019999980926513672,
385
+ "tpp_threshold_20_unintended_diff_only": 0.0012499988079071045,
386
+ "tpp_threshold_50_total_metric": 0.09974999725818634,
387
+ "tpp_threshold_50_intended_diff_only": 0.10600000619888306,
388
+ "tpp_threshold_50_unintended_diff_only": 0.006250008940696716,
389
+ "tpp_threshold_100_total_metric": 0.15024995803833008,
390
+ "tpp_threshold_100_intended_diff_only": 0.15999996662139893,
391
+ "tpp_threshold_100_unintended_diff_only": 0.009750008583068848,
392
+ "tpp_threshold_500_total_metric": 0.3464999496936798,
393
+ "tpp_threshold_500_intended_diff_only": 0.4099999666213989,
394
+ "tpp_threshold_500_unintended_diff_only": 0.06350001692771912
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.013750001788139343,
398
+ "tpp_threshold_2_intended_diff_only": 0.018999993801116943,
399
+ "tpp_threshold_2_unintended_diff_only": 0.0052499920129776,
400
+ "tpp_threshold_5_total_metric": 0.0417499840259552,
401
+ "tpp_threshold_5_intended_diff_only": 0.041999995708465576,
402
+ "tpp_threshold_5_unintended_diff_only": 0.000250011682510376,
403
+ "tpp_threshold_10_total_metric": 0.07400000095367432,
404
+ "tpp_threshold_10_intended_diff_only": 0.07899999618530273,
405
+ "tpp_threshold_10_unintended_diff_only": 0.004999995231628418,
406
+ "tpp_threshold_20_total_metric": 0.11574998497962952,
407
+ "tpp_threshold_20_intended_diff_only": 0.12699997425079346,
408
+ "tpp_threshold_20_unintended_diff_only": 0.01124998927116394,
409
+ "tpp_threshold_50_total_metric": 0.1939999908208847,
410
+ "tpp_threshold_50_intended_diff_only": 0.20499998331069946,
411
+ "tpp_threshold_50_unintended_diff_only": 0.010999992489814758,
412
+ "tpp_threshold_100_total_metric": 0.25824999809265137,
413
+ "tpp_threshold_100_intended_diff_only": 0.26899999380111694,
414
+ "tpp_threshold_100_unintended_diff_only": 0.010749995708465576,
415
+ "tpp_threshold_500_total_metric": 0.34550002217292786,
416
+ "tpp_threshold_500_intended_diff_only": 0.37300002574920654,
417
+ "tpp_threshold_500_unintended_diff_only": 0.027500003576278687
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-12207-tokens-50000000/scr/saebench_scr_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,8 +69,8 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "06f0ebff-0b93-452d-8db9-accbec9aa577",
69
- "datetime_epoch_millis": 1776883672083,
70
  "eval_result_metrics": {
71
  "scr_metrics": {
72
  "scr_dir1_threshold_2": 0.20930437004095767,
@@ -78,18 +82,18 @@
78
  "scr_dir1_threshold_10": 0.25138475324743553,
79
  "scr_metric_threshold_10": 0.2690253055472029,
80
  "scr_dir2_threshold_10": 0.2691963281664894,
81
- "scr_dir1_threshold_20": null,
82
- "scr_metric_threshold_20": null,
83
- "scr_dir2_threshold_20": null,
84
- "scr_dir1_threshold_50": null,
85
- "scr_metric_threshold_50": null,
86
- "scr_dir2_threshold_50": null,
87
- "scr_dir1_threshold_100": null,
88
- "scr_metric_threshold_100": null,
89
- "scr_dir2_threshold_100": null,
90
- "scr_dir1_threshold_500": null,
91
- "scr_metric_threshold_500": null,
92
- "scr_dir2_threshold_500": null
93
  }
94
  },
95
  "eval_result_details": [
@@ -104,18 +108,18 @@
104
  "scr_dir1_threshold_10": 0.5384614208981502,
105
  "scr_metric_threshold_10": 0.11658034288417893,
106
  "scr_dir2_threshold_10": 0.11658034288417893,
107
- "scr_dir1_threshold_20": null,
108
- "scr_metric_threshold_20": null,
109
- "scr_dir2_threshold_20": null,
110
- "scr_dir1_threshold_50": null,
111
- "scr_metric_threshold_50": null,
112
- "scr_dir2_threshold_50": null,
113
- "scr_dir1_threshold_100": null,
114
- "scr_metric_threshold_100": null,
115
- "scr_dir2_threshold_100": null,
116
- "scr_dir1_threshold_500": null,
117
- "scr_metric_threshold_500": null,
118
- "scr_dir2_threshold_500": null
119
  },
120
  {
121
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
@@ -128,18 +132,18 @@
128
  "scr_dir1_threshold_10": 0.21212114370450913,
129
  "scr_metric_threshold_10": 0.34250758728798886,
130
  "scr_dir2_threshold_10": 0.34250758728798886,
131
- "scr_dir1_threshold_20": null,
132
- "scr_metric_threshold_20": null,
133
- "scr_dir2_threshold_20": null,
134
- "scr_dir1_threshold_50": null,
135
- "scr_metric_threshold_50": null,
136
- "scr_dir2_threshold_50": null,
137
- "scr_dir1_threshold_100": null,
138
- "scr_metric_threshold_100": null,
139
- "scr_dir2_threshold_100": null,
140
- "scr_dir1_threshold_500": null,
141
- "scr_metric_threshold_500": null,
142
- "scr_dir2_threshold_500": null
143
  },
144
  {
145
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
@@ -152,18 +156,18 @@
152
  "scr_dir1_threshold_10": 0.5312495925464404,
153
  "scr_metric_threshold_10": 0.10301508102112025,
154
  "scr_dir2_threshold_10": 0.10301508102112025,
155
- "scr_dir1_threshold_20": null,
156
- "scr_metric_threshold_20": null,
157
- "scr_dir2_threshold_20": null,
158
- "scr_dir1_threshold_50": null,
159
- "scr_metric_threshold_50": null,
160
- "scr_dir2_threshold_50": null,
161
- "scr_dir1_threshold_100": null,
162
- "scr_metric_threshold_100": null,
163
- "scr_dir2_threshold_100": null,
164
- "scr_dir1_threshold_500": null,
165
- "scr_metric_threshold_500": null,
166
- "scr_dir2_threshold_500": null
167
  },
168
  {
169
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
@@ -176,18 +180,18 @@
176
  "scr_dir1_threshold_10": 0.24832216912919208,
177
  "scr_metric_threshold_10": 0.27388542165349355,
178
  "scr_dir2_threshold_10": 0.27388542165349355,
179
- "scr_dir1_threshold_20": null,
180
- "scr_metric_threshold_20": null,
181
- "scr_dir2_threshold_20": null,
182
- "scr_dir1_threshold_50": null,
183
- "scr_metric_threshold_50": null,
184
- "scr_dir2_threshold_50": null,
185
- "scr_dir1_threshold_100": null,
186
- "scr_metric_threshold_100": null,
187
- "scr_dir2_threshold_100": null,
188
- "scr_dir1_threshold_500": null,
189
- "scr_metric_threshold_500": null,
190
- "scr_dir2_threshold_500": null
191
  },
192
  {
193
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
@@ -200,18 +204,18 @@
200
  "scr_dir1_threshold_10": 0.0631580235225028,
201
  "scr_metric_threshold_10": 0.5198413665167169,
202
  "scr_dir2_threshold_10": 0.5198413665167169,
203
- "scr_dir1_threshold_20": null,
204
- "scr_metric_threshold_20": null,
205
- "scr_dir2_threshold_20": null,
206
- "scr_dir1_threshold_50": null,
207
- "scr_metric_threshold_50": null,
208
- "scr_dir2_threshold_50": null,
209
- "scr_dir1_threshold_100": null,
210
- "scr_metric_threshold_100": null,
211
- "scr_dir2_threshold_100": null,
212
- "scr_dir1_threshold_500": null,
213
- "scr_metric_threshold_500": null,
214
- "scr_dir2_threshold_500": null
215
  },
216
  {
217
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
@@ -224,18 +228,18 @@
224
  "scr_dir1_threshold_10": 0.21311465264817295,
225
  "scr_metric_threshold_10": 0.15648856785294307,
226
  "scr_dir2_threshold_10": 0.15648856785294307,
227
- "scr_dir1_threshold_20": null,
228
- "scr_metric_threshold_20": null,
229
- "scr_dir2_threshold_20": null,
230
- "scr_dir1_threshold_50": null,
231
- "scr_metric_threshold_50": null,
232
- "scr_dir2_threshold_50": null,
233
- "scr_dir1_threshold_100": null,
234
- "scr_metric_threshold_100": null,
235
- "scr_dir2_threshold_100": null,
236
- "scr_dir1_threshold_500": null,
237
- "scr_metric_threshold_500": null,
238
- "scr_dir2_threshold_500": null
239
  },
240
  {
241
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
@@ -248,18 +252,18 @@
248
  "scr_dir1_threshold_10": 0.08837205047154198,
249
  "scr_metric_threshold_10": 0.5236051041022068,
250
  "scr_dir2_threshold_10": 0.5236051041022068,
251
- "scr_dir1_threshold_20": null,
252
- "scr_metric_threshold_20": null,
253
- "scr_dir2_threshold_20": null,
254
- "scr_dir1_threshold_50": null,
255
- "scr_metric_threshold_50": null,
256
- "scr_dir2_threshold_50": null,
257
- "scr_dir1_threshold_100": null,
258
- "scr_metric_threshold_100": null,
259
- "scr_dir2_threshold_100": null,
260
- "scr_dir1_threshold_500": null,
261
- "scr_metric_threshold_500": null,
262
- "scr_dir2_threshold_500": null
263
  },
264
  {
265
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
@@ -272,18 +276,18 @@
272
  "scr_dir1_threshold_10": 0.1162789730589745,
273
  "scr_metric_threshold_10": 0.1162789730589745,
274
  "scr_dir2_threshold_10": 0.11764715401326675,
275
- "scr_dir1_threshold_20": null,
276
- "scr_metric_threshold_20": null,
277
- "scr_dir2_threshold_20": null,
278
- "scr_dir1_threshold_50": null,
279
- "scr_metric_threshold_50": null,
280
- "scr_dir2_threshold_50": null,
281
- "scr_dir1_threshold_100": null,
282
- "scr_metric_threshold_100": null,
283
- "scr_dir2_threshold_100": null,
284
- "scr_dir1_threshold_500": null,
285
- "scr_metric_threshold_500": null,
286
- "scr_dir2_threshold_500": null
287
  }
288
  ],
289
  "sae_bench_commit_hash": "Unknown",
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "f13a845e-4d8e-420f-a76e-c8300215f674",
73
+ "datetime_epoch_millis": 1777315454740,
74
  "eval_result_metrics": {
75
  "scr_metrics": {
76
  "scr_dir1_threshold_2": 0.20930437004095767,
 
82
  "scr_dir1_threshold_10": 0.25138475324743553,
83
  "scr_metric_threshold_10": 0.2690253055472029,
84
  "scr_dir2_threshold_10": 0.2691963281664894,
85
+ "scr_dir1_threshold_20": 0.2509970209733557,
86
+ "scr_metric_threshold_20": 0.32767724537306003,
87
+ "scr_dir2_threshold_20": 0.32541218987999554,
88
+ "scr_dir1_threshold_50": 0.23604931453324968,
89
+ "scr_metric_threshold_50": 0.4076231162769153,
90
+ "scr_dir2_threshold_50": 0.40576581846881726,
91
+ "scr_dir1_threshold_100": 0.10870387574017482,
92
+ "scr_metric_threshold_100": 0.44131054126569613,
93
+ "scr_dir2_threshold_100": 0.4490844228433037,
94
+ "scr_dir1_threshold_500": 0.10266567723365157,
95
+ "scr_metric_threshold_500": 0.3333851707358609,
96
+ "scr_dir2_threshold_500": 0.3470808513435148
97
  }
98
  },
99
  "eval_result_details": [
 
108
  "scr_dir1_threshold_10": 0.5384614208981502,
109
  "scr_metric_threshold_10": 0.11658034288417893,
110
  "scr_dir2_threshold_10": 0.11658034288417893,
111
+ "scr_dir1_threshold_20": 0.5769232238773122,
112
+ "scr_metric_threshold_20": 0.14248705143262683,
113
+ "scr_dir2_threshold_20": 0.14248705143262683,
114
+ "scr_dir1_threshold_50": 0.5769232238773122,
115
+ "scr_metric_threshold_50": 0.25906739431680575,
116
+ "scr_dir2_threshold_50": 0.25906739431680575,
117
+ "scr_dir1_threshold_100": 0.4487182328294704,
118
+ "scr_metric_threshold_100": 0.27720202853425396,
119
+ "scr_dir2_threshold_100": 0.27720202853425396,
120
+ "scr_dir1_threshold_500": 0.10256429850308302,
121
+ "scr_metric_threshold_500": 0.17616583431207444,
122
+ "scr_dir2_threshold_500": 0.17616583431207444
123
  },
124
  {
125
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
 
132
  "scr_dir1_threshold_10": 0.21212114370450913,
133
  "scr_metric_threshold_10": 0.34250758728798886,
134
  "scr_dir2_threshold_10": 0.34250758728798886,
135
+ "scr_dir1_threshold_20": 0.24999988711244006,
136
+ "scr_metric_threshold_20": 0.4128439798401429,
137
+ "scr_dir2_threshold_20": 0.4128439798401429,
138
+ "scr_dir1_threshold_50": 0.15151542518196345,
139
+ "scr_metric_threshold_50": 0.5321101318775462,
140
+ "scr_dir2_threshold_50": 0.5321101318775462,
141
+ "scr_dir1_threshold_100": -0.7272726862227055,
142
+ "scr_metric_threshold_100": 0.6116208999024817,
143
+ "scr_dir2_threshold_100": 0.6116208999024817,
144
+ "scr_dir1_threshold_500": 0.23484857036936357,
145
+ "scr_metric_threshold_500": 0.2599389371417114,
146
+ "scr_dir2_threshold_500": 0.2599389371417114
147
  },
148
  {
149
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
 
156
  "scr_dir1_threshold_10": 0.5312495925464404,
157
  "scr_metric_threshold_10": 0.10301508102112025,
158
  "scr_dir2_threshold_10": 0.10301508102112025,
159
+ "scr_dir1_threshold_20": 0.3749997671693945,
160
+ "scr_metric_threshold_20": 0.1457286913802262,
161
+ "scr_dir2_threshold_20": 0.1457286913802262,
162
+ "scr_dir1_threshold_50": 0.32812537834973393,
163
+ "scr_metric_threshold_50": 0.22613070218304224,
164
+ "scr_dir2_threshold_50": 0.22613070218304224,
165
+ "scr_dir1_threshold_100": 0.250000465661211,
166
+ "scr_metric_threshold_100": 0.29145723300006154,
167
+ "scr_dir2_threshold_100": 0.29145723300006154,
168
+ "scr_dir1_threshold_500": 0.03125052386886235,
169
+ "scr_metric_threshold_500": 0.05276388042950681,
170
+ "scr_dir2_threshold_500": 0.05276388042950681
171
  },
172
  {
173
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
 
180
  "scr_dir1_threshold_10": 0.24832216912919208,
181
  "scr_metric_threshold_10": 0.27388542165349355,
182
  "scr_dir2_threshold_10": 0.27388542165349355,
183
+ "scr_dir1_threshold_20": 0.04026834093059259,
184
+ "scr_metric_threshold_20": 0.37579626297872115,
185
+ "scr_dir2_threshold_20": 0.37579626297872115,
186
+ "scr_dir1_threshold_50": 0.08724800534441686,
187
+ "scr_metric_threshold_50": 0.4968153277325217,
188
+ "scr_dir2_threshold_50": 0.4968153277325217,
189
+ "scr_dir1_threshold_100": 0.13422806978944357,
190
+ "scr_metric_threshold_100": 0.5509555155744654,
191
+ "scr_dir2_threshold_100": 0.5509555155744654,
192
+ "scr_dir1_threshold_500": -0.1140940993397485,
193
+ "scr_metric_threshold_500": 0.1305732713799386,
194
+ "scr_dir2_threshold_500": 0.1305732713799386
195
  },
196
  {
197
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
 
204
  "scr_dir1_threshold_10": 0.0631580235225028,
205
  "scr_metric_threshold_10": 0.5198413665167169,
206
  "scr_dir2_threshold_10": 0.5198413665167169,
207
+ "scr_dir1_threshold_20": 0.12105264313663487,
208
+ "scr_metric_threshold_20": 0.5753969089319204,
209
+ "scr_dir2_threshold_20": 0.5753969089319204,
210
+ "scr_dir1_threshold_50": -0.026315764707211,
211
+ "scr_metric_threshold_50": 0.595238157185469,
212
+ "scr_dir2_threshold_50": 0.595238157185469,
213
+ "scr_dir1_threshold_100": -0.07368420392192293,
214
+ "scr_metric_threshold_100": 0.6269840124753451,
215
+ "scr_dir2_threshold_100": 0.6269840124753451,
216
+ "scr_dir1_threshold_500": -0.13157882353605502,
217
+ "scr_metric_threshold_500": 0.6190476077844602,
218
+ "scr_dir2_threshold_500": 0.6190476077844602
219
  },
220
  {
221
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
 
228
  "scr_dir1_threshold_10": 0.21311465264817295,
229
  "scr_metric_threshold_10": 0.15648856785294307,
230
  "scr_dir2_threshold_10": 0.15648856785294307,
231
+ "scr_dir1_threshold_20": 0.3005465388586523,
232
+ "scr_metric_threshold_20": 0.2137404163361301,
233
+ "scr_dir2_threshold_20": 0.2137404163361301,
234
+ "scr_dir1_threshold_50": 0.2732241558450018,
235
+ "scr_metric_threshold_50": 0.29389318621149046,
236
+ "scr_dir2_threshold_50": 0.29389318621149046,
237
+ "scr_dir1_threshold_100": 0.2841529787670631,
238
+ "scr_metric_threshold_100": 0.3320610852002818,
239
+ "scr_dir2_threshold_100": 0.3320610852002818,
240
+ "scr_dir1_threshold_500": 0.3169397732417441,
241
+ "scr_metric_threshold_500": 0.5343510408403254,
242
+ "scr_dir2_threshold_500": 0.5343510408403254
243
  },
244
  {
245
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
 
252
  "scr_dir1_threshold_10": 0.08837205047154198,
253
  "scr_metric_threshold_10": 0.5236051041022068,
254
  "scr_dir2_threshold_10": 0.5236051041022068,
255
+ "scr_dir1_threshold_20": 0.17674410094308396,
256
+ "scr_metric_threshold_20": 0.587982986325977,
257
+ "scr_dir2_threshold_20": 0.587982986325977,
258
+ "scr_dir1_threshold_50": 0.28372083739245524,
259
+ "scr_metric_threshold_50": 0.6437767757261215,
260
+ "scr_dir2_threshold_50": 0.6437767757261215,
261
+ "scr_dir1_threshold_100": 0.3441859975126847,
262
+ "scr_metric_threshold_100": 0.6309014039325255,
263
+ "scr_dir2_threshold_100": 0.6309014039325255,
264
+ "scr_dir1_threshold_500": 0.08372094828480038,
265
+ "scr_metric_threshold_500": 0.5965665675217078,
266
+ "scr_dir2_threshold_500": 0.5965665675217078
267
  },
268
  {
269
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
 
276
  "scr_dir1_threshold_10": 0.1162789730589745,
277
  "scr_metric_threshold_10": 0.1162789730589745,
278
  "scr_dir2_threshold_10": 0.11764715401326675,
279
+ "scr_dir1_threshold_20": 0.16744166575873534,
280
+ "scr_metric_threshold_20": 0.16744166575873534,
281
+ "scr_dir2_threshold_20": 0.1493212218142196,
282
+ "scr_dir1_threshold_50": 0.21395325498232515,
283
+ "scr_metric_threshold_50": 0.21395325498232515,
284
+ "scr_dir2_threshold_50": 0.19909487251754088,
285
+ "scr_dir1_threshold_100": 0.2093021515061541,
286
+ "scr_metric_threshold_100": 0.2093021515061541,
287
+ "scr_dir2_threshold_100": 0.2714932041270145,
288
+ "scr_dir1_threshold_500": 0.2976742264771627,
289
+ "scr_metric_threshold_500": 0.2976742264771627,
290
+ "scr_dir2_threshold_500": 0.4072396713383939
291
  }
292
  ],
293
  "sae_bench_commit_hash": "Unknown",
gemma-2-2b-btk/k-100/seed-0/snapshots/step-12207-tokens-50000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "b2714d67-9f92-442f-852f-7db2ff16de12",
69
- "datetime_epoch_millis": 1776883839636,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.012475007772445678,
73
- "tpp_threshold_2_intended_diff_only": 0.01220000982284546,
74
- "tpp_threshold_2_unintended_diff_only": -0.0002749979496002197,
75
- "tpp_threshold_5_total_metric": 0.015849997103214265,
76
- "tpp_threshold_5_intended_diff_only": 0.01790000200271606,
77
- "tpp_threshold_5_unintended_diff_only": 0.0020500048995018005,
78
- "tpp_threshold_10_total_metric": 0.02477500140666962,
79
- "tpp_threshold_10_intended_diff_only": 0.027300006151199343,
80
- "tpp_threshold_10_unintended_diff_only": 0.0025250047445297243,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.015350013971328735,
99
- "tpp_threshold_2_intended_diff_only": 0.01500002145767212,
100
- "tpp_threshold_2_unintended_diff_only": -0.0003499925136566162,
101
- "tpp_threshold_5_total_metric": 0.016349998116493226,
102
- "tpp_threshold_5_intended_diff_only": 0.018200016021728514,
103
- "tpp_threshold_5_unintended_diff_only": 0.0018500179052352906,
104
- "tpp_threshold_10_total_metric": 0.02030000984668732,
105
- "tpp_threshold_10_intended_diff_only": 0.022600018978118898,
106
- "tpp_threshold_10_unintended_diff_only": 0.0023000091314315796,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.009600001573562621,
123
- "tpp_threshold_2_intended_diff_only": 0.009399998188018798,
124
- "tpp_threshold_2_unintended_diff_only": -0.00020000338554382324,
125
- "tpp_threshold_5_total_metric": 0.015349996089935303,
126
- "tpp_threshold_5_intended_diff_only": 0.017599987983703613,
127
- "tpp_threshold_5_unintended_diff_only": 0.0022499918937683105,
128
- "tpp_threshold_10_total_metric": 0.029249992966651917,
129
- "tpp_threshold_10_intended_diff_only": 0.03199999332427979,
130
- "tpp_threshold_10_unintended_diff_only": 0.0027500003576278686,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.0015000253915786743,
190
  "tpp_threshold_10_total_metric": 0.00424996018409729,
191
  "tpp_threshold_10_intended_diff_only": 0.006999969482421875,
192
- "tpp_threshold_10_unintended_diff_only": 0.002750009298324585
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": 0.00100003182888031,
196
- "tpp_threshold_2_intended_diff_only": 0.0020000338554382324,
197
- "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224,
198
- "tpp_threshold_5_total_metric": 0.0034999847412109375,
199
- "tpp_threshold_5_intended_diff_only": 0.004999995231628418,
200
- "tpp_threshold_5_unintended_diff_only": 0.0015000104904174805,
201
- "tpp_threshold_10_total_metric": 0.004250004887580872,
202
- "tpp_threshold_10_intended_diff_only": 0.008000016212463379,
203
- "tpp_threshold_10_unintended_diff_only": 0.0037500113248825073
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.05475004017353058,
207
- "tpp_threshold_2_intended_diff_only": 0.05200004577636719,
208
- "tpp_threshold_2_unintended_diff_only": -0.002749994397163391,
209
- "tpp_threshold_5_total_metric": 0.05849997699260712,
210
- "tpp_threshold_5_intended_diff_only": 0.06000000238418579,
211
- "tpp_threshold_5_unintended_diff_only": 0.0015000253915786743,
212
- "tpp_threshold_10_total_metric": 0.06525003910064697,
213
- "tpp_threshold_10_intended_diff_only": 0.06600004434585571,
214
- "tpp_threshold_10_unintended_diff_only": 0.0007500052452087402
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.012999996542930603,
218
  "tpp_threshold_2_intended_diff_only": 0.009999990463256836,
219
- "tpp_threshold_2_unintended_diff_only": -0.003000006079673767,
220
- "tpp_threshold_5_total_metric": 0.003000035881996155,
221
- "tpp_threshold_5_intended_diff_only": 0.006000041961669922,
222
- "tpp_threshold_5_unintended_diff_only": 0.003000006079673767,
223
- "tpp_threshold_10_total_metric": 0.012750029563903809,
224
- "tpp_threshold_10_intended_diff_only": 0.01500004529953003,
225
- "tpp_threshold_10_unintended_diff_only": 0.0022500157356262207
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.0007500201463699341,
229
- "tpp_threshold_2_intended_diff_only": 0.0020000338554382324,
230
- "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983,
231
- "tpp_threshold_5_total_metric": 0.004250019788742065,
232
- "tpp_threshold_5_intended_diff_only": 0.006000041961669922,
233
- "tpp_threshold_5_unintended_diff_only": 0.0017500221729278564,
234
- "tpp_threshold_10_total_metric": 0.015000015497207642,
235
- "tpp_threshold_10_intended_diff_only": 0.017000019550323486,
236
- "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.009000003337860107,
242
- "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
243
- "tpp_threshold_2_unintended_diff_only": -0.004999995231628418,
244
- "tpp_threshold_5_total_metric": 0.008749991655349731,
245
- "tpp_threshold_5_intended_diff_only": 0.0059999823570251465,
246
- "tpp_threshold_5_unintended_diff_only": -0.002750009298324585,
247
- "tpp_threshold_10_total_metric": 0.0029999762773513794,
248
  "tpp_threshold_10_intended_diff_only": 0.006999969482421875,
249
- "tpp_threshold_10_unintended_diff_only": 0.003999993205070496
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": 0.009500056505203247,
253
- "tpp_threshold_2_intended_diff_only": 0.01500004529953003,
254
- "tpp_threshold_2_unintended_diff_only": 0.005499988794326782,
255
- "tpp_threshold_5_total_metric": 0.019000038504600525,
256
- "tpp_threshold_5_intended_diff_only": 0.026000022888183594,
257
- "tpp_threshold_5_unintended_diff_only": 0.006999984383583069,
258
- "tpp_threshold_10_total_metric": 0.04950004816055298,
259
- "tpp_threshold_10_intended_diff_only": 0.05200004577636719,
260
- "tpp_threshold_10_unintended_diff_only": 0.002499997615814209
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.012250006198883057,
264
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
265
- "tpp_threshold_2_unintended_diff_only": -0.007250010967254639,
266
- "tpp_threshold_5_total_metric": 0.015499994158744812,
267
- "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
268
- "tpp_threshold_5_unintended_diff_only": -0.0014999955892562866,
269
- "tpp_threshold_10_total_metric": 0.019250020384788513,
270
- "tpp_threshold_10_intended_diff_only": 0.021000027656555176,
271
- "tpp_threshold_10_unintended_diff_only": 0.0017500072717666626
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.004250019788742065,
275
  "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
276
- "tpp_threshold_2_unintended_diff_only": 0.0062499940395355225,
277
- "tpp_threshold_5_total_metric": -0.004250019788742065,
278
- "tpp_threshold_5_intended_diff_only": -0.0020000338554382324,
279
- "tpp_threshold_5_unintended_diff_only": 0.002249985933303833,
280
- "tpp_threshold_10_total_metric": 0.007999956607818604,
281
- "tpp_threshold_10_intended_diff_only": 0.011999964714050293,
282
- "tpp_threshold_10_unintended_diff_only": 0.0040000081062316895
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.021499961614608765,
286
- "tpp_threshold_2_intended_diff_only": 0.0209999680519104,
287
- "tpp_threshold_2_unintended_diff_only": -0.0004999935626983643,
288
- "tpp_threshold_5_total_metric": 0.03774997591972351,
289
- "tpp_threshold_5_intended_diff_only": 0.04399996995925903,
290
- "tpp_threshold_5_unintended_diff_only": 0.0062499940395355225,
291
- "tpp_threshold_10_total_metric": 0.06649996340274811,
292
- "tpp_threshold_10_intended_diff_only": 0.0679999589920044,
293
- "tpp_threshold_10_unintended_diff_only": 0.0014999955892562866
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "670be528-00c3-4655-b732-01db0612bc83",
73
+ "datetime_epoch_millis": 1777315727268,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.010499992966651916,
77
+ "tpp_threshold_2_intended_diff_only": 0.011399996280670167,
78
+ "tpp_threshold_2_unintended_diff_only": 0.0009000033140182495,
79
+ "tpp_threshold_5_total_metric": 0.01495000422000885,
80
+ "tpp_threshold_5_intended_diff_only": 0.015800011157989503,
81
+ "tpp_threshold_5_unintended_diff_only": 0.0008500069379806519,
82
+ "tpp_threshold_10_total_metric": 0.02567498981952667,
83
+ "tpp_threshold_10_intended_diff_only": 0.028399997949600218,
84
+ "tpp_threshold_10_unintended_diff_only": 0.0027250081300735475,
85
+ "tpp_threshold_20_total_metric": 0.046775002777576444,
86
+ "tpp_threshold_20_intended_diff_only": 0.05060001015663147,
87
+ "tpp_threshold_20_unintended_diff_only": 0.003825007379055023,
88
+ "tpp_threshold_50_total_metric": 0.0995750069618225,
89
+ "tpp_threshold_50_intended_diff_only": 0.1049000084400177,
90
+ "tpp_threshold_50_unintended_diff_only": 0.005325001478195191,
91
+ "tpp_threshold_100_total_metric": 0.20667501986026765,
92
+ "tpp_threshold_100_intended_diff_only": 0.2150000214576721,
93
+ "tpp_threshold_100_unintended_diff_only": 0.00832500159740448,
94
+ "tpp_threshold_500_total_metric": 0.39562502056360244,
95
+ "tpp_threshold_500_intended_diff_only": 0.4092000246047973,
96
+ "tpp_threshold_500_unintended_diff_only": 0.013575004041194917
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.013549983501434326,
103
+ "tpp_threshold_2_intended_diff_only": 0.01499999761581421,
104
+ "tpp_threshold_2_unintended_diff_only": 0.0014500141143798829,
105
+ "tpp_threshold_5_total_metric": 0.016849994659423828,
106
+ "tpp_threshold_5_intended_diff_only": 0.017400014400482177,
107
+ "tpp_threshold_5_unintended_diff_only": 0.0005500197410583496,
108
+ "tpp_threshold_10_total_metric": 0.020699986815452577,
109
+ "tpp_threshold_10_intended_diff_only": 0.022800004482269286,
110
+ "tpp_threshold_10_unintended_diff_only": 0.0021000176668167113,
111
+ "tpp_threshold_20_total_metric": 0.04675001204013825,
112
+ "tpp_threshold_20_intended_diff_only": 0.04900002479553223,
113
+ "tpp_threshold_20_unintended_diff_only": 0.0022500127553939818,
114
+ "tpp_threshold_50_total_metric": 0.09805000424385071,
115
+ "tpp_threshold_50_intended_diff_only": 0.10160001516342163,
116
+ "tpp_threshold_50_unintended_diff_only": 0.003550010919570923,
117
+ "tpp_threshold_100_total_metric": 0.22385002672672272,
118
+ "tpp_threshold_100_intended_diff_only": 0.2290000319480896,
119
+ "tpp_threshold_100_unintended_diff_only": 0.005150005221366882,
120
+ "tpp_threshold_500_total_metric": 0.43855002522468567,
121
+ "tpp_threshold_500_intended_diff_only": 0.44720003604888914,
122
+ "tpp_threshold_500_unintended_diff_only": 0.008650010824203492
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.007450002431869507,
127
+ "tpp_threshold_2_intended_diff_only": 0.007799994945526123,
128
+ "tpp_threshold_2_unintended_diff_only": 0.0003499925136566162,
129
+ "tpp_threshold_5_total_metric": 0.013050013780593872,
130
+ "tpp_threshold_5_intended_diff_only": 0.014200007915496827,
131
+ "tpp_threshold_5_unintended_diff_only": 0.0011499941349029541,
132
+ "tpp_threshold_10_total_metric": 0.03064999282360077,
133
+ "tpp_threshold_10_intended_diff_only": 0.03399999141693115,
134
+ "tpp_threshold_10_unintended_diff_only": 0.0033499985933303834,
135
+ "tpp_threshold_20_total_metric": 0.04679999351501465,
136
+ "tpp_threshold_20_intended_diff_only": 0.05219999551773071,
137
+ "tpp_threshold_20_unintended_diff_only": 0.0054000020027160645,
138
+ "tpp_threshold_50_total_metric": 0.10110000967979431,
139
+ "tpp_threshold_50_intended_diff_only": 0.10820000171661377,
140
+ "tpp_threshold_50_unintended_diff_only": 0.007099992036819458,
141
+ "tpp_threshold_100_total_metric": 0.18950001299381256,
142
+ "tpp_threshold_100_intended_diff_only": 0.20100001096725464,
143
+ "tpp_threshold_100_unintended_diff_only": 0.011499997973442078,
144
+ "tpp_threshold_500_total_metric": 0.3527000159025192,
145
+ "tpp_threshold_500_intended_diff_only": 0.37120001316070556,
146
+ "tpp_threshold_500_unintended_diff_only": 0.01849999725818634
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.0015000253915786743,
194
  "tpp_threshold_10_total_metric": 0.00424996018409729,
195
  "tpp_threshold_10_intended_diff_only": 0.006999969482421875,
196
+ "tpp_threshold_10_unintended_diff_only": 0.002750009298324585,
197
+ "tpp_threshold_20_total_metric": 0.03949996829032898,
198
+ "tpp_threshold_20_intended_diff_only": 0.041999995708465576,
199
+ "tpp_threshold_20_unintended_diff_only": 0.0025000274181365967,
200
+ "tpp_threshold_50_total_metric": 0.07549996674060822,
201
+ "tpp_threshold_50_intended_diff_only": 0.07999998331069946,
202
+ "tpp_threshold_50_unintended_diff_only": 0.0045000165700912476,
203
+ "tpp_threshold_100_total_metric": 0.24300000071525574,
204
+ "tpp_threshold_100_intended_diff_only": 0.24900001287460327,
205
+ "tpp_threshold_100_unintended_diff_only": 0.006000012159347534,
206
+ "tpp_threshold_500_total_metric": 0.4324999749660492,
207
+ "tpp_threshold_500_intended_diff_only": 0.4399999976158142,
208
+ "tpp_threshold_500_unintended_diff_only": 0.007500022649765015
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.004749968647956848,
212
+ "tpp_threshold_2_intended_diff_only": 0.0059999823570251465,
213
+ "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983,
214
+ "tpp_threshold_5_total_metric": -0.000250011682510376,
215
+ "tpp_threshold_5_intended_diff_only": 0.0,
216
+ "tpp_threshold_5_unintended_diff_only": 0.000250011682510376,
217
+ "tpp_threshold_10_total_metric": 0.011249944567680359,
218
+ "tpp_threshold_10_intended_diff_only": 0.010999977588653564,
219
+ "tpp_threshold_10_unintended_diff_only": -0.00024996697902679443,
220
+ "tpp_threshold_20_total_metric": 0.03499998152256012,
221
+ "tpp_threshold_20_intended_diff_only": 0.04100000858306885,
222
+ "tpp_threshold_20_unintended_diff_only": 0.006000027060508728,
223
+ "tpp_threshold_50_total_metric": 0.09724999964237213,
224
+ "tpp_threshold_50_intended_diff_only": 0.10600000619888306,
225
+ "tpp_threshold_50_unintended_diff_only": 0.008750006556510925,
226
+ "tpp_threshold_100_total_metric": 0.22550003230571747,
227
+ "tpp_threshold_100_intended_diff_only": 0.23400002717971802,
228
+ "tpp_threshold_100_unintended_diff_only": 0.00849999487400055,
229
+ "tpp_threshold_500_total_metric": 0.4415000081062317,
230
+ "tpp_threshold_500_intended_diff_only": 0.4570000171661377,
231
+ "tpp_threshold_500_unintended_diff_only": 0.015500009059906006
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.04249998927116394,
235
+ "tpp_threshold_2_intended_diff_only": 0.046000003814697266,
236
+ "tpp_threshold_2_unintended_diff_only": 0.003500014543533325,
237
+ "tpp_threshold_5_total_metric": 0.05324998497962952,
238
+ "tpp_threshold_5_intended_diff_only": 0.05500000715255737,
239
+ "tpp_threshold_5_unintended_diff_only": 0.0017500221729278564,
240
+ "tpp_threshold_10_total_metric": 0.0637500137090683,
241
+ "tpp_threshold_10_intended_diff_only": 0.06700003147125244,
242
+ "tpp_threshold_10_unintended_diff_only": 0.003250017762184143,
243
+ "tpp_threshold_20_total_metric": 0.10150006413459778,
244
+ "tpp_threshold_20_intended_diff_only": 0.10200005769729614,
245
+ "tpp_threshold_20_unintended_diff_only": 0.0004999935626983643,
246
+ "tpp_threshold_50_total_metric": 0.16050000488758087,
247
+ "tpp_threshold_50_intended_diff_only": 0.16100001335144043,
248
+ "tpp_threshold_50_unintended_diff_only": 0.0005000084638595581,
249
+ "tpp_threshold_100_total_metric": 0.2682500332593918,
250
+ "tpp_threshold_100_intended_diff_only": 0.27000004053115845,
251
+ "tpp_threshold_100_unintended_diff_only": 0.0017500072717666626,
252
+ "tpp_threshold_500_total_metric": 0.4240000396966934,
253
+ "tpp_threshold_500_intended_diff_only": 0.4320000410079956,
254
+ "tpp_threshold_500_unintended_diff_only": 0.008000001311302185
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.010249972343444824,
258
  "tpp_threshold_2_intended_diff_only": 0.009999990463256836,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0002499818801879883,
260
+ "tpp_threshold_5_total_metric": 0.009750008583068848,
261
+ "tpp_threshold_5_intended_diff_only": 0.00700002908706665,
262
+ "tpp_threshold_5_unintended_diff_only": -0.0027499794960021973,
263
+ "tpp_threshold_10_total_metric": 0.015750020742416382,
264
+ "tpp_threshold_10_intended_diff_only": 0.016000032424926758,
265
+ "tpp_threshold_10_unintended_diff_only": 0.000250011682510376,
266
+ "tpp_threshold_20_total_metric": 0.016750022768974304,
267
+ "tpp_threshold_20_intended_diff_only": 0.016000032424926758,
268
+ "tpp_threshold_20_unintended_diff_only": -0.0007499903440475464,
269
+ "tpp_threshold_50_total_metric": 0.022000014781951904,
270
+ "tpp_threshold_50_intended_diff_only": 0.021000027656555176,
271
+ "tpp_threshold_50_unintended_diff_only": -0.0009999871253967285,
272
+ "tpp_threshold_100_total_metric": 0.06149999797344208,
273
+ "tpp_threshold_100_intended_diff_only": 0.06400001049041748,
274
+ "tpp_threshold_100_unintended_diff_only": 0.002500012516975403,
275
+ "tpp_threshold_500_total_metric": 0.42900004982948303,
276
+ "tpp_threshold_500_intended_diff_only": 0.4310000538825989,
277
+ "tpp_threshold_500_unintended_diff_only": 0.0020000040531158447
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.003000006079673767,
281
+ "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224,
283
+ "tpp_threshold_5_total_metric": 0.009000018239021301,
284
+ "tpp_threshold_5_intended_diff_only": 0.01100003719329834,
285
+ "tpp_threshold_5_unintended_diff_only": 0.0020000189542770386,
286
+ "tpp_threshold_10_total_metric": 0.00849999487400055,
287
+ "tpp_threshold_10_intended_diff_only": 0.013000011444091797,
288
+ "tpp_threshold_10_unintended_diff_only": 0.0045000165700912476,
289
+ "tpp_threshold_20_total_metric": 0.04100002348423004,
290
+ "tpp_threshold_20_intended_diff_only": 0.04400002956390381,
291
+ "tpp_threshold_20_unintended_diff_only": 0.003000006079673767,
292
+ "tpp_threshold_50_total_metric": 0.13500003516674042,
293
+ "tpp_threshold_50_intended_diff_only": 0.14000004529953003,
294
+ "tpp_threshold_50_unintended_diff_only": 0.005000010132789612,
295
+ "tpp_threshold_100_total_metric": 0.3210000693798065,
296
+ "tpp_threshold_100_intended_diff_only": 0.3280000686645508,
297
+ "tpp_threshold_100_unintended_diff_only": 0.006999999284744263,
298
+ "tpp_threshold_500_total_metric": 0.465750053524971,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.010250017046928406
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.0037499666213989258,
306
+ "tpp_threshold_2_intended_diff_only": 0.006999969482421875,
307
+ "tpp_threshold_2_unintended_diff_only": 0.0032500028610229492,
308
+ "tpp_threshold_5_total_metric": 0.0032500028610229492,
309
+ "tpp_threshold_5_intended_diff_only": 0.004999995231628418,
310
+ "tpp_threshold_5_unintended_diff_only": 0.0017499923706054688,
311
+ "tpp_threshold_10_total_metric": 0.004749968647956848,
312
  "tpp_threshold_10_intended_diff_only": 0.006999969482421875,
313
+ "tpp_threshold_10_unintended_diff_only": 0.002250000834465027,
314
+ "tpp_threshold_20_total_metric": 0.01524999737739563,
315
+ "tpp_threshold_20_intended_diff_only": 0.018999993801116943,
316
+ "tpp_threshold_20_unintended_diff_only": 0.0037499964237213135,
317
+ "tpp_threshold_50_total_metric": 0.04849998652935028,
318
+ "tpp_threshold_50_intended_diff_only": 0.04799997806549072,
319
+ "tpp_threshold_50_unintended_diff_only": -0.0005000084638595581,
320
+ "tpp_threshold_100_total_metric": 0.10199999809265137,
321
+ "tpp_threshold_100_intended_diff_only": 0.10600000619888306,
322
+ "tpp_threshold_100_unintended_diff_only": 0.0040000081062316895,
323
+ "tpp_threshold_500_total_metric": 0.34375,
324
+ "tpp_threshold_500_intended_diff_only": 0.3479999899864197,
325
+ "tpp_threshold_500_unintended_diff_only": 0.004249989986419678
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.01775003969669342,
329
+ "tpp_threshold_2_intended_diff_only": 0.016000032424926758,
330
+ "tpp_threshold_2_unintended_diff_only": -0.0017500072717666626,
331
+ "tpp_threshold_5_total_metric": 0.014500036835670471,
332
+ "tpp_threshold_5_intended_diff_only": 0.022000014781951904,
333
+ "tpp_threshold_5_unintended_diff_only": 0.007499977946281433,
334
+ "tpp_threshold_10_total_metric": 0.05075004696846008,
335
+ "tpp_threshold_10_intended_diff_only": 0.058000028133392334,
336
+ "tpp_threshold_10_unintended_diff_only": 0.007249981164932251,
337
+ "tpp_threshold_20_total_metric": 0.07225000858306885,
338
+ "tpp_threshold_20_intended_diff_only": 0.07400000095367432,
339
+ "tpp_threshold_20_unintended_diff_only": 0.0017499923706054688,
340
+ "tpp_threshold_50_total_metric": 0.1297500729560852,
341
+ "tpp_threshold_50_intended_diff_only": 0.1390000581741333,
342
+ "tpp_threshold_50_unintended_diff_only": 0.009249985218048096,
343
+ "tpp_threshold_100_total_metric": 0.3250000476837158,
344
+ "tpp_threshold_100_intended_diff_only": 0.34400004148483276,
345
+ "tpp_threshold_100_unintended_diff_only": 0.018999993801116943,
346
+ "tpp_threshold_500_total_metric": 0.4045000523328781,
347
+ "tpp_threshold_500_intended_diff_only": 0.4360000491142273,
348
+ "tpp_threshold_500_unintended_diff_only": 0.03149999678134918
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.002750009298324585,
352
+ "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
353
+ "tpp_threshold_2_unintended_diff_only": -0.0017500221729278564,
354
+ "tpp_threshold_5_total_metric": 0.010500028729438782,
355
+ "tpp_threshold_5_intended_diff_only": 0.00700002908706665,
356
+ "tpp_threshold_5_unintended_diff_only": -0.0034999996423721313,
357
+ "tpp_threshold_10_total_metric": 0.0222499817609787,
358
+ "tpp_threshold_10_intended_diff_only": 0.019999980926513672,
359
+ "tpp_threshold_10_unintended_diff_only": -0.002250000834465027,
360
+ "tpp_threshold_20_total_metric": 0.01124998927116394,
361
+ "tpp_threshold_20_intended_diff_only": 0.013999998569488525,
362
+ "tpp_threshold_20_unintended_diff_only": 0.002750009298324585,
363
+ "tpp_threshold_50_total_metric": 0.05125004053115845,
364
+ "tpp_threshold_50_intended_diff_only": 0.053000032901763916,
365
+ "tpp_threshold_50_unintended_diff_only": 0.0017499923706054688,
366
+ "tpp_threshold_100_total_metric": 0.08875004947185516,
367
+ "tpp_threshold_100_intended_diff_only": 0.09500002861022949,
368
+ "tpp_threshold_100_unintended_diff_only": 0.006249979138374329,
369
+ "tpp_threshold_500_total_metric": 0.320000022649765,
370
+ "tpp_threshold_500_intended_diff_only": 0.3320000171661377,
371
+ "tpp_threshold_500_unintended_diff_only": 0.01199999451637268
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.0029999762773513794,
375
  "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
376
+ "tpp_threshold_2_unintended_diff_only": -0.0010000020265579224,
377
+ "tpp_threshold_5_total_metric": 0.0004999935626983643,
378
+ "tpp_threshold_5_intended_diff_only": 0.0,
379
+ "tpp_threshold_5_unintended_diff_only": -0.0004999935626983643,
380
+ "tpp_threshold_10_total_metric": -0.005000025033950806,
381
+ "tpp_threshold_10_intended_diff_only": 0.001999974250793457,
382
+ "tpp_threshold_10_unintended_diff_only": 0.006999999284744263,
383
+ "tpp_threshold_20_total_metric": 0.013999983668327332,
384
+ "tpp_threshold_20_intended_diff_only": 0.018999993801116943,
385
+ "tpp_threshold_20_unintended_diff_only": 0.005000010132789612,
386
+ "tpp_threshold_50_total_metric": 0.06074996292591095,
387
+ "tpp_threshold_50_intended_diff_only": 0.0679999589920044,
388
+ "tpp_threshold_50_unintended_diff_only": 0.007249996066093445,
389
+ "tpp_threshold_100_total_metric": 0.12399999797344208,
390
+ "tpp_threshold_100_intended_diff_only": 0.1340000033378601,
391
+ "tpp_threshold_100_unintended_diff_only": 0.01000000536441803,
392
+ "tpp_threshold_500_total_metric": 0.3507499694824219,
393
+ "tpp_threshold_500_intended_diff_only": 0.3669999837875366,
394
+ "tpp_threshold_500_unintended_diff_only": 0.016250014305114746
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.010000020265579224,
398
+ "tpp_threshold_2_intended_diff_only": 0.013000011444091797,
399
+ "tpp_threshold_2_unintended_diff_only": 0.0029999911785125732,
400
+ "tpp_threshold_5_total_metric": 0.036500006914138794,
401
+ "tpp_threshold_5_intended_diff_only": 0.03700000047683716,
402
+ "tpp_threshold_5_unintended_diff_only": 0.0004999935626983643,
403
+ "tpp_threshold_10_total_metric": 0.08049999177455902,
404
+ "tpp_threshold_10_intended_diff_only": 0.08300000429153442,
405
+ "tpp_threshold_10_unintended_diff_only": 0.002500012516975403,
406
+ "tpp_threshold_20_total_metric": 0.12124998867511749,
407
+ "tpp_threshold_20_intended_diff_only": 0.13499999046325684,
408
+ "tpp_threshold_20_unintended_diff_only": 0.013750001788139343,
409
+ "tpp_threshold_50_total_metric": 0.21524998545646667,
410
+ "tpp_threshold_50_intended_diff_only": 0.2329999804496765,
411
+ "tpp_threshold_50_unintended_diff_only": 0.01774999499320984,
412
+ "tpp_threshold_100_total_metric": 0.3077499717473984,
413
+ "tpp_threshold_100_intended_diff_only": 0.3259999752044678,
414
+ "tpp_threshold_100_unintended_diff_only": 0.018250003457069397,
415
+ "tpp_threshold_500_total_metric": 0.34450003504753113,
416
+ "tpp_threshold_500_intended_diff_only": 0.37300002574920654,
417
+ "tpp_threshold_500_unintended_diff_only": 0.028499990701675415
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-122070-tokens-500000000/scr/saebench_scr_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,8 +69,8 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "a8b883cd-cabb-41bb-875f-a774bc78f704",
69
- "datetime_epoch_millis": 1776883807856,
70
  "eval_result_metrics": {
71
  "scr_metrics": {
72
  "scr_dir1_threshold_2": 0.27700654085449283,
@@ -78,18 +82,18 @@
78
  "scr_dir1_threshold_10": 0.21640243118517796,
79
  "scr_metric_threshold_10": 0.23223524384144018,
80
  "scr_dir2_threshold_10": 0.22135975708935063,
81
- "scr_dir1_threshold_20": null,
82
- "scr_metric_threshold_20": null,
83
- "scr_dir2_threshold_20": null,
84
- "scr_dir1_threshold_50": null,
85
- "scr_metric_threshold_50": null,
86
- "scr_dir2_threshold_50": null,
87
- "scr_dir1_threshold_100": null,
88
- "scr_metric_threshold_100": null,
89
- "scr_dir2_threshold_100": null,
90
- "scr_dir1_threshold_500": null,
91
- "scr_metric_threshold_500": null,
92
- "scr_dir2_threshold_500": null
93
  }
94
  },
95
  "eval_result_details": [
@@ -104,18 +108,18 @@
104
  "scr_dir1_threshold_10": 0.4102564298503083,
105
  "scr_metric_threshold_10": 0.06994820573050735,
106
  "scr_dir2_threshold_10": 0.06994820573050735,
107
- "scr_dir1_threshold_20": null,
108
- "scr_metric_threshold_20": null,
109
- "scr_dir2_threshold_20": null,
110
- "scr_dir1_threshold_50": null,
111
- "scr_metric_threshold_50": null,
112
- "scr_dir2_threshold_50": null,
113
- "scr_dir1_threshold_100": null,
114
- "scr_metric_threshold_100": null,
115
- "scr_dir2_threshold_100": null,
116
- "scr_dir1_threshold_500": null,
117
- "scr_metric_threshold_500": null,
118
- "scr_dir2_threshold_500": null
119
  },
120
  {
121
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
@@ -128,18 +132,18 @@
128
  "scr_dir1_threshold_10": 0.31060605718522544,
129
  "scr_metric_threshold_10": 0.3241590186196148,
130
  "scr_dir2_threshold_10": 0.3241590186196148,
131
- "scr_dir1_threshold_20": null,
132
- "scr_metric_threshold_20": null,
133
- "scr_dir2_threshold_20": null,
134
- "scr_dir1_threshold_50": null,
135
- "scr_metric_threshold_50": null,
136
- "scr_dir2_threshold_50": null,
137
- "scr_dir1_threshold_100": null,
138
- "scr_metric_threshold_100": null,
139
- "scr_dir2_threshold_100": null,
140
- "scr_dir1_threshold_500": null,
141
- "scr_metric_threshold_500": null,
142
- "scr_dir2_threshold_500": null
143
  },
144
  {
145
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
@@ -152,18 +156,18 @@
152
  "scr_dir1_threshold_10": 0.17187462165026607,
153
  "scr_metric_threshold_10": 0.10804029093651613,
154
  "scr_dir2_threshold_10": 0.10804029093651613,
155
- "scr_dir1_threshold_20": null,
156
- "scr_metric_threshold_20": null,
157
- "scr_dir2_threshold_20": null,
158
- "scr_dir1_threshold_50": null,
159
- "scr_metric_threshold_50": null,
160
- "scr_dir2_threshold_50": null,
161
- "scr_dir1_threshold_100": null,
162
- "scr_metric_threshold_100": null,
163
- "scr_dir2_threshold_100": null,
164
- "scr_dir1_threshold_500": null,
165
- "scr_metric_threshold_500": null,
166
- "scr_dir2_threshold_500": null
167
  },
168
  {
169
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
@@ -176,18 +180,18 @@
176
  "scr_dir1_threshold_10": -0.2416108456459604,
177
  "scr_metric_threshold_10": 0.14649682254103324,
178
  "scr_dir2_threshold_10": 0.14649682254103324,
179
- "scr_dir1_threshold_20": null,
180
- "scr_metric_threshold_20": null,
181
- "scr_dir2_threshold_20": null,
182
- "scr_dir1_threshold_50": null,
183
- "scr_metric_threshold_50": null,
184
- "scr_dir2_threshold_50": null,
185
- "scr_dir1_threshold_100": null,
186
- "scr_metric_threshold_100": null,
187
- "scr_dir2_threshold_100": null,
188
- "scr_dir1_threshold_500": null,
189
- "scr_metric_threshold_500": null,
190
- "scr_dir2_threshold_500": null
191
  },
192
  {
193
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
@@ -200,18 +204,18 @@
200
  "scr_dir1_threshold_10": 0.08421069803000372,
201
  "scr_metric_threshold_10": 0.4999998817368318,
202
  "scr_dir2_threshold_10": 0.4999998817368318,
203
- "scr_dir1_threshold_20": null,
204
- "scr_metric_threshold_20": null,
205
- "scr_dir2_threshold_20": null,
206
- "scr_dir1_threshold_50": null,
207
- "scr_metric_threshold_50": null,
208
- "scr_dir2_threshold_50": null,
209
- "scr_dir1_threshold_100": null,
210
- "scr_metric_threshold_100": null,
211
- "scr_dir2_threshold_100": null,
212
- "scr_dir1_threshold_500": null,
213
- "scr_metric_threshold_500": null,
214
- "scr_dir2_threshold_500": null
215
  },
216
  {
217
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
@@ -224,18 +228,18 @@
224
  "scr_dir1_threshold_10": 0.4098360709132541,
225
  "scr_metric_threshold_10": 0.1183206688641517,
226
  "scr_dir2_threshold_10": 0.1183206688641517,
227
- "scr_dir1_threshold_20": null,
228
- "scr_metric_threshold_20": null,
229
- "scr_dir2_threshold_20": null,
230
- "scr_dir1_threshold_50": null,
231
- "scr_metric_threshold_50": null,
232
- "scr_dir2_threshold_50": null,
233
- "scr_dir1_threshold_100": null,
234
- "scr_metric_threshold_100": null,
235
- "scr_dir2_threshold_100": null,
236
- "scr_dir1_threshold_500": null,
237
- "scr_metric_threshold_500": null,
238
- "scr_dir2_threshold_500": null
239
  },
240
  {
241
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
@@ -248,18 +252,18 @@
248
  "scr_dir1_threshold_10": 0.3813953694683433,
249
  "scr_metric_threshold_10": 0.3862660142728834,
250
  "scr_dir2_threshold_10": 0.3862660142728834,
251
- "scr_dir1_threshold_20": null,
252
- "scr_metric_threshold_20": null,
253
- "scr_dir2_threshold_20": null,
254
- "scr_dir1_threshold_50": null,
255
- "scr_metric_threshold_50": null,
256
- "scr_dir2_threshold_50": null,
257
- "scr_dir1_threshold_100": null,
258
- "scr_metric_threshold_100": null,
259
- "scr_dir2_threshold_100": null,
260
- "scr_dir1_threshold_500": null,
261
- "scr_metric_threshold_500": null,
262
- "scr_dir2_threshold_500": null
263
  },
264
  {
265
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
@@ -272,18 +276,18 @@
272
  "scr_dir1_threshold_10": 0.20465104802998307,
273
  "scr_metric_threshold_10": 0.20465104802998307,
274
  "scr_dir2_threshold_10": 0.11764715401326675,
275
- "scr_dir1_threshold_20": null,
276
- "scr_metric_threshold_20": null,
277
- "scr_dir2_threshold_20": null,
278
- "scr_dir1_threshold_50": null,
279
- "scr_metric_threshold_50": null,
280
- "scr_dir2_threshold_50": null,
281
- "scr_dir1_threshold_100": null,
282
- "scr_metric_threshold_100": null,
283
- "scr_dir2_threshold_100": null,
284
- "scr_dir1_threshold_500": null,
285
- "scr_metric_threshold_500": null,
286
- "scr_dir2_threshold_500": null
287
  }
288
  ],
289
  "sae_bench_commit_hash": "Unknown",
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "007d2088-7969-42f0-a21c-11e05d637da7",
73
+ "datetime_epoch_millis": 1777315503017,
74
  "eval_result_metrics": {
75
  "scr_metrics": {
76
  "scr_dir1_threshold_2": 0.27700654085449283,
 
82
  "scr_dir1_threshold_10": 0.21640243118517796,
83
  "scr_metric_threshold_10": 0.23223524384144018,
84
  "scr_dir2_threshold_10": 0.22135975708935063,
85
+ "scr_dir1_threshold_20": 0.04097264054927978,
86
+ "scr_metric_threshold_20": 0.29427585256810657,
87
+ "scr_dir2_threshold_20": 0.2820639250435518,
88
+ "scr_dir1_threshold_50": 0.25708903377113695,
89
+ "scr_metric_threshold_50": 0.3417567551129011,
90
+ "scr_dir2_threshold_50": 0.32764278327844015,
91
+ "scr_dir1_threshold_100": -0.03167553686039668,
92
+ "scr_metric_threshold_100": 0.3373470777000024,
93
+ "scr_dir2_threshold_100": 0.3179690100659854,
94
+ "scr_dir1_threshold_500": -0.9412289882116809,
95
+ "scr_metric_threshold_500": 0.19818225611848905,
96
+ "scr_dir2_threshold_500": 0.18342375782034986
97
  }
98
  },
99
  "eval_result_details": [
 
108
  "scr_dir1_threshold_10": 0.4102564298503083,
109
  "scr_metric_threshold_10": 0.06994820573050735,
110
  "scr_dir2_threshold_10": 0.06994820573050735,
111
+ "scr_dir1_threshold_20": 0.3589742805987668,
112
+ "scr_metric_threshold_20": 0.13730561707323927,
113
+ "scr_dir2_threshold_20": 0.13730561707323927,
114
+ "scr_dir1_threshold_50": 0.5256410746257707,
115
+ "scr_metric_threshold_50": 0.27461138856264183,
116
+ "scr_dir2_threshold_50": 0.27461138856264183,
117
+ "scr_dir1_threshold_100": 0.4487182328294704,
118
+ "scr_metric_threshold_100": 0.27461138856264183,
119
+ "scr_dir2_threshold_100": 0.27461138856264183,
120
+ "scr_dir1_threshold_500": -2.2820506746404425,
121
+ "scr_metric_threshold_500": 0.08808283994795557,
122
+ "scr_dir2_threshold_500": 0.08808283994795557
123
  },
124
  {
125
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
 
132
  "scr_dir1_threshold_10": 0.31060605718522544,
133
  "scr_metric_threshold_10": 0.3241590186196148,
134
  "scr_dir2_threshold_10": 0.3241590186196148,
135
+ "scr_dir1_threshold_20": -0.08333314518740012,
136
+ "scr_metric_threshold_20": 0.4311927307857058,
137
+ "scr_dir2_threshold_20": 0.4311927307857058,
138
+ "scr_dir1_threshold_50": 0.3181817155567637,
139
+ "scr_metric_threshold_50": 0.5657492048157633,
140
+ "scr_dir2_threshold_50": 0.5657492048157633,
141
+ "scr_dir1_threshold_100": -0.1287875469668693,
142
+ "scr_metric_threshold_100": 0.6513761016377605,
143
+ "scr_dir2_threshold_100": 0.6513761016377605,
144
+ "scr_dir1_threshold_500": -1.272726862227055,
145
+ "scr_metric_threshold_500": 0.321100954221084,
146
+ "scr_dir2_threshold_500": 0.321100954221084
147
  },
148
  {
149
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
 
156
  "scr_dir1_threshold_10": 0.17187462165026607,
157
  "scr_metric_threshold_10": 0.10804029093651613,
158
  "scr_dir2_threshold_10": 0.10804029093651613,
159
+ "scr_dir1_threshold_20": -0.42187508731147705,
160
+ "scr_metric_threshold_20": 0.17336682175353543,
161
+ "scr_dir2_threshold_20": 0.17336682175353543,
162
+ "scr_dir1_threshold_50": 0.3906245634426147,
163
+ "scr_metric_threshold_50": 0.22864323226054475,
164
+ "scr_dir2_threshold_50": 0.22864323226054475,
165
+ "scr_dir1_threshold_100": -0.6874994179234862,
166
+ "scr_metric_threshold_100": 0.06532668057741017,
167
+ "scr_dir2_threshold_100": 0.06532668057741017,
168
+ "scr_dir1_threshold_500": -3.484374272404358,
169
+ "scr_metric_threshold_500": -0.027638130373309224,
170
+ "scr_dir2_threshold_500": -0.027638130373309224
171
  },
172
  {
173
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
 
180
  "scr_dir1_threshold_10": -0.2416108456459604,
181
  "scr_metric_threshold_10": 0.14649682254103324,
182
  "scr_dir2_threshold_10": 0.14649682254103324,
183
+ "scr_dir1_threshold_20": -0.7248325369378812,
184
+ "scr_metric_threshold_20": 0.16560504596960618,
185
+ "scr_dir2_threshold_20": 0.16560504596960618,
186
+ "scr_dir1_threshold_50": -0.36912759195217226,
187
+ "scr_metric_threshold_50": 0.06050953237690028,
188
+ "scr_dir2_threshold_50": 0.06050953237690028,
189
+ "scr_dir1_threshold_100": -1.0872488054068217,
190
+ "scr_metric_threshold_100": 0.00955420662613804,
191
+ "scr_dir2_threshold_100": 0.00955420662613804,
192
+ "scr_dir1_threshold_500": -1.5234904322537157,
193
+ "scr_metric_threshold_500": -0.5031846722674783,
194
+ "scr_dir2_threshold_500": -0.5031846722674783
195
  },
196
  {
197
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
 
204
  "scr_dir1_threshold_10": 0.08421069803000372,
205
  "scr_metric_threshold_10": 0.4999998817368318,
206
  "scr_dir2_threshold_10": 0.4999998817368318,
207
+ "scr_dir1_threshold_20": 0.11052646273721473,
208
+ "scr_metric_threshold_20": 0.5515872218065929,
209
+ "scr_dir2_threshold_20": 0.5515872218065929,
210
+ "scr_dir1_threshold_50": -0.11052614902855408,
211
+ "scr_metric_threshold_50": 0.6507936996006726,
212
+ "scr_dir2_threshold_50": 0.6507936996006726,
213
+ "scr_dir1_threshold_100": -0.22631570196547887,
214
+ "scr_metric_threshold_100": 0.7063492420158761,
215
+ "scr_dir2_threshold_100": 0.7063492420158761,
216
+ "scr_dir1_threshold_500": -0.20526302745797795,
217
+ "scr_metric_threshold_500": 0.6666667455087788,
218
+ "scr_dir2_threshold_500": 0.6666667455087788
219
  },
220
  {
221
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
 
228
  "scr_dir1_threshold_10": 0.4098360709132541,
229
  "scr_metric_threshold_10": 0.1183206688641517,
230
  "scr_dir2_threshold_10": 0.1183206688641517,
231
+ "scr_dir1_threshold_20": 0.4371584539269045,
232
+ "scr_metric_threshold_20": 0.17938926174649328,
233
+ "scr_dir2_threshold_20": 0.17938926174649328,
234
+ "scr_dir1_threshold_50": 0.497267631415236,
235
+ "scr_metric_threshold_50": 0.26335877602100816,
236
+ "scr_dir2_threshold_50": 0.26335877602100816,
237
+ "scr_dir1_threshold_100": 0.5300547515984144,
238
+ "scr_metric_threshold_100": 0.2671755204201627,
239
+ "scr_dir2_threshold_100": 0.2671755204201627,
240
+ "scr_dir1_threshold_500": 0.5683059575341262,
241
+ "scr_metric_threshold_500": 0.2709922648193172,
242
+ "scr_dir2_threshold_500": 0.2709922648193172
243
  },
244
  {
245
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
 
252
  "scr_dir1_threshold_10": 0.3813953694683433,
253
  "scr_metric_threshold_10": 0.3862660142728834,
254
  "scr_dir2_threshold_10": 0.3862660142728834,
255
+ "scr_dir1_threshold_20": 0.3860464716550849,
256
+ "scr_metric_threshold_20": 0.4506438964966536,
257
+ "scr_dir2_threshold_20": 0.4506438964966536,
258
+ "scr_dir1_threshold_50": 0.47906962431336847,
259
+ "scr_metric_threshold_50": 0.3648068054696092,
260
+ "scr_dir2_threshold_50": 0.3648068054696092,
261
+ "scr_dir1_threshold_100": 0.5209300984557687,
262
+ "scr_metric_threshold_100": 0.3476393872642003,
263
+ "scr_dir2_threshold_100": 0.3476393872642003,
264
+ "scr_dir1_threshold_500": 0.3209302093481138,
265
+ "scr_metric_threshold_500": 0.4206008506837011,
266
+ "scr_dir2_threshold_500": 0.4206008506837011
267
  },
268
  {
269
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
 
276
  "scr_dir1_threshold_10": 0.20465104802998307,
277
  "scr_metric_threshold_10": 0.20465104802998307,
278
  "scr_dir2_threshold_10": 0.11764715401326675,
279
+ "scr_dir1_threshold_20": 0.2651162249130257,
280
+ "scr_metric_threshold_20": 0.2651162249130257,
281
+ "scr_dir2_threshold_20": 0.16742080471658802,
282
+ "scr_dir1_threshold_50": 0.32558140179606837,
283
+ "scr_metric_threshold_50": 0.32558140179606837,
284
+ "scr_dir2_threshold_50": 0.21266962712038115,
285
+ "scr_dir1_threshold_100": 0.3767440944958292,
286
+ "scr_metric_threshold_100": 0.3767440944958292,
287
+ "scr_dir2_threshold_100": 0.22171955342369326,
288
+ "scr_dir1_threshold_500": 0.34883719640786326,
289
+ "scr_metric_threshold_500": 0.34883719640786326,
290
+ "scr_dir2_threshold_500": 0.23076921002274955
291
  }
292
  ],
293
  "sae_bench_commit_hash": "Unknown",
gemma-2-2b-btk/k-100/seed-0/snapshots/step-122070-tokens-500000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "949d3d21-e9f7-4054-8410-d6325ea1d9d4",
69
- "datetime_epoch_millis": 1776883959512,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.022575005888938904,
73
- "tpp_threshold_2_intended_diff_only": 0.02390000820159912,
74
- "tpp_threshold_2_unintended_diff_only": 0.0013250023126602174,
75
- "tpp_threshold_5_total_metric": 0.055699999630451205,
76
- "tpp_threshold_5_intended_diff_only": 0.07050000429153443,
77
- "tpp_threshold_5_unintended_diff_only": 0.014800004661083221,
78
- "tpp_threshold_10_total_metric": 0.10322501361370087,
79
- "tpp_threshold_10_intended_diff_only": 0.13190001845359803,
80
- "tpp_threshold_10_unintended_diff_only": 0.028675004839897156,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.034050014615058896,
99
- "tpp_threshold_2_intended_diff_only": 0.036400020122528076,
100
- "tpp_threshold_2_unintended_diff_only": 0.0023500055074691774,
101
- "tpp_threshold_5_total_metric": 0.0950499951839447,
102
  "tpp_threshold_5_intended_diff_only": 0.12220001220703125,
103
- "tpp_threshold_5_unintended_diff_only": 0.02715001702308655,
104
- "tpp_threshold_10_total_metric": 0.17530001401901246,
105
  "tpp_threshold_10_intended_diff_only": 0.23020002841949463,
106
- "tpp_threshold_10_unintended_diff_only": 0.05490001440048218,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.011099997162818908,
123
- "tpp_threshold_2_intended_diff_only": 0.011399996280670167,
124
- "tpp_threshold_2_unintended_diff_only": 0.0002999991178512573,
125
- "tpp_threshold_5_total_metric": 0.016350004076957702,
126
- "tpp_threshold_5_intended_diff_only": 0.018799996376037596,
127
- "tpp_threshold_5_unintended_diff_only": 0.002449992299079895,
128
- "tpp_threshold_10_total_metric": 0.031150013208389282,
129
- "tpp_threshold_10_intended_diff_only": 0.033600008487701415,
130
- "tpp_threshold_10_unintended_diff_only": 0.0024499952793121337,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.0020000189542770386,
190
  "tpp_threshold_10_total_metric": 0.006499990820884705,
191
  "tpp_threshold_10_intended_diff_only": 0.009000003337860107,
192
- "tpp_threshold_10_unintended_diff_only": 0.002500012516975403
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": 0.0017500072717666626,
196
- "tpp_threshold_2_intended_diff_only": 0.003000020980834961,
197
- "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983,
198
- "tpp_threshold_5_total_metric": 0.15999996662139893,
199
  "tpp_threshold_5_intended_diff_only": 0.18599998950958252,
200
- "tpp_threshold_5_unintended_diff_only": 0.026000022888183594,
201
- "tpp_threshold_10_total_metric": 0.20624999701976776,
202
  "tpp_threshold_10_intended_diff_only": 0.2670000195503235,
203
- "tpp_threshold_10_unintended_diff_only": 0.060750022530555725
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.1430000364780426,
207
- "tpp_threshold_2_intended_diff_only": 0.15400004386901855,
208
- "tpp_threshold_2_unintended_diff_only": 0.011000007390975952,
209
- "tpp_threshold_5_total_metric": 0.20775003731250763,
210
- "tpp_threshold_5_intended_diff_only": 0.2930000424385071,
211
- "tpp_threshold_5_unintended_diff_only": 0.08525000512599945,
212
- "tpp_threshold_10_total_metric": 0.21100002527236938,
213
- "tpp_threshold_10_intended_diff_only": 0.3160000443458557,
214
- "tpp_threshold_10_unintended_diff_only": 0.10500001907348633
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.011750012636184692,
218
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
219
- "tpp_threshold_2_unintended_diff_only": -0.002750009298324585,
220
- "tpp_threshold_5_total_metric": 0.078249990940094,
221
- "tpp_threshold_5_intended_diff_only": 0.09200000762939453,
222
- "tpp_threshold_5_unintended_diff_only": 0.013750016689300537,
223
- "tpp_threshold_10_total_metric": 0.27650004625320435,
224
  "tpp_threshold_10_intended_diff_only": 0.3290000557899475,
225
- "tpp_threshold_10_unintended_diff_only": 0.052500009536743164
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.0050000399351119995,
229
- "tpp_threshold_2_intended_diff_only": 0.006000041961669922,
230
- "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224,
231
- "tpp_threshold_5_total_metric": 0.016250014305114746,
232
  "tpp_threshold_5_intended_diff_only": 0.025000035762786865,
233
- "tpp_threshold_5_unintended_diff_only": 0.00875002145767212,
234
- "tpp_threshold_10_total_metric": 0.17625001072883606,
235
- "tpp_threshold_10_intended_diff_only": 0.23000001907348633,
236
- "tpp_threshold_10_unintended_diff_only": 0.05375000834465027
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.008750006556510925,
242
- "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
243
- "tpp_threshold_2_unintended_diff_only": -0.004749998450279236,
244
- "tpp_threshold_5_total_metric": 0.01374998688697815,
245
- "tpp_threshold_5_intended_diff_only": 0.010999977588653564,
246
- "tpp_threshold_5_unintended_diff_only": -0.002750009298324585,
247
- "tpp_threshold_10_total_metric": 0.003500014543533325,
248
- "tpp_threshold_10_intended_diff_only": 0.008000016212463379,
249
- "tpp_threshold_10_unintended_diff_only": 0.004500001668930054
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": 0.00575004518032074,
253
- "tpp_threshold_2_intended_diff_only": 0.01100003719329834,
254
- "tpp_threshold_2_unintended_diff_only": 0.0052499920129776,
255
- "tpp_threshold_5_total_metric": 0.004750043153762817,
256
- "tpp_threshold_5_intended_diff_only": 0.012000024318695068,
257
- "tpp_threshold_5_unintended_diff_only": 0.007249981164932251,
258
- "tpp_threshold_10_total_metric": 0.044000014662742615,
259
- "tpp_threshold_10_intended_diff_only": 0.046000003814697266,
260
- "tpp_threshold_10_unintended_diff_only": 0.001999989151954651
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.011749997735023499,
264
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
265
- "tpp_threshold_2_unintended_diff_only": -0.006750002503395081,
266
- "tpp_threshold_5_total_metric": 0.014750003814697266,
267
- "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
268
- "tpp_threshold_5_unintended_diff_only": -0.0007500052452087402,
269
- "tpp_threshold_10_total_metric": 0.015000015497207642,
270
- "tpp_threshold_10_intended_diff_only": 0.018000006675720215,
271
- "tpp_threshold_10_unintended_diff_only": 0.0029999911785125732
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.005750030279159546,
275
- "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
276
- "tpp_threshold_2_unintended_diff_only": 0.007750004529953003,
277
- "tpp_threshold_5_total_metric": 0.003999978303909302,
278
- "tpp_threshold_5_intended_diff_only": 0.0059999823570251465,
279
- "tpp_threshold_5_unintended_diff_only": 0.0020000040531158447,
280
- "tpp_threshold_10_total_metric": 0.011500000953674316,
281
- "tpp_threshold_10_intended_diff_only": 0.013999998569488525,
282
- "tpp_threshold_10_unintended_diff_only": 0.002499997615814209
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.034999966621398926,
286
- "tpp_threshold_2_intended_diff_only": 0.034999966621398926,
287
- "tpp_threshold_2_unintended_diff_only": 0.0,
288
- "tpp_threshold_5_total_metric": 0.04450000822544098,
289
- "tpp_threshold_5_intended_diff_only": 0.050999999046325684,
290
- "tpp_threshold_5_unintended_diff_only": 0.006499990820884705,
291
- "tpp_threshold_10_total_metric": 0.08175002038478851,
292
- "tpp_threshold_10_intended_diff_only": 0.0820000171661377,
293
- "tpp_threshold_10_unintended_diff_only": 0.00024999678134918213
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "a9e83ab3-97ad-4e7b-9a39-78b6523f812c",
73
+ "datetime_epoch_millis": 1777315777509,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.02144999951124191,
77
+ "tpp_threshold_2_intended_diff_only": 0.024100005626678467,
78
+ "tpp_threshold_2_unintended_diff_only": 0.002650006115436554,
79
+ "tpp_threshold_5_total_metric": 0.05662499815225601,
80
+ "tpp_threshold_5_intended_diff_only": 0.06930000185966492,
81
+ "tpp_threshold_5_unintended_diff_only": 0.012675003707408905,
82
+ "tpp_threshold_10_total_metric": 0.10545000731945038,
83
+ "tpp_threshold_10_intended_diff_only": 0.13310000896453858,
84
+ "tpp_threshold_10_unintended_diff_only": 0.027650001645088195,
85
+ "tpp_threshold_20_total_metric": 0.15019999593496322,
86
+ "tpp_threshold_20_intended_diff_only": 0.19020000100135803,
87
+ "tpp_threshold_20_unintended_diff_only": 0.04000000506639481,
88
+ "tpp_threshold_50_total_metric": 0.18925000429153443,
89
+ "tpp_threshold_50_intended_diff_only": 0.26040000915527345,
90
+ "tpp_threshold_50_unintended_diff_only": 0.07115000486373901,
91
+ "tpp_threshold_100_total_metric": 0.2069500148296356,
92
+ "tpp_threshold_100_intended_diff_only": 0.32150002717971804,
93
+ "tpp_threshold_100_unintended_diff_only": 0.11455001235008239,
94
+ "tpp_threshold_500_total_metric": 0.2484000191092491,
95
+ "tpp_threshold_500_intended_diff_only": 0.43350002765655515,
96
+ "tpp_threshold_500_unintended_diff_only": 0.18510000854730607
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.033350002765655515,
103
+ "tpp_threshold_2_intended_diff_only": 0.037600016593933104,
104
+ "tpp_threshold_2_unintended_diff_only": 0.0042500138282775875,
105
+ "tpp_threshold_5_total_metric": 0.09699999988079071,
106
  "tpp_threshold_5_intended_diff_only": 0.12220001220703125,
107
+ "tpp_threshold_5_unintended_diff_only": 0.02520001232624054,
108
+ "tpp_threshold_10_total_metric": 0.1768500179052353,
109
  "tpp_threshold_10_intended_diff_only": 0.23020002841949463,
110
+ "tpp_threshold_10_unintended_diff_only": 0.05335001051425934,
111
+ "tpp_threshold_20_total_metric": 0.2408499985933304,
112
+ "tpp_threshold_20_intended_diff_only": 0.31660001277923583,
113
+ "tpp_threshold_20_unintended_diff_only": 0.07575001418590546,
114
+ "tpp_threshold_50_total_metric": 0.2548500090837479,
115
+ "tpp_threshold_50_intended_diff_only": 0.3928000211715698,
116
+ "tpp_threshold_50_unintended_diff_only": 0.13795001208782195,
117
+ "tpp_threshold_100_total_metric": 0.2173000156879425,
118
+ "tpp_threshold_100_intended_diff_only": 0.43680003881454466,
119
+ "tpp_threshold_100_unintended_diff_only": 0.21950002312660216,
120
+ "tpp_threshold_500_total_metric": 0.12845001816749574,
121
+ "tpp_threshold_500_intended_diff_only": 0.4674000382423401,
122
+ "tpp_threshold_500_unintended_diff_only": 0.33895002007484437
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.009549996256828308,
127
+ "tpp_threshold_2_intended_diff_only": 0.010599994659423828,
128
+ "tpp_threshold_2_unintended_diff_only": 0.00104999840259552,
129
+ "tpp_threshold_5_total_metric": 0.016249996423721314,
130
+ "tpp_threshold_5_intended_diff_only": 0.016399991512298585,
131
+ "tpp_threshold_5_unintended_diff_only": 0.0001499950885772705,
132
+ "tpp_threshold_10_total_metric": 0.03404999673366547,
133
+ "tpp_threshold_10_intended_diff_only": 0.03599998950958252,
134
+ "tpp_threshold_10_unintended_diff_only": 0.0019499927759170533,
135
+ "tpp_threshold_20_total_metric": 0.05954999327659607,
136
+ "tpp_threshold_20_intended_diff_only": 0.06379998922348022,
137
+ "tpp_threshold_20_unintended_diff_only": 0.004249995946884156,
138
+ "tpp_threshold_50_total_metric": 0.12364999949932098,
139
+ "tpp_threshold_50_intended_diff_only": 0.12799999713897706,
140
+ "tpp_threshold_50_unintended_diff_only": 0.004349997639656067,
141
+ "tpp_threshold_100_total_metric": 0.19660001397132873,
142
+ "tpp_threshold_100_intended_diff_only": 0.20620001554489137,
143
+ "tpp_threshold_100_unintended_diff_only": 0.009600001573562621,
144
+ "tpp_threshold_500_total_metric": 0.3683500200510025,
145
+ "tpp_threshold_500_intended_diff_only": 0.39960001707077025,
146
+ "tpp_threshold_500_unintended_diff_only": 0.031249997019767762
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.0020000189542770386,
194
  "tpp_threshold_10_total_metric": 0.006499990820884705,
195
  "tpp_threshold_10_intended_diff_only": 0.009000003337860107,
196
+ "tpp_threshold_10_unintended_diff_only": 0.002500012516975403,
197
+ "tpp_threshold_20_total_metric": 0.2032499611377716,
198
+ "tpp_threshold_20_intended_diff_only": 0.23799997568130493,
199
+ "tpp_threshold_20_unintended_diff_only": 0.034750014543533325,
200
+ "tpp_threshold_50_total_metric": 0.2617499679327011,
201
+ "tpp_threshold_50_intended_diff_only": 0.35199999809265137,
202
+ "tpp_threshold_50_unintended_diff_only": 0.09025003015995026,
203
+ "tpp_threshold_100_total_metric": 0.24049995839595795,
204
+ "tpp_threshold_100_intended_diff_only": 0.4269999861717224,
205
+ "tpp_threshold_100_unintended_diff_only": 0.18650002777576447,
206
+ "tpp_threshold_500_total_metric": 0.1407500058412552,
207
+ "tpp_threshold_500_intended_diff_only": 0.45600003004074097,
208
+ "tpp_threshold_500_unintended_diff_only": 0.3152500241994858
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.0065000057220458984,
212
+ "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
213
+ "tpp_threshold_2_unintended_diff_only": 0.000500023365020752,
214
+ "tpp_threshold_5_total_metric": 0.16374997794628143,
215
  "tpp_threshold_5_intended_diff_only": 0.18599998950958252,
216
+ "tpp_threshold_5_unintended_diff_only": 0.022250011563301086,
217
+ "tpp_threshold_10_total_metric": 0.20775000751018524,
218
  "tpp_threshold_10_intended_diff_only": 0.2670000195503235,
219
+ "tpp_threshold_10_unintended_diff_only": 0.059250012040138245,
220
+ "tpp_threshold_20_total_metric": 0.23549999296665192,
221
+ "tpp_threshold_20_intended_diff_only": 0.30000001192092896,
222
+ "tpp_threshold_20_unintended_diff_only": 0.06450001895427704,
223
+ "tpp_threshold_50_total_metric": 0.24700000882148743,
224
+ "tpp_threshold_50_intended_diff_only": 0.3700000047683716,
225
+ "tpp_threshold_50_unintended_diff_only": 0.12299999594688416,
226
+ "tpp_threshold_100_total_metric": 0.2277500182390213,
227
+ "tpp_threshold_100_intended_diff_only": 0.42200005054473877,
228
+ "tpp_threshold_100_unintended_diff_only": 0.19425003230571747,
229
+ "tpp_threshold_500_total_metric": 0.11350002884864807,
230
+ "tpp_threshold_500_intended_diff_only": 0.4590000510215759,
231
+ "tpp_threshold_500_unintended_diff_only": 0.34550002217292786
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.1337500363588333,
235
+ "tpp_threshold_2_intended_diff_only": 0.15300005674362183,
236
+ "tpp_threshold_2_unintended_diff_only": 0.019250020384788513,
237
+ "tpp_threshold_5_total_metric": 0.21225003898143768,
238
+ "tpp_threshold_5_intended_diff_only": 0.29200005531311035,
239
+ "tpp_threshold_5_unintended_diff_only": 0.07975001633167267,
240
+ "tpp_threshold_10_total_metric": 0.21400001645088196,
241
+ "tpp_threshold_10_intended_diff_only": 0.31700003147125244,
242
+ "tpp_threshold_10_unintended_diff_only": 0.10300001502037048,
243
+ "tpp_threshold_20_total_metric": 0.1940000057220459,
244
+ "tpp_threshold_20_intended_diff_only": 0.35500001907348633,
245
+ "tpp_threshold_20_unintended_diff_only": 0.16100001335144043,
246
+ "tpp_threshold_50_total_metric": 0.16449999809265137,
247
+ "tpp_threshold_50_intended_diff_only": 0.37400001287460327,
248
+ "tpp_threshold_50_unintended_diff_only": 0.2095000147819519,
249
+ "tpp_threshold_100_total_metric": 0.11375002562999725,
250
+ "tpp_threshold_100_intended_diff_only": 0.40400004386901855,
251
+ "tpp_threshold_100_unintended_diff_only": 0.2902500182390213,
252
+ "tpp_threshold_500_total_metric": 0.06400001049041748,
253
+ "tpp_threshold_500_intended_diff_only": 0.4520000219345093,
254
+ "tpp_threshold_500_unintended_diff_only": 0.3880000114440918
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.009499981999397278,
258
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704,
260
+ "tpp_threshold_5_total_metric": 0.08399999141693115,
261
+ "tpp_threshold_5_intended_diff_only": 0.09299999475479126,
262
+ "tpp_threshold_5_unintended_diff_only": 0.009000003337860107,
263
+ "tpp_threshold_10_total_metric": 0.2797500491142273,
264
  "tpp_threshold_10_intended_diff_only": 0.3290000557899475,
265
+ "tpp_threshold_10_unintended_diff_only": 0.049250006675720215,
266
+ "tpp_threshold_20_total_metric": 0.33900001645088196,
267
+ "tpp_threshold_20_intended_diff_only": 0.409000039100647,
268
+ "tpp_threshold_20_unintended_diff_only": 0.07000002264976501,
269
+ "tpp_threshold_50_total_metric": 0.3007500171661377,
270
+ "tpp_threshold_50_intended_diff_only": 0.4790000319480896,
271
+ "tpp_threshold_50_unintended_diff_only": 0.1782500147819519,
272
+ "tpp_threshold_100_total_metric": 0.24150002002716064,
273
+ "tpp_threshold_100_intended_diff_only": 0.4830000400543213,
274
+ "tpp_threshold_100_unintended_diff_only": 0.24150002002716064,
275
+ "tpp_threshold_500_total_metric": 0.16449999809265137,
276
+ "tpp_threshold_500_intended_diff_only": 0.49400001764297485,
277
+ "tpp_threshold_500_unintended_diff_only": 0.3295000195503235
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.008250012993812561,
281
+ "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464,
283
+ "tpp_threshold_5_total_metric": 0.012000024318695068,
284
  "tpp_threshold_5_intended_diff_only": 0.025000035762786865,
285
+ "tpp_threshold_5_unintended_diff_only": 0.013000011444091797,
286
+ "tpp_threshold_10_total_metric": 0.17625002562999725,
287
+ "tpp_threshold_10_intended_diff_only": 0.2290000319480896,
288
+ "tpp_threshold_10_unintended_diff_only": 0.052750006318092346,
289
+ "tpp_threshold_20_total_metric": 0.23250001668930054,
290
+ "tpp_threshold_20_intended_diff_only": 0.281000018119812,
291
+ "tpp_threshold_20_unintended_diff_only": 0.048500001430511475,
292
+ "tpp_threshold_50_total_metric": 0.3002500534057617,
293
+ "tpp_threshold_50_intended_diff_only": 0.3890000581741333,
294
+ "tpp_threshold_50_unintended_diff_only": 0.08875000476837158,
295
+ "tpp_threshold_100_total_metric": 0.2630000561475754,
296
+ "tpp_threshold_100_intended_diff_only": 0.44800007343292236,
297
+ "tpp_threshold_100_unintended_diff_only": 0.18500001728534698,
298
+ "tpp_threshold_500_total_metric": 0.15950004756450653,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.3165000230073929
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.0012499988079071045,
306
+ "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
307
+ "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135,
308
+ "tpp_threshold_5_total_metric": 0.004749983549118042,
309
+ "tpp_threshold_5_intended_diff_only": 0.0059999823570251465,
310
+ "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045,
311
+ "tpp_threshold_10_total_metric": 0.004749983549118042,
312
+ "tpp_threshold_10_intended_diff_only": 0.006999969482421875,
313
+ "tpp_threshold_10_unintended_diff_only": 0.002249985933303833,
314
+ "tpp_threshold_20_total_metric": 0.013249978423118591,
315
+ "tpp_threshold_20_intended_diff_only": 0.015999972820281982,
316
+ "tpp_threshold_20_unintended_diff_only": 0.002749994397163391,
317
+ "tpp_threshold_50_total_metric": 0.052750006318092346,
318
+ "tpp_threshold_50_intended_diff_only": 0.050999999046325684,
319
+ "tpp_threshold_50_unintended_diff_only": -0.0017500072717666626,
320
+ "tpp_threshold_100_total_metric": 0.10175003111362457,
321
+ "tpp_threshold_100_intended_diff_only": 0.10900002717971802,
322
+ "tpp_threshold_100_unintended_diff_only": 0.007249996066093445,
323
+ "tpp_threshold_500_total_metric": 0.3980000168085098,
324
+ "tpp_threshold_500_intended_diff_only": 0.40700000524520874,
325
+ "tpp_threshold_500_unintended_diff_only": 0.008999988436698914
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.015000015497207642,
329
+ "tpp_threshold_2_intended_diff_only": 0.013000011444091797,
330
+ "tpp_threshold_2_unintended_diff_only": -0.0020000040531158447,
331
+ "tpp_threshold_5_total_metric": 0.003000020980834961,
332
+ "tpp_threshold_5_intended_diff_only": 0.009000003337860107,
333
+ "tpp_threshold_5_unintended_diff_only": 0.0059999823570251465,
334
+ "tpp_threshold_10_total_metric": 0.043750062584877014,
335
+ "tpp_threshold_10_intended_diff_only": 0.04700005054473877,
336
+ "tpp_threshold_10_unintended_diff_only": 0.0032499879598617554,
337
+ "tpp_threshold_20_total_metric": 0.07650002837181091,
338
+ "tpp_threshold_20_intended_diff_only": 0.08300000429153442,
339
+ "tpp_threshold_20_unintended_diff_only": 0.006499975919723511,
340
+ "tpp_threshold_50_total_metric": 0.14325003325939178,
341
+ "tpp_threshold_50_intended_diff_only": 0.156000018119812,
342
+ "tpp_threshold_50_unintended_diff_only": 0.012749984860420227,
343
+ "tpp_threshold_100_total_metric": 0.25325004756450653,
344
+ "tpp_threshold_100_intended_diff_only": 0.27000004053115845,
345
+ "tpp_threshold_100_unintended_diff_only": 0.016749992966651917,
346
+ "tpp_threshold_500_total_metric": 0.4085000604391098,
347
+ "tpp_threshold_500_intended_diff_only": 0.4360000491142273,
348
+ "tpp_threshold_500_unintended_diff_only": 0.027499988675117493
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.003500014543533325,
352
+ "tpp_threshold_2_intended_diff_only": 0.003000020980834961,
353
+ "tpp_threshold_2_unintended_diff_only": -0.0004999935626983643,
354
+ "tpp_threshold_5_total_metric": 0.01000000536441803,
355
+ "tpp_threshold_5_intended_diff_only": 0.004999995231628418,
356
+ "tpp_threshold_5_unintended_diff_only": -0.005000010132789612,
357
+ "tpp_threshold_10_total_metric": 0.017749980092048645,
358
+ "tpp_threshold_10_intended_diff_only": 0.015999972820281982,
359
+ "tpp_threshold_10_unintended_diff_only": -0.0017500072717666626,
360
+ "tpp_threshold_20_total_metric": 0.013500019907951355,
361
+ "tpp_threshold_20_intended_diff_only": 0.017000019550323486,
362
+ "tpp_threshold_20_unintended_diff_only": 0.0034999996423721313,
363
+ "tpp_threshold_50_total_metric": 0.07150000333786011,
364
+ "tpp_threshold_50_intended_diff_only": 0.0690000057220459,
365
+ "tpp_threshold_50_unintended_diff_only": -0.002499997615814209,
366
+ "tpp_threshold_100_total_metric": 0.13300000131130219,
367
+ "tpp_threshold_100_intended_diff_only": 0.1380000114440918,
368
+ "tpp_threshold_100_unintended_diff_only": 0.005000010132789612,
369
+ "tpp_threshold_500_total_metric": 0.34925003349781036,
370
+ "tpp_threshold_500_intended_diff_only": 0.3680000305175781,
371
+ "tpp_threshold_500_unintended_diff_only": 0.01874999701976776
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.0032499581575393677,
375
+ "tpp_threshold_2_intended_diff_only": 0.0029999613761901855,
376
+ "tpp_threshold_2_unintended_diff_only": -0.00024999678134918213,
377
+ "tpp_threshold_5_total_metric": 0.015999972820281982,
378
+ "tpp_threshold_5_intended_diff_only": 0.014999985694885254,
379
+ "tpp_threshold_5_unintended_diff_only": -0.0009999871253967285,
380
+ "tpp_threshold_10_total_metric": 0.011999964714050293,
381
+ "tpp_threshold_10_intended_diff_only": 0.015999972820281982,
382
+ "tpp_threshold_10_unintended_diff_only": 0.0040000081062316895,
383
+ "tpp_threshold_20_total_metric": 0.04399995505809784,
384
+ "tpp_threshold_20_intended_diff_only": 0.04399996995925903,
385
+ "tpp_threshold_20_unintended_diff_only": 1.4901161193847656e-08,
386
+ "tpp_threshold_50_total_metric": 0.12949998676776886,
387
+ "tpp_threshold_50_intended_diff_only": 0.13499999046325684,
388
+ "tpp_threshold_50_unintended_diff_only": 0.005500003695487976,
389
+ "tpp_threshold_100_total_metric": 0.21249999105930328,
390
+ "tpp_threshold_100_intended_diff_only": 0.22200000286102295,
391
+ "tpp_threshold_100_unintended_diff_only": 0.009500011801719666,
392
+ "tpp_threshold_500_total_metric": 0.34349997341632843,
393
+ "tpp_threshold_500_intended_diff_only": 0.4139999747276306,
394
+ "tpp_threshold_500_unintended_diff_only": 0.07050000131130219
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.0247499942779541,
398
+ "tpp_threshold_2_intended_diff_only": 0.02899998426437378,
399
+ "tpp_threshold_2_unintended_diff_only": 0.004249989986419678,
400
+ "tpp_threshold_5_total_metric": 0.04749999940395355,
401
+ "tpp_threshold_5_intended_diff_only": 0.046999990940093994,
402
+ "tpp_threshold_5_unintended_diff_only": -0.0005000084638595581,
403
+ "tpp_threshold_10_total_metric": 0.09199999272823334,
404
+ "tpp_threshold_10_intended_diff_only": 0.09399998188018799,
405
+ "tpp_threshold_10_unintended_diff_only": 0.001999989151954651,
406
+ "tpp_threshold_20_total_metric": 0.15049998462200165,
407
+ "tpp_threshold_20_intended_diff_only": 0.1589999794960022,
408
+ "tpp_threshold_20_unintended_diff_only": 0.00849999487400055,
409
+ "tpp_threshold_50_total_metric": 0.22124996781349182,
410
+ "tpp_threshold_50_intended_diff_only": 0.22899997234344482,
411
+ "tpp_threshold_50_unintended_diff_only": 0.007750004529953003,
412
+ "tpp_threshold_100_total_metric": 0.2824999988079071,
413
+ "tpp_threshold_100_intended_diff_only": 0.2919999957084656,
414
+ "tpp_threshold_100_unintended_diff_only": 0.009499996900558472,
415
+ "tpp_threshold_500_total_metric": 0.3425000160932541,
416
+ "tpp_threshold_500_intended_diff_only": 0.37300002574920654,
417
+ "tpp_threshold_500_unintended_diff_only": 0.030500009655952454
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-134277-tokens-550000000/scr/saebench_scr_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,31 +69,31 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "4a893800-28cc-4140-a764-87637efe1b4a",
69
- "datetime_epoch_millis": 1776883995505,
70
  "eval_result_metrics": {
71
  "scr_metrics": {
72
  "scr_dir1_threshold_2": 0.27679038925488475,
73
  "scr_metric_threshold_2": 0.11448470285338772,
74
  "scr_dir2_threshold_2": 0.11011504535000179,
75
  "scr_dir1_threshold_5": 0.2516291575551625,
76
- "scr_metric_threshold_5": 0.18619278477797802,
77
- "scr_dir2_threshold_5": 0.1800000058909232,
78
  "scr_dir1_threshold_10": 0.25503683173522657,
79
  "scr_metric_threshold_10": 0.23262178467385908,
80
  "scr_dir2_threshold_10": 0.2263658672576473,
81
- "scr_dir1_threshold_20": null,
82
- "scr_metric_threshold_20": null,
83
- "scr_dir2_threshold_20": null,
84
- "scr_dir1_threshold_50": null,
85
- "scr_metric_threshold_50": null,
86
- "scr_dir2_threshold_50": null,
87
- "scr_dir1_threshold_100": null,
88
- "scr_metric_threshold_100": null,
89
- "scr_dir2_threshold_100": null,
90
- "scr_dir1_threshold_500": null,
91
- "scr_metric_threshold_500": null,
92
- "scr_dir2_threshold_500": null
93
  }
94
  },
95
  "eval_result_details": [
@@ -104,18 +108,18 @@
104
  "scr_dir1_threshold_10": 0.5641028776049328,
105
  "scr_metric_threshold_10": 0.08290156000473131,
106
  "scr_dir2_threshold_10": 0.08290156000473131,
107
- "scr_dir1_threshold_20": null,
108
- "scr_metric_threshold_20": null,
109
- "scr_dir2_threshold_20": null,
110
- "scr_dir1_threshold_50": null,
111
- "scr_metric_threshold_50": null,
112
- "scr_dir2_threshold_50": null,
113
- "scr_dir1_threshold_100": null,
114
- "scr_metric_threshold_100": null,
115
- "scr_dir2_threshold_100": null,
116
- "scr_dir1_threshold_500": null,
117
- "scr_metric_threshold_500": null,
118
- "scr_dir2_threshold_500": null
119
  },
120
  {
121
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
@@ -128,18 +132,18 @@
128
  "scr_dir1_threshold_10": 0.23484857036936357,
129
  "scr_metric_threshold_10": 0.33639145849092716,
130
  "scr_dir2_threshold_10": 0.33639145849092716,
131
- "scr_dir1_threshold_20": null,
132
- "scr_metric_threshold_20": null,
133
- "scr_dir2_threshold_20": null,
134
- "scr_dir1_threshold_50": null,
135
- "scr_metric_threshold_50": null,
136
- "scr_dir2_threshold_50": null,
137
- "scr_dir1_threshold_100": null,
138
- "scr_metric_threshold_100": null,
139
- "scr_dir2_threshold_100": null,
140
- "scr_dir1_threshold_500": null,
141
- "scr_metric_threshold_500": null,
142
- "scr_dir2_threshold_500": null
143
  },
144
  {
145
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
@@ -152,18 +156,18 @@
152
  "scr_dir1_threshold_10": 0.31249965075409175,
153
  "scr_metric_threshold_10": 0.1482412214577287,
154
  "scr_dir2_threshold_10": 0.1482412214577287,
155
- "scr_dir1_threshold_20": null,
156
- "scr_metric_threshold_20": null,
157
- "scr_dir2_threshold_20": null,
158
- "scr_dir1_threshold_50": null,
159
- "scr_metric_threshold_50": null,
160
- "scr_dir2_threshold_50": null,
161
- "scr_dir1_threshold_100": null,
162
- "scr_metric_threshold_100": null,
163
- "scr_dir2_threshold_100": null,
164
- "scr_dir1_threshold_500": null,
165
- "scr_metric_threshold_500": null,
166
- "scr_dir2_threshold_500": null
167
  },
168
  {
169
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
@@ -171,23 +175,23 @@
171
  "scr_metric_threshold_2": 0.03184710232218929,
172
  "scr_dir2_threshold_2": 0.03184710232218929,
173
  "scr_dir1_threshold_5": 0.04697966441382428,
174
- "scr_metric_threshold_5": 0.10509551359270591,
175
- "scr_dir2_threshold_5": 0.10509551359270591,
176
  "scr_dir1_threshold_10": -0.10067145237328512,
177
  "scr_metric_threshold_10": 0.1305732713799386,
178
  "scr_dir2_threshold_10": 0.1305732713799386,
179
- "scr_dir1_threshold_20": null,
180
- "scr_metric_threshold_20": null,
181
- "scr_dir2_threshold_20": null,
182
- "scr_dir1_threshold_50": null,
183
- "scr_metric_threshold_50": null,
184
- "scr_dir2_threshold_50": null,
185
- "scr_dir1_threshold_100": null,
186
- "scr_metric_threshold_100": null,
187
- "scr_dir2_threshold_100": null,
188
- "scr_dir1_threshold_500": null,
189
- "scr_metric_threshold_500": null,
190
- "scr_dir2_threshold_500": null
191
  },
192
  {
193
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
@@ -200,18 +204,18 @@
200
  "scr_dir1_threshold_10": 0.08421069803000372,
201
  "scr_metric_threshold_10": 0.4166666863771947,
202
  "scr_dir2_threshold_10": 0.4166666863771947,
203
- "scr_dir1_threshold_20": null,
204
- "scr_metric_threshold_20": null,
205
- "scr_dir2_threshold_20": null,
206
- "scr_dir1_threshold_50": null,
207
- "scr_metric_threshold_50": null,
208
- "scr_dir2_threshold_50": null,
209
- "scr_dir1_threshold_100": null,
210
- "scr_metric_threshold_100": null,
211
- "scr_dir2_threshold_100": null,
212
- "scr_dir1_threshold_500": null,
213
- "scr_metric_threshold_500": null,
214
- "scr_dir2_threshold_500": null
215
  },
216
  {
217
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
@@ -224,18 +228,18 @@
224
  "scr_dir1_threshold_10": 0.3825136878996036,
225
  "scr_metric_threshold_10": 0.14885507905463402,
226
  "scr_dir2_threshold_10": 0.14885507905463402,
227
- "scr_dir1_threshold_20": null,
228
- "scr_metric_threshold_20": null,
229
- "scr_dir2_threshold_20": null,
230
- "scr_dir1_threshold_50": null,
231
- "scr_metric_threshold_50": null,
232
- "scr_dir2_threshold_50": null,
233
- "scr_dir1_threshold_100": null,
234
- "scr_metric_threshold_100": null,
235
- "scr_dir2_threshold_100": null,
236
- "scr_dir1_threshold_500": null,
237
- "scr_metric_threshold_500": null,
238
- "scr_dir2_threshold_500": null
239
  },
240
  {
241
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
@@ -248,18 +252,18 @@
248
  "scr_dir1_threshold_10": 0.3860464716550849,
249
  "scr_metric_threshold_10": 0.4206008506837011,
250
  "scr_dir2_threshold_10": 0.4206008506837011,
251
- "scr_dir1_threshold_20": null,
252
- "scr_metric_threshold_20": null,
253
- "scr_dir2_threshold_20": null,
254
- "scr_dir1_threshold_50": null,
255
- "scr_metric_threshold_50": null,
256
- "scr_dir2_threshold_50": null,
257
- "scr_dir1_threshold_100": null,
258
- "scr_metric_threshold_100": null,
259
- "scr_dir2_threshold_100": null,
260
- "scr_dir1_threshold_500": null,
261
- "scr_metric_threshold_500": null,
262
- "scr_dir2_threshold_500": null
263
  },
264
  {
265
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
@@ -272,18 +276,18 @@
272
  "scr_dir1_threshold_10": 0.17674414994201715,
273
  "scr_metric_threshold_10": 0.17674414994201715,
274
  "scr_dir2_threshold_10": 0.12669681061232305,
275
- "scr_dir1_threshold_20": null,
276
- "scr_metric_threshold_20": null,
277
- "scr_dir2_threshold_20": null,
278
- "scr_dir1_threshold_50": null,
279
- "scr_metric_threshold_50": null,
280
- "scr_dir2_threshold_50": null,
281
- "scr_dir1_threshold_100": null,
282
- "scr_metric_threshold_100": null,
283
- "scr_dir2_threshold_100": null,
284
- "scr_dir1_threshold_500": null,
285
- "scr_metric_threshold_500": null,
286
- "scr_dir2_threshold_500": null
287
  }
288
  ],
289
  "sae_bench_commit_hash": "Unknown",
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "52d0a3c8-3d5e-4e60-9281-3c3ff4323bb1",
73
+ "datetime_epoch_millis": 1777315626029,
74
  "eval_result_metrics": {
75
  "scr_metrics": {
76
  "scr_dir1_threshold_2": 0.27679038925488475,
77
  "scr_metric_threshold_2": 0.11448470285338772,
78
  "scr_dir2_threshold_2": 0.11011504535000179,
79
  "scr_dir1_threshold_5": 0.2516291575551625,
80
+ "scr_metric_threshold_5": 0.1857947007445432,
81
+ "scr_dir2_threshold_5": 0.17960192185748838,
82
  "scr_dir1_threshold_10": 0.25503683173522657,
83
  "scr_metric_threshold_10": 0.23262178467385908,
84
  "scr_dir2_threshold_10": 0.2263658672576473,
85
+ "scr_dir1_threshold_20": -0.022267688790582715,
86
+ "scr_metric_threshold_20": 0.2600485209905197,
87
+ "scr_dir2_threshold_20": 0.254184610575226,
88
+ "scr_dir1_threshold_50": 0.22170874833388707,
89
+ "scr_metric_threshold_50": 0.32764610137984584,
90
+ "scr_dir2_threshold_50": 0.3239814665851593,
91
+ "scr_dir1_threshold_100": 0.0005200397136160759,
92
+ "scr_metric_threshold_100": 0.33646740091203153,
93
+ "scr_dir2_threshold_100": 0.3285593847314374,
94
+ "scr_dir1_threshold_500": -0.730662973340079,
95
+ "scr_metric_threshold_500": 0.14416851198599903,
96
+ "scr_dir2_threshold_500": 0.11886596932476784
97
  }
98
  },
99
  "eval_result_details": [
 
108
  "scr_dir1_threshold_10": 0.5641028776049328,
109
  "scr_metric_threshold_10": 0.08290156000473131,
110
  "scr_dir2_threshold_10": 0.08290156000473131,
111
+ "scr_dir1_threshold_20": 0.08974395223070357,
112
+ "scr_metric_threshold_20": 0.1398964114610147,
113
+ "scr_dir2_threshold_20": 0.1398964114610147,
114
+ "scr_dir1_threshold_50": 0.3205132417816285,
115
+ "scr_metric_threshold_50": 0.27461138856264183,
116
+ "scr_dir2_threshold_50": 0.27461138856264183,
117
+ "scr_dir1_threshold_100": 0.4487182328294704,
118
+ "scr_metric_threshold_100": 0.33160624001892525,
119
+ "scr_dir2_threshold_100": 0.33160624001892525,
120
+ "scr_dir1_threshold_500": -2.0769228417963004,
121
+ "scr_metric_threshold_500": 0.09585491427895527,
122
+ "scr_dir2_threshold_500": 0.09585491427895527
123
  },
124
  {
125
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
 
132
  "scr_dir1_threshold_10": 0.23484857036936357,
133
  "scr_metric_threshold_10": 0.33639145849092716,
134
  "scr_dir2_threshold_10": 0.33639145849092716,
135
+ "scr_dir1_threshold_20": 0.022727426664854444,
136
+ "scr_metric_threshold_20": 0.20795111325793134,
137
+ "scr_dir2_threshold_20": 0.20795111325793134,
138
+ "scr_dir1_threshold_50": -0.08333314518740012,
139
+ "scr_metric_threshold_50": 0.5198776920062339,
140
+ "scr_dir2_threshold_50": 0.5198776920062339,
141
+ "scr_dir1_threshold_100": 0.04545485332970889,
142
+ "scr_metric_threshold_100": 0.5137615632091722,
143
+ "scr_dir2_threshold_100": 0.5137615632091722,
144
+ "scr_dir1_threshold_500": -0.3939392023726256,
145
+ "scr_metric_threshold_500": -0.24464825059467932,
146
+ "scr_dir2_threshold_500": -0.24464825059467932
147
  },
148
  {
149
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
 
156
  "scr_dir1_threshold_10": 0.31249965075409175,
157
  "scr_metric_threshold_10": 0.1482412214577287,
158
  "scr_dir2_threshold_10": 0.1482412214577287,
159
+ "scr_dir1_threshold_20": -0.5,
160
+ "scr_metric_threshold_20": 0.16582923152102794,
161
+ "scr_dir2_threshold_20": 0.16582923152102794,
162
+ "scr_dir1_threshold_50": 0.3906245634426147,
163
+ "scr_metric_threshold_50": 0.13065336115482035,
164
+ "scr_dir2_threshold_50": 0.13065336115482035,
165
+ "scr_dir1_threshold_100": -1.2656243306120092,
166
+ "scr_metric_threshold_100": 0.037688550204100946,
167
+ "scr_dir2_threshold_100": 0.037688550204100946,
168
+ "scr_dir1_threshold_500": -2.7187499417923484,
169
+ "scr_metric_threshold_500": -0.13316574147193197,
170
+ "scr_dir2_threshold_500": -0.13316574147193197
171
  },
172
  {
173
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
 
175
  "scr_metric_threshold_2": 0.03184710232218929,
176
  "scr_dir2_threshold_2": 0.03184710232218929,
177
  "scr_dir1_threshold_5": 0.04697966441382428,
178
+ "scr_metric_threshold_5": 0.10191084132522761,
179
+ "scr_dir2_threshold_5": 0.10191084132522761,
180
  "scr_dir1_threshold_10": -0.10067145237328512,
181
  "scr_metric_threshold_10": 0.1305732713799386,
182
  "scr_dir2_threshold_10": 0.1305732713799386,
183
+ "scr_dir1_threshold_20": -0.8791949772082223,
184
+ "scr_metric_threshold_20": 0.14968149480851153,
185
+ "scr_dir2_threshold_20": 0.14968149480851153,
186
+ "scr_dir1_threshold_50": 0.12080502279177775,
187
+ "scr_metric_threshold_50": 0.14968149480851153,
188
+ "scr_dir2_threshold_50": 0.14968149480851153,
189
+ "scr_dir1_threshold_100": -0.36912759195217226,
190
+ "scr_metric_threshold_100": 0.1624203737021279,
191
+ "scr_dir2_threshold_100": 0.1624203737021279,
192
+ "scr_dir1_threshold_500": -1.738255583935547,
193
+ "scr_metric_threshold_500": -0.509554016802435,
194
+ "scr_dir2_threshold_500": -0.509554016802435
195
  },
196
  {
197
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
 
204
  "scr_dir1_threshold_10": 0.08421069803000372,
205
  "scr_metric_threshold_10": 0.4166666863771947,
206
  "scr_dir2_threshold_10": 0.4166666863771947,
207
+ "scr_dir1_threshold_20": 0.08421069803000372,
208
+ "scr_metric_threshold_20": 0.5357141758984867,
209
+ "scr_dir2_threshold_20": 0.5357141758984867,
210
+ "scr_dir1_threshold_50": -0.19999993725826787,
211
+ "scr_metric_threshold_50": 0.6230158101299026,
212
+ "scr_dir2_threshold_50": 0.6230158101299026,
213
+ "scr_dir1_threshold_100": -0.21578952156605874,
214
+ "scr_metric_threshold_100": 0.6349206536925663,
215
+ "scr_dir2_threshold_100": 0.6349206536925663,
216
+ "scr_dir1_threshold_500": -0.16842108235134678,
217
+ "scr_metric_threshold_500": 0.6190476077844602,
218
+ "scr_dir2_threshold_500": 0.6190476077844602
219
  },
220
  {
221
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
 
228
  "scr_dir1_threshold_10": 0.3825136878996036,
229
  "scr_metric_threshold_10": 0.14885507905463402,
230
  "scr_dir2_threshold_10": 0.14885507905463402,
231
+ "scr_dir1_threshold_20": 0.4043716594522235,
232
+ "scr_metric_threshold_20": 0.19847321124088896,
233
+ "scr_dir2_threshold_20": 0.19847321124088896,
234
+ "scr_dir1_threshold_50": 0.4808743970321442,
235
+ "scr_metric_threshold_50": 0.28625946991455836,
236
+ "scr_dir2_threshold_50": 0.28625946991455836,
237
+ "scr_dir1_threshold_100": 0.4535520140184937,
238
+ "scr_metric_threshold_100": 0.28625946991455836,
239
+ "scr_dir2_threshold_100": 0.28625946991455836,
240
+ "scr_dir1_threshold_500": 0.5300547515984144,
241
+ "scr_metric_threshold_500": 0.38931293368346886,
242
+ "scr_dir2_threshold_500": 0.38931293368346886
243
  },
244
  {
245
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
 
252
  "scr_dir1_threshold_10": 0.3860464716550849,
253
  "scr_metric_threshold_10": 0.4206008506837011,
254
  "scr_dir2_threshold_10": 0.4206008506837011,
255
+ "scr_dir1_threshold_20": 0.3720928878639972,
256
+ "scr_metric_threshold_20": 0.4549356870945189,
257
+ "scr_dir2_threshold_20": 0.4549356870945189,
258
+ "scr_dir1_threshold_50": 0.49767431029119774,
259
+ "scr_metric_threshold_50": 0.3905580606846963,
260
+ "scr_dir2_threshold_50": 0.3905580606846963,
261
+ "scr_dir1_threshold_100": 0.5767441563892566,
262
+ "scr_metric_threshold_100": 0.3948498512825616,
263
+ "scr_dir2_threshold_100": 0.3948498512825616,
264
+ "scr_dir1_threshold_500": 0.2651161514146259,
265
+ "scr_metric_threshold_500": 0.48068668649565843,
266
+ "scr_dir2_threshold_500": 0.48068668649565843
267
  },
268
  {
269
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
 
276
  "scr_dir1_threshold_10": 0.17674414994201715,
277
  "scr_metric_threshold_10": 0.17674414994201715,
278
  "scr_dir2_threshold_10": 0.12669681061232305,
279
+ "scr_dir1_threshold_20": 0.227906842641778,
280
+ "scr_metric_threshold_20": 0.227906842641778,
281
+ "scr_dir2_threshold_20": 0.1809955593194283,
282
+ "scr_dir1_threshold_50": 0.24651153377740184,
283
+ "scr_metric_threshold_50": 0.24651153377740184,
284
+ "scr_dir2_threshold_50": 0.2171944554199093,
285
+ "scr_dir1_threshold_100": 0.3302325052722394,
286
+ "scr_metric_threshold_100": 0.3302325052722394,
287
+ "scr_dir2_threshold_100": 0.2669683758274864,
288
+ "scr_dir1_threshold_500": 0.4558139625144957,
289
+ "scr_metric_threshold_500": 0.4558139625144957,
290
+ "scr_dir2_threshold_500": 0.2533936212246461
291
  }
292
  ],
293
  "sae_bench_commit_hash": "Unknown",
gemma-2-2b-btk/k-100/seed-0/snapshots/step-134277-tokens-550000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "a32b1258-f85a-4ba5-9120-84849dd3f983",
69
- "datetime_epoch_millis": 1776884158771,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.021700002253055573,
73
- "tpp_threshold_2_intended_diff_only": 0.022700005769729616,
74
- "tpp_threshold_2_unintended_diff_only": 0.0010000035166740418,
75
- "tpp_threshold_5_total_metric": 0.061674997210502625,
76
- "tpp_threshold_5_intended_diff_only": 0.07070000171661377,
77
- "tpp_threshold_5_unintended_diff_only": 0.009025004506111146,
78
- "tpp_threshold_10_total_metric": 0.10635001063346863,
79
- "tpp_threshold_10_intended_diff_only": 0.1358000159263611,
80
- "tpp_threshold_10_unintended_diff_only": 0.029450005292892455,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.03174999058246612,
99
- "tpp_threshold_2_intended_diff_only": 0.033399999141693115,
100
- "tpp_threshold_2_unintended_diff_only": 0.0016500085592269897,
101
- "tpp_threshold_5_total_metric": 0.10250000357627868,
102
  "tpp_threshold_5_intended_diff_only": 0.11820001602172851,
103
- "tpp_threshold_5_unintended_diff_only": 0.01570001244544983,
104
- "tpp_threshold_10_total_metric": 0.18215001821517945,
105
  "tpp_threshold_10_intended_diff_only": 0.23820003271102905,
106
- "tpp_threshold_10_unintended_diff_only": 0.05605001449584961,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.011650013923645019,
123
- "tpp_threshold_2_intended_diff_only": 0.012000012397766113,
124
- "tpp_threshold_2_unintended_diff_only": 0.00034999847412109375,
125
- "tpp_threshold_5_total_metric": 0.020849990844726562,
126
- "tpp_threshold_5_intended_diff_only": 0.023199987411499024,
127
- "tpp_threshold_5_unintended_diff_only": 0.002349996566772461,
128
- "tpp_threshold_10_total_metric": 0.030550003051757812,
129
- "tpp_threshold_10_intended_diff_only": 0.033399999141693115,
130
- "tpp_threshold_10_unintended_diff_only": 0.0028499960899353027,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.0020000189542770386,
190
  "tpp_threshold_10_total_metric": 0.013249963521957397,
191
  "tpp_threshold_10_intended_diff_only": 0.015999972820281982,
192
- "tpp_threshold_10_unintended_diff_only": 0.002750009298324585
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": -2.9802322387695312e-08,
196
- "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
197
- "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162,
198
- "tpp_threshold_5_total_metric": 0.15024997293949127,
199
  "tpp_threshold_5_intended_diff_only": 0.17599999904632568,
200
- "tpp_threshold_5_unintended_diff_only": 0.02575002610683441,
201
- "tpp_threshold_10_total_metric": 0.200500026345253,
202
  "tpp_threshold_10_intended_diff_only": 0.26100003719329834,
203
- "tpp_threshold_10_unintended_diff_only": 0.06050001084804535
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.13449999690055847,
207
- "tpp_threshold_2_intended_diff_only": 0.14300000667572021,
208
- "tpp_threshold_2_unintended_diff_only": 0.008500009775161743,
209
- "tpp_threshold_5_total_metric": 0.22200004756450653,
210
- "tpp_threshold_5_intended_diff_only": 0.2510000467300415,
211
- "tpp_threshold_5_unintended_diff_only": 0.028999999165534973,
212
- "tpp_threshold_10_total_metric": 0.22675003111362457,
213
- "tpp_threshold_10_intended_diff_only": 0.315000057220459,
214
- "tpp_threshold_10_unintended_diff_only": 0.08825002610683441
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.011000022292137146,
218
  "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
219
- "tpp_threshold_2_unintended_diff_only": -0.003000006079673767,
220
- "tpp_threshold_5_total_metric": 0.10750003159046173,
221
- "tpp_threshold_5_intended_diff_only": 0.12200003862380981,
222
- "tpp_threshold_5_unintended_diff_only": 0.014500007033348083,
223
- "tpp_threshold_10_total_metric": 0.29575005173683167,
224
  "tpp_threshold_10_intended_diff_only": 0.37000006437301636,
225
- "tpp_threshold_10_unintended_diff_only": 0.07425001263618469
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.003999993205070496,
229
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
230
  "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224,
231
- "tpp_threshold_5_total_metric": 0.017750024795532227,
232
  "tpp_threshold_5_intended_diff_only": 0.025000035762786865,
233
- "tpp_threshold_5_unintended_diff_only": 0.007250010967254639,
234
- "tpp_threshold_10_total_metric": 0.1745000183582306,
235
- "tpp_threshold_10_intended_diff_only": 0.2290000319480896,
236
- "tpp_threshold_10_unintended_diff_only": 0.05450001358985901
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.007750019431114197,
242
- "tpp_threshold_2_intended_diff_only": 0.003000020980834961,
243
- "tpp_threshold_2_unintended_diff_only": -0.004749998450279236,
244
- "tpp_threshold_5_total_metric": 0.013499975204467773,
245
- "tpp_threshold_5_intended_diff_only": 0.010999977588653564,
246
- "tpp_threshold_5_unintended_diff_only": -0.002499997615814209,
247
- "tpp_threshold_10_total_metric": 0.0024999678134918213,
248
  "tpp_threshold_10_intended_diff_only": 0.006999969482421875,
249
- "tpp_threshold_10_unintended_diff_only": 0.004500001668930054
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": 0.0020000338554382324,
253
- "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
254
- "tpp_threshold_2_unintended_diff_only": 0.004999995231628418,
255
- "tpp_threshold_5_total_metric": 0.008250057697296143,
256
- "tpp_threshold_5_intended_diff_only": 0.016000032424926758,
257
- "tpp_threshold_5_unintended_diff_only": 0.007749974727630615,
258
- "tpp_threshold_10_total_metric": 0.04075002670288086,
259
- "tpp_threshold_10_intended_diff_only": 0.04400002956390381,
260
- "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.013750031590461731,
264
- "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
265
- "tpp_threshold_2_unintended_diff_only": -0.006750002503395081,
266
- "tpp_threshold_5_total_metric": 0.01825001835823059,
267
- "tpp_threshold_5_intended_diff_only": 0.017000019550323486,
268
- "tpp_threshold_5_unintended_diff_only": -0.0012499988079071045,
269
- "tpp_threshold_10_total_metric": 0.015000015497207642,
270
- "tpp_threshold_10_intended_diff_only": 0.018000006675720215,
271
- "tpp_threshold_10_unintended_diff_only": 0.0029999911785125732
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.006750017404556274,
275
- "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
276
- "tpp_threshold_2_unintended_diff_only": 0.007750004529953003,
277
- "tpp_threshold_5_total_metric": 0.0029999464750289917,
278
- "tpp_threshold_5_intended_diff_only": 0.003999948501586914,
279
- "tpp_threshold_5_unintended_diff_only": 0.0010000020265579224,
280
- "tpp_threshold_10_total_metric": 0.007750004529953003,
281
- "tpp_threshold_10_intended_diff_only": 0.009999990463256836,
282
- "tpp_threshold_10_unintended_diff_only": 0.002249985933303833
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.04150000214576721,
286
- "tpp_threshold_2_intended_diff_only": 0.041999995708465576,
287
- "tpp_threshold_2_unintended_diff_only": 0.0004999935626983643,
288
- "tpp_threshold_5_total_metric": 0.061249956488609314,
289
- "tpp_threshold_5_intended_diff_only": 0.0679999589920044,
290
- "tpp_threshold_5_unintended_diff_only": 0.006750002503395081,
291
- "tpp_threshold_10_total_metric": 0.08675000071525574,
292
- "tpp_threshold_10_intended_diff_only": 0.08799999952316284,
293
- "tpp_threshold_10_unintended_diff_only": 0.0012499988079071045
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "607f6535-ca12-468e-9875-eb95b41b0228",
73
+ "datetime_epoch_millis": 1777315903998,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.020499995350837706,
77
+ "tpp_threshold_2_intended_diff_only": 0.02290000319480896,
78
+ "tpp_threshold_2_unintended_diff_only": 0.0024000078439712523,
79
+ "tpp_threshold_5_total_metric": 0.06154999881982803,
80
+ "tpp_threshold_5_intended_diff_only": 0.06920000314712524,
81
+ "tpp_threshold_5_unintended_diff_only": 0.00765000432729721,
82
+ "tpp_threshold_10_total_metric": 0.10792499929666519,
83
+ "tpp_threshold_10_intended_diff_only": 0.13700000643730165,
84
+ "tpp_threshold_10_unintended_diff_only": 0.029075007140636447,
85
+ "tpp_threshold_20_total_metric": 0.14760000854730607,
86
+ "tpp_threshold_20_intended_diff_only": 0.1952000141143799,
87
+ "tpp_threshold_20_unintended_diff_only": 0.04760000556707382,
88
+ "tpp_threshold_50_total_metric": 0.17302500754594802,
89
+ "tpp_threshold_50_intended_diff_only": 0.254500013589859,
90
+ "tpp_threshold_50_unintended_diff_only": 0.08147500604391099,
91
+ "tpp_threshold_100_total_metric": 0.1921500101685524,
92
+ "tpp_threshold_100_intended_diff_only": 0.3089000165462494,
93
+ "tpp_threshold_100_unintended_diff_only": 0.11675000637769699,
94
+ "tpp_threshold_500_total_metric": 0.24855002611875535,
95
+ "tpp_threshold_500_intended_diff_only": 0.4323000371456146,
96
+ "tpp_threshold_500_unintended_diff_only": 0.18375001102685928
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.030799999833106995,
103
+ "tpp_threshold_2_intended_diff_only": 0.03420001268386841,
104
+ "tpp_threshold_2_unintended_diff_only": 0.0034000128507614136,
105
+ "tpp_threshold_5_total_metric": 0.10330000221729278,
106
  "tpp_threshold_5_intended_diff_only": 0.11820001602172851,
107
+ "tpp_threshold_5_unintended_diff_only": 0.01490001380443573,
108
+ "tpp_threshold_10_total_metric": 0.1831500142812729,
109
  "tpp_threshold_10_intended_diff_only": 0.23820003271102905,
110
+ "tpp_threshold_10_unintended_diff_only": 0.05505001842975617,
111
+ "tpp_threshold_20_total_metric": 0.24570001363754274,
112
+ "tpp_threshold_20_intended_diff_only": 0.3366000294685364,
113
+ "tpp_threshold_20_unintended_diff_only": 0.09090001583099365,
114
+ "tpp_threshold_50_total_metric": 0.24230002462863923,
115
+ "tpp_threshold_50_intended_diff_only": 0.4008000373840332,
116
+ "tpp_threshold_50_unintended_diff_only": 0.158500012755394,
117
+ "tpp_threshold_100_total_metric": 0.2114500105381012,
118
+ "tpp_threshold_100_intended_diff_only": 0.43480002880096436,
119
+ "tpp_threshold_100_unintended_diff_only": 0.22335001826286316,
120
+ "tpp_threshold_500_total_metric": 0.12215001881122589,
121
+ "tpp_threshold_500_intended_diff_only": 0.4670000433921814,
122
+ "tpp_threshold_500_unintended_diff_only": 0.3448500245809555
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.010199990868568421,
127
+ "tpp_threshold_2_intended_diff_only": 0.011599993705749512,
128
+ "tpp_threshold_2_unintended_diff_only": 0.0014000028371810914,
129
+ "tpp_threshold_5_total_metric": 0.01979999542236328,
130
+ "tpp_threshold_5_intended_diff_only": 0.02019999027252197,
131
+ "tpp_threshold_5_unintended_diff_only": 0.0003999948501586914,
132
+ "tpp_threshold_10_total_metric": 0.03269998431205749,
133
+ "tpp_threshold_10_intended_diff_only": 0.03579998016357422,
134
+ "tpp_threshold_10_unintended_diff_only": 0.003099995851516724,
135
+ "tpp_threshold_20_total_metric": 0.0495000034570694,
136
+ "tpp_threshold_20_intended_diff_only": 0.053799998760223386,
137
+ "tpp_threshold_20_unintended_diff_only": 0.0042999953031539915,
138
+ "tpp_threshold_50_total_metric": 0.10374999046325684,
139
+ "tpp_threshold_50_intended_diff_only": 0.10819998979568482,
140
+ "tpp_threshold_50_unintended_diff_only": 0.004449999332427979,
141
+ "tpp_threshold_100_total_metric": 0.1728500097990036,
142
+ "tpp_threshold_100_intended_diff_only": 0.18300000429153443,
143
+ "tpp_threshold_100_unintended_diff_only": 0.010149994492530822,
144
+ "tpp_threshold_500_total_metric": 0.3749500334262848,
145
+ "tpp_threshold_500_intended_diff_only": 0.39760003089904783,
146
+ "tpp_threshold_500_unintended_diff_only": 0.022649997472763063
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.0020000189542770386,
194
  "tpp_threshold_10_total_metric": 0.013249963521957397,
195
  "tpp_threshold_10_intended_diff_only": 0.015999972820281982,
196
+ "tpp_threshold_10_unintended_diff_only": 0.002750009298324585,
197
+ "tpp_threshold_20_total_metric": 0.23674999177455902,
198
+ "tpp_threshold_20_intended_diff_only": 0.30800002813339233,
199
+ "tpp_threshold_20_unintended_diff_only": 0.07125003635883331,
200
+ "tpp_threshold_50_total_metric": 0.2720000147819519,
201
+ "tpp_threshold_50_intended_diff_only": 0.3970000147819519,
202
+ "tpp_threshold_50_unintended_diff_only": 0.125,
203
+ "tpp_threshold_100_total_metric": 0.2347499430179596,
204
+ "tpp_threshold_100_intended_diff_only": 0.43699997663497925,
205
+ "tpp_threshold_100_unintended_diff_only": 0.20225003361701965,
206
+ "tpp_threshold_500_total_metric": 0.14925000071525574,
207
+ "tpp_threshold_500_intended_diff_only": 0.45600003004074097,
208
+ "tpp_threshold_500_unintended_diff_only": 0.30675002932548523
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.004499971866607666,
212
+ "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
213
+ "tpp_threshold_2_unintended_diff_only": 0.000500023365020752,
214
+ "tpp_threshold_5_total_metric": 0.15549997985363007,
215
  "tpp_threshold_5_intended_diff_only": 0.17599999904632568,
216
+ "tpp_threshold_5_unintended_diff_only": 0.020500019192695618,
217
+ "tpp_threshold_10_total_metric": 0.20125000178813934,
218
  "tpp_threshold_10_intended_diff_only": 0.26100003719329834,
219
+ "tpp_threshold_10_unintended_diff_only": 0.059750035405159,
220
+ "tpp_threshold_20_total_metric": 0.23274998366832733,
221
+ "tpp_threshold_20_intended_diff_only": 0.296999990940094,
222
+ "tpp_threshold_20_unintended_diff_only": 0.06425000727176666,
223
+ "tpp_threshold_50_total_metric": 0.2394999861717224,
224
+ "tpp_threshold_50_intended_diff_only": 0.3700000047683716,
225
+ "tpp_threshold_50_unintended_diff_only": 0.13050001859664917,
226
+ "tpp_threshold_100_total_metric": 0.216000035405159,
227
+ "tpp_threshold_100_intended_diff_only": 0.40800005197525024,
228
+ "tpp_threshold_100_unintended_diff_only": 0.19200001657009125,
229
+ "tpp_threshold_500_total_metric": 0.10525001585483551,
230
+ "tpp_threshold_500_intended_diff_only": 0.4590000510215759,
231
+ "tpp_threshold_500_unintended_diff_only": 0.3537500351667404
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.12450003623962402,
235
+ "tpp_threshold_2_intended_diff_only": 0.14000004529953003,
236
+ "tpp_threshold_2_unintended_diff_only": 0.015500009059906006,
237
+ "tpp_threshold_5_total_metric": 0.21875004470348358,
238
+ "tpp_threshold_5_intended_diff_only": 0.2500000596046448,
239
+ "tpp_threshold_5_unintended_diff_only": 0.031250014901161194,
240
+ "tpp_threshold_10_total_metric": 0.22950002551078796,
241
+ "tpp_threshold_10_intended_diff_only": 0.3160000443458557,
242
+ "tpp_threshold_10_unintended_diff_only": 0.08650001883506775,
243
+ "tpp_threshold_20_total_metric": 0.2030000388622284,
244
+ "tpp_threshold_20_intended_diff_only": 0.35700005292892456,
245
+ "tpp_threshold_20_unintended_diff_only": 0.15400001406669617,
246
+ "tpp_threshold_50_total_metric": 0.15275003015995026,
247
+ "tpp_threshold_50_intended_diff_only": 0.3760000467300415,
248
+ "tpp_threshold_50_unintended_diff_only": 0.22325001657009125,
249
+ "tpp_threshold_100_total_metric": 0.12000003457069397,
250
+ "tpp_threshold_100_intended_diff_only": 0.41200006008148193,
251
+ "tpp_threshold_100_unintended_diff_only": 0.29200002551078796,
252
+ "tpp_threshold_500_total_metric": 0.06074999272823334,
253
+ "tpp_threshold_500_intended_diff_only": 0.4520000219345093,
254
+ "tpp_threshold_500_unintended_diff_only": 0.39125002920627594
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.008750006556510925,
258
  "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0007499903440475464,
260
+ "tpp_threshold_5_total_metric": 0.11275002360343933,
261
+ "tpp_threshold_5_intended_diff_only": 0.12300002574920654,
262
+ "tpp_threshold_5_unintended_diff_only": 0.010250002145767212,
263
+ "tpp_threshold_10_total_metric": 0.2980000525712967,
264
  "tpp_threshold_10_intended_diff_only": 0.37000006437301636,
265
+ "tpp_threshold_10_unintended_diff_only": 0.07200001180171967,
266
+ "tpp_threshold_20_total_metric": 0.32750000059604645,
267
+ "tpp_threshold_20_intended_diff_only": 0.44300001859664917,
268
+ "tpp_threshold_20_unintended_diff_only": 0.11550001800060272,
269
+ "tpp_threshold_50_total_metric": 0.28025004267692566,
270
+ "tpp_threshold_50_intended_diff_only": 0.47700005769729614,
271
+ "tpp_threshold_50_unintended_diff_only": 0.19675001502037048,
272
+ "tpp_threshold_100_total_metric": 0.2265000343322754,
273
+ "tpp_threshold_100_intended_diff_only": 0.4830000400543213,
274
+ "tpp_threshold_100_unintended_diff_only": 0.2565000057220459,
275
+ "tpp_threshold_500_total_metric": 0.14800003170967102,
276
+ "tpp_threshold_500_intended_diff_only": 0.4920000433921814,
277
+ "tpp_threshold_500_unintended_diff_only": 0.3440000116825104
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.0070000141859054565,
281
+ "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
282
  "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224,
283
+ "tpp_threshold_5_total_metric": 0.014500021934509277,
284
  "tpp_threshold_5_intended_diff_only": 0.025000035762786865,
285
+ "tpp_threshold_5_unintended_diff_only": 0.010500013828277588,
286
+ "tpp_threshold_10_total_metric": 0.17375002801418304,
287
+ "tpp_threshold_10_intended_diff_only": 0.22800004482269287,
288
+ "tpp_threshold_10_unintended_diff_only": 0.05425001680850983,
289
+ "tpp_threshold_20_total_metric": 0.22850005328655243,
290
+ "tpp_threshold_20_intended_diff_only": 0.2780000567436218,
291
+ "tpp_threshold_20_unintended_diff_only": 0.0495000034570694,
292
+ "tpp_threshold_50_total_metric": 0.2670000493526459,
293
+ "tpp_threshold_50_intended_diff_only": 0.3840000629425049,
294
+ "tpp_threshold_50_unintended_diff_only": 0.11700001358985901,
295
+ "tpp_threshold_100_total_metric": 0.26000000536441803,
296
+ "tpp_threshold_100_intended_diff_only": 0.43400001525878906,
297
+ "tpp_threshold_100_unintended_diff_only": 0.17400000989437103,
298
+ "tpp_threshold_500_total_metric": 0.14750005304813385,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.32850001752376556
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.002249985933303833,
306
+ "tpp_threshold_2_intended_diff_only": 0.0059999823570251465,
307
+ "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135,
308
+ "tpp_threshold_5_total_metric": 0.0059999823570251465,
309
+ "tpp_threshold_5_intended_diff_only": 0.006999969482421875,
310
+ "tpp_threshold_5_unintended_diff_only": 0.0009999871253967285,
311
+ "tpp_threshold_10_total_metric": 0.005249977111816406,
312
  "tpp_threshold_10_intended_diff_only": 0.006999969482421875,
313
+ "tpp_threshold_10_unintended_diff_only": 0.0017499923706054688,
314
+ "tpp_threshold_20_total_metric": 0.014000028371810913,
315
+ "tpp_threshold_20_intended_diff_only": 0.017000019550323486,
316
+ "tpp_threshold_20_unintended_diff_only": 0.0029999911785125732,
317
+ "tpp_threshold_50_total_metric": 0.03949999809265137,
318
+ "tpp_threshold_50_intended_diff_only": 0.03799998760223389,
319
+ "tpp_threshold_50_unintended_diff_only": -0.0015000104904174805,
320
+ "tpp_threshold_100_total_metric": 0.10574999451637268,
321
+ "tpp_threshold_100_intended_diff_only": 0.1119999885559082,
322
+ "tpp_threshold_100_unintended_diff_only": 0.0062499940395355225,
323
+ "tpp_threshold_500_total_metric": 0.3827500343322754,
324
+ "tpp_threshold_500_intended_diff_only": 0.39500004053115845,
325
+ "tpp_threshold_500_unintended_diff_only": 0.012250006198883057
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.011250004172325134,
329
+ "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
330
+ "tpp_threshold_2_unintended_diff_only": -0.002250000834465027,
331
+ "tpp_threshold_5_total_metric": 0.006500020623207092,
332
+ "tpp_threshold_5_intended_diff_only": 0.013000011444091797,
333
+ "tpp_threshold_5_unintended_diff_only": 0.006499990820884705,
334
+ "tpp_threshold_10_total_metric": 0.041750043630599976,
335
+ "tpp_threshold_10_intended_diff_only": 0.0480000376701355,
336
+ "tpp_threshold_10_unintended_diff_only": 0.0062499940395355225,
337
+ "tpp_threshold_20_total_metric": 0.06575003266334534,
338
+ "tpp_threshold_20_intended_diff_only": 0.0690000057220459,
339
+ "tpp_threshold_20_unintended_diff_only": 0.0032499730587005615,
340
+ "tpp_threshold_50_total_metric": 0.10825006663799286,
341
+ "tpp_threshold_50_intended_diff_only": 0.11600005626678467,
342
+ "tpp_threshold_50_unintended_diff_only": 0.007749989628791809,
343
+ "tpp_threshold_100_total_metric": 0.18625007569789886,
344
+ "tpp_threshold_100_intended_diff_only": 0.20000004768371582,
345
+ "tpp_threshold_100_unintended_diff_only": 0.013749971985816956,
346
+ "tpp_threshold_500_total_metric": 0.40400007367134094,
347
+ "tpp_threshold_500_intended_diff_only": 0.43000006675720215,
348
+ "tpp_threshold_500_unintended_diff_only": 0.025999993085861206
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.004500001668930054,
352
+ "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
353
+ "tpp_threshold_2_unintended_diff_only": -0.0004999935626983643,
354
+ "tpp_threshold_5_total_metric": 0.013000011444091797,
355
+ "tpp_threshold_5_intended_diff_only": 0.008000016212463379,
356
+ "tpp_threshold_5_unintended_diff_only": -0.004999995231628418,
357
+ "tpp_threshold_10_total_metric": 0.015749990940093994,
358
+ "tpp_threshold_10_intended_diff_only": 0.014999985694885254,
359
+ "tpp_threshold_10_unintended_diff_only": -0.0007500052452087402,
360
+ "tpp_threshold_20_total_metric": 0.013750031590461731,
361
+ "tpp_threshold_20_intended_diff_only": 0.017000019550323486,
362
+ "tpp_threshold_20_unintended_diff_only": 0.0032499879598617554,
363
+ "tpp_threshold_50_total_metric": 0.04874996840953827,
364
+ "tpp_threshold_50_intended_diff_only": 0.04799997806549072,
365
+ "tpp_threshold_50_unintended_diff_only": -0.0007499903440475464,
366
+ "tpp_threshold_100_total_metric": 0.11499997973442078,
367
+ "tpp_threshold_100_intended_diff_only": 0.12199997901916504,
368
+ "tpp_threshold_100_unintended_diff_only": 0.006999999284744263,
369
+ "tpp_threshold_500_total_metric": 0.3657500296831131,
370
+ "tpp_threshold_500_intended_diff_only": 0.38600003719329834,
371
+ "tpp_threshold_500_unintended_diff_only": 0.02025000751018524
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.00449998676776886,
375
+ "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
376
+ "tpp_threshold_2_unintended_diff_only": 0.0005000084638595581,
377
+ "tpp_threshold_5_total_metric": 0.010249942541122437,
378
+ "tpp_threshold_5_intended_diff_only": 0.008999943733215332,
379
+ "tpp_threshold_5_unintended_diff_only": -0.0012499988079071045,
380
+ "tpp_threshold_10_total_metric": 0.003999948501586914,
381
+ "tpp_threshold_10_intended_diff_only": 0.008999943733215332,
382
+ "tpp_threshold_10_unintended_diff_only": 0.004999995231628418,
383
+ "tpp_threshold_20_total_metric": 0.01949995756149292,
384
+ "tpp_threshold_20_intended_diff_only": 0.0209999680519104,
385
+ "tpp_threshold_20_unintended_diff_only": 0.0015000104904174805,
386
+ "tpp_threshold_50_total_metric": 0.0887499451637268,
387
+ "tpp_threshold_50_intended_diff_only": 0.09599995613098145,
388
+ "tpp_threshold_50_unintended_diff_only": 0.007250010967254639,
389
+ "tpp_threshold_100_total_metric": 0.16875000298023224,
390
+ "tpp_threshold_100_intended_diff_only": 0.18000000715255737,
391
+ "tpp_threshold_100_unintended_diff_only": 0.011250004172325134,
392
+ "tpp_threshold_500_total_metric": 0.37824998795986176,
393
+ "tpp_threshold_500_intended_diff_only": 0.4039999842643738,
394
+ "tpp_threshold_500_unintended_diff_only": 0.025749996304512024
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.02849997580051422,
398
+ "tpp_threshold_2_intended_diff_only": 0.0339999794960022,
399
+ "tpp_threshold_2_unintended_diff_only": 0.005500003695487976,
400
+ "tpp_threshold_5_total_metric": 0.06325002014636993,
401
+ "tpp_threshold_5_intended_diff_only": 0.06400001049041748,
402
+ "tpp_threshold_5_unintended_diff_only": 0.0007499903440475464,
403
+ "tpp_threshold_10_total_metric": 0.09674996137619019,
404
+ "tpp_threshold_10_intended_diff_only": 0.09999996423721313,
405
+ "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492,
406
+ "tpp_threshold_20_total_metric": 0.13449996709823608,
407
+ "tpp_threshold_20_intended_diff_only": 0.14499998092651367,
408
+ "tpp_threshold_20_unintended_diff_only": 0.010500013828277588,
409
+ "tpp_threshold_50_total_metric": 0.23349997401237488,
410
+ "tpp_threshold_50_intended_diff_only": 0.24299997091293335,
411
+ "tpp_threshold_50_unintended_diff_only": 0.009499996900558472,
412
+ "tpp_threshold_100_total_metric": 0.28849999606609344,
413
+ "tpp_threshold_100_intended_diff_only": 0.3009999990463257,
414
+ "tpp_threshold_100_unintended_diff_only": 0.012500002980232239,
415
+ "tpp_threshold_500_total_metric": 0.34400004148483276,
416
+ "tpp_threshold_500_intended_diff_only": 0.37300002574920654,
417
+ "tpp_threshold_500_unintended_diff_only": 0.02899998426437378
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-146484-tokens-600000000/scr/saebench_scr_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,8 +69,8 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "56c595ad-e2bf-4f48-aade-2c8de2be3566",
69
- "datetime_epoch_millis": 1776884036103,
70
  "eval_result_metrics": {
71
  "scr_metrics": {
72
  "scr_dir1_threshold_2": 0.27738809689751404,
@@ -78,18 +82,18 @@
78
  "scr_dir1_threshold_10": 0.3321123269633328,
79
  "scr_metric_threshold_10": 0.23550300413495637,
80
  "scr_dir2_threshold_10": 0.2263243279951899,
81
- "scr_dir1_threshold_20": null,
82
- "scr_metric_threshold_20": null,
83
- "scr_dir2_threshold_20": null,
84
- "scr_dir1_threshold_50": null,
85
- "scr_metric_threshold_50": null,
86
- "scr_dir2_threshold_50": null,
87
- "scr_dir1_threshold_100": null,
88
- "scr_metric_threshold_100": null,
89
- "scr_dir2_threshold_100": null,
90
- "scr_dir1_threshold_500": null,
91
- "scr_metric_threshold_500": null,
92
- "scr_dir2_threshold_500": null
93
  }
94
  },
95
  "eval_result_details": [
@@ -104,18 +108,18 @@
104
  "scr_dir1_threshold_10": 0.5769232238773122,
105
  "scr_metric_threshold_10": 0.06217613139950766,
106
  "scr_dir2_threshold_10": 0.06217613139950766,
107
- "scr_dir1_threshold_20": null,
108
- "scr_metric_threshold_20": null,
109
- "scr_dir2_threshold_20": null,
110
- "scr_dir1_threshold_50": null,
111
- "scr_metric_threshold_50": null,
112
- "scr_dir2_threshold_50": null,
113
- "scr_dir1_threshold_100": null,
114
- "scr_metric_threshold_100": null,
115
- "scr_dir2_threshold_100": null,
116
- "scr_dir1_threshold_500": null,
117
- "scr_metric_threshold_500": null,
118
- "scr_dir2_threshold_500": null
119
  },
120
  {
121
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
@@ -128,18 +132,18 @@
128
  "scr_dir1_threshold_10": 0.42424228740901826,
129
  "scr_metric_threshold_10": 0.34862389836223945,
130
  "scr_dir2_threshold_10": 0.34862389836223945,
131
- "scr_dir1_threshold_20": null,
132
- "scr_metric_threshold_20": null,
133
- "scr_dir2_threshold_20": null,
134
- "scr_dir1_threshold_50": null,
135
- "scr_metric_threshold_50": null,
136
- "scr_dir2_threshold_50": null,
137
- "scr_dir1_threshold_100": null,
138
- "scr_metric_threshold_100": null,
139
- "scr_dir2_threshold_100": null,
140
- "scr_dir1_threshold_500": null,
141
- "scr_metric_threshold_500": null,
142
- "scr_dir2_threshold_500": null
143
  },
144
  {
145
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
@@ -152,18 +156,18 @@
152
  "scr_dir1_threshold_10": 0.3749997671693945,
153
  "scr_metric_threshold_10": 0.10050255094361776,
154
  "scr_dir2_threshold_10": 0.10050255094361776,
155
- "scr_dir1_threshold_20": null,
156
- "scr_metric_threshold_20": null,
157
- "scr_dir2_threshold_20": null,
158
- "scr_dir1_threshold_50": null,
159
- "scr_metric_threshold_50": null,
160
- "scr_dir2_threshold_50": null,
161
- "scr_dir1_threshold_100": null,
162
- "scr_metric_threshold_100": null,
163
- "scr_dir2_threshold_100": null,
164
- "scr_dir1_threshold_500": null,
165
- "scr_metric_threshold_500": null,
166
- "scr_dir2_threshold_500": null
167
  },
168
  {
169
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
@@ -176,18 +180,18 @@
176
  "scr_dir1_threshold_10": 0.2617448160956555,
177
  "scr_metric_threshold_10": 0.1687899080607876,
178
  "scr_dir2_threshold_10": 0.1687899080607876,
179
- "scr_dir1_threshold_20": null,
180
- "scr_metric_threshold_20": null,
181
- "scr_dir2_threshold_20": null,
182
- "scr_dir1_threshold_50": null,
183
- "scr_metric_threshold_50": null,
184
- "scr_dir2_threshold_50": null,
185
- "scr_dir1_threshold_100": null,
186
- "scr_metric_threshold_100": null,
187
- "scr_dir2_threshold_100": null,
188
- "scr_dir1_threshold_500": null,
189
- "scr_metric_threshold_500": null,
190
- "scr_dir2_threshold_500": null
191
  },
192
  {
193
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
@@ -200,18 +204,18 @@
200
  "scr_dir1_threshold_10": 0.07368420392192293,
201
  "scr_metric_threshold_10": 0.4365079346307434,
202
  "scr_dir2_threshold_10": 0.4365079346307434,
203
- "scr_dir1_threshold_20": null,
204
- "scr_metric_threshold_20": null,
205
- "scr_dir2_threshold_20": null,
206
- "scr_dir1_threshold_50": null,
207
- "scr_metric_threshold_50": null,
208
- "scr_dir2_threshold_50": null,
209
- "scr_dir1_threshold_100": null,
210
- "scr_metric_threshold_100": null,
211
- "scr_dir2_threshold_100": null,
212
- "scr_dir1_threshold_500": null,
213
- "scr_metric_threshold_500": null,
214
- "scr_dir2_threshold_500": null
215
  },
216
  {
217
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
@@ -224,18 +228,18 @@
224
  "scr_dir1_threshold_10": 0.3825136878996036,
225
  "scr_metric_threshold_10": 0.13358787395939287,
226
  "scr_dir2_threshold_10": 0.13358787395939287,
227
- "scr_dir1_threshold_20": null,
228
- "scr_metric_threshold_20": null,
229
- "scr_dir2_threshold_20": null,
230
- "scr_dir1_threshold_50": null,
231
- "scr_metric_threshold_50": null,
232
- "scr_dir2_threshold_50": null,
233
- "scr_dir1_threshold_100": null,
234
- "scr_metric_threshold_100": null,
235
- "scr_dir2_threshold_100": null,
236
- "scr_dir1_threshold_500": null,
237
- "scr_metric_threshold_500": null,
238
- "scr_dir2_threshold_500": null
239
  },
240
  {
241
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
@@ -248,18 +252,18 @@
248
  "scr_dir1_threshold_10": 0.3581395813037724,
249
  "scr_metric_threshold_10": 0.42918468769337936,
250
  "scr_dir2_threshold_10": 0.42918468769337936,
251
- "scr_dir1_threshold_20": null,
252
- "scr_metric_threshold_20": null,
253
- "scr_dir2_threshold_20": null,
254
- "scr_dir1_threshold_50": null,
255
- "scr_metric_threshold_50": null,
256
- "scr_dir2_threshold_50": null,
257
- "scr_dir1_threshold_100": null,
258
- "scr_metric_threshold_100": null,
259
- "scr_dir2_threshold_100": null,
260
- "scr_dir1_threshold_500": null,
261
- "scr_metric_threshold_500": null,
262
- "scr_dir2_threshold_500": null
263
  },
264
  {
265
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
@@ -272,18 +276,18 @@
272
  "scr_dir1_threshold_10": 0.20465104802998307,
273
  "scr_metric_threshold_10": 0.20465104802998307,
274
  "scr_dir2_threshold_10": 0.1312216389118512,
275
- "scr_dir1_threshold_20": null,
276
- "scr_metric_threshold_20": null,
277
- "scr_dir2_threshold_20": null,
278
- "scr_dir1_threshold_50": null,
279
- "scr_metric_threshold_50": null,
280
- "scr_dir2_threshold_50": null,
281
- "scr_dir1_threshold_100": null,
282
- "scr_metric_threshold_100": null,
283
- "scr_dir2_threshold_100": null,
284
- "scr_dir1_threshold_500": null,
285
- "scr_metric_threshold_500": null,
286
- "scr_dir2_threshold_500": null
287
  }
288
  ],
289
  "sae_bench_commit_hash": "Unknown",
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "e46c9f21-46ba-403d-b877-38fe589d355a",
73
+ "datetime_epoch_millis": 1777315968594,
74
  "eval_result_metrics": {
75
  "scr_metrics": {
76
  "scr_dir1_threshold_2": 0.27738809689751404,
 
82
  "scr_dir1_threshold_10": 0.3321123269633328,
83
  "scr_metric_threshold_10": 0.23550300413495637,
84
  "scr_dir2_threshold_10": 0.2263243279951899,
85
+ "scr_dir1_threshold_20": 0.2469655041244911,
86
+ "scr_metric_threshold_20": 0.2674926661046679,
87
+ "scr_dir2_threshold_20": 0.25760632497206615,
88
+ "scr_dir1_threshold_50": 0.18479817640625043,
89
+ "scr_metric_threshold_50": 0.3553871309435371,
90
+ "scr_dir2_threshold_50": 0.3500099011394471,
91
+ "scr_dir1_threshold_100": 0.10319157981943705,
92
+ "scr_metric_threshold_100": 0.39137358639136266,
93
+ "scr_dir2_threshold_100": 0.3857753722056411,
94
+ "scr_dir1_threshold_500": -0.5537493792316611,
95
+ "scr_metric_threshold_500": 0.15692243971978725,
96
+ "scr_dir2_threshold_500": 0.12993883713028137
97
  }
98
  },
99
  "eval_result_details": [
 
108
  "scr_dir1_threshold_10": 0.5769232238773122,
109
  "scr_metric_threshold_10": 0.06217613139950766,
110
  "scr_dir2_threshold_10": 0.06217613139950766,
111
+ "scr_dir1_threshold_20": 0.43589788655709094,
112
+ "scr_metric_threshold_20": 0.1398964114610147,
113
+ "scr_dir2_threshold_20": 0.1398964114610147,
114
+ "scr_dir1_threshold_50": 0.43589788655709094,
115
+ "scr_metric_threshold_50": 0.2124352571631342,
116
+ "scr_dir2_threshold_50": 0.2124352571631342,
117
+ "scr_dir1_threshold_100": 0.4615385791018498,
118
+ "scr_metric_threshold_100": 0.32383416568792556,
119
+ "scr_dir2_threshold_100": 0.32383416568792556,
120
+ "scr_dir1_threshold_500": -1.794871402993834,
121
+ "scr_metric_threshold_500": 0.020725428605223653,
122
+ "scr_dir2_threshold_500": 0.020725428605223653
123
  },
124
  {
125
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
 
132
  "scr_dir1_threshold_10": 0.42424228740901826,
133
  "scr_metric_threshold_10": 0.34862389836223945,
134
  "scr_dir2_threshold_10": 0.34862389836223945,
135
+ "scr_dir1_threshold_20": 0.16666674192503994,
136
+ "scr_metric_threshold_20": 0.23853212179761762,
137
+ "scr_dir2_threshold_20": 0.23853212179761762,
138
+ "scr_dir1_threshold_50": 0.5681820542194435,
139
+ "scr_metric_threshold_50": 0.5840979557613263,
140
+ "scr_dir2_threshold_50": 0.5840979557613263,
141
+ "scr_dir1_threshold_100": 0.26515165540575625,
142
+ "scr_metric_threshold_100": 0.5321101318775462,
143
+ "scr_dir2_threshold_100": 0.5321101318775462,
144
+ "scr_dir1_threshold_500": -0.7499996613373202,
145
+ "scr_metric_threshold_500": -0.23241581072336703,
146
+ "scr_dir2_threshold_500": -0.23241581072336703
147
  },
148
  {
149
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
 
156
  "scr_dir1_threshold_10": 0.3749997671693945,
157
  "scr_metric_threshold_10": 0.10050255094361776,
158
  "scr_dir2_threshold_10": 0.10050255094361776,
159
+ "scr_dir1_threshold_20": -0.015624796273220196,
160
+ "scr_metric_threshold_20": 0.15577896145062706,
161
+ "scr_dir2_threshold_20": 0.15577896145062706,
162
+ "scr_dir1_threshold_50": -0.8124996507540918,
163
+ "scr_metric_threshold_50": 0.18341709182393628,
164
+ "scr_dir2_threshold_50": 0.18341709182393628,
165
+ "scr_dir1_threshold_100": -0.5312495925464404,
166
+ "scr_metric_threshold_100": 0.13819095138732784,
167
+ "scr_dir2_threshold_100": 0.13819095138732784,
168
+ "scr_dir1_threshold_500": -1.8593749708961742,
169
+ "scr_metric_threshold_500": -0.025125600295806724,
170
+ "scr_dir2_threshold_500": -0.025125600295806724
171
  },
172
  {
173
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
 
180
  "scr_dir1_threshold_10": 0.2617448160956555,
181
  "scr_metric_threshold_10": 0.1687899080607876,
182
  "scr_dir2_threshold_10": 0.1687899080607876,
183
+ "scr_dir1_threshold_20": 0.22818779864829455,
184
+ "scr_metric_threshold_20": 0.18152878695440397,
185
+ "scr_dir2_threshold_20": 0.18152878695440397,
186
+ "scr_dir1_threshold_50": 0.2617448160956555,
187
+ "scr_metric_threshold_50": 0.3407642985653504,
188
+ "scr_dir2_threshold_50": 0.3407642985653504,
189
+ "scr_dir1_threshold_100": -0.5838927436340035,
190
+ "scr_metric_threshold_100": 0.47452243203647043,
191
+ "scr_dir2_threshold_100": 0.47452243203647043,
192
+ "scr_dir1_threshold_500": -1.5234904322537157,
193
+ "scr_metric_threshold_500": -0.5127388788936164,
194
+ "scr_dir2_threshold_500": -0.5127388788936164
195
  },
196
  {
197
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
 
204
  "scr_dir1_threshold_10": 0.07368420392192293,
205
  "scr_metric_threshold_10": 0.4365079346307434,
206
  "scr_dir2_threshold_10": 0.4365079346307434,
207
+ "scr_dir1_threshold_20": 0.11052646273721473,
208
+ "scr_metric_threshold_20": 0.5238095688621593,
209
+ "scr_dir2_threshold_20": 0.5238095688621593,
210
+ "scr_dir1_threshold_50": -0.19999993725826787,
211
+ "scr_metric_threshold_50": 0.6111109665672388,
212
+ "scr_dir2_threshold_50": 0.6111109665672388,
213
+ "scr_dir1_threshold_100": -0.184210352950477,
214
+ "scr_metric_threshold_100": 0.6468252607288938,
215
+ "scr_dir2_threshold_100": 0.6468252607288938,
216
+ "scr_dir1_threshold_500": -0.15789458824326602,
217
+ "scr_metric_threshold_500": 0.7023808031440972,
218
+ "scr_dir2_threshold_500": 0.7023808031440972
219
  },
220
  {
221
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
 
228
  "scr_dir1_threshold_10": 0.3825136878996036,
229
  "scr_metric_threshold_10": 0.13358787395939287,
230
  "scr_dir2_threshold_10": 0.13358787395939287,
231
+ "scr_dir1_threshold_20": 0.3989072479911928,
232
+ "scr_metric_threshold_20": 0.19465646684173443,
233
+ "scr_dir2_threshold_20": 0.19465646684173443,
234
+ "scr_dir1_threshold_50": 0.4808743970321442,
235
+ "scr_metric_threshold_50": 0.24809157092576697,
236
+ "scr_dir2_threshold_50": 0.24809157092576697,
237
+ "scr_dir1_threshold_100": 0.5191256029678558,
238
+ "scr_metric_threshold_100": 0.2786259811162493,
239
+ "scr_dir2_threshold_100": 0.2786259811162493,
240
+ "scr_dir1_threshold_500": 0.5300547515984144,
241
+ "scr_metric_threshold_500": 0.36641223978991866,
242
+ "scr_dir2_threshold_500": 0.36641223978991866
243
  },
244
  {
245
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
 
252
  "scr_dir1_threshold_10": 0.3581395813037724,
253
  "scr_metric_threshold_10": 0.42918468769337936,
254
  "scr_dir2_threshold_10": 0.42918468769337936,
255
+ "scr_dir1_threshold_20": 0.4046511576329142,
256
+ "scr_metric_threshold_20": 0.45922747769238426,
257
+ "scr_dir2_threshold_20": 0.45922747769238426,
258
+ "scr_dir1_threshold_50": 0.49302320810445616,
259
+ "scr_metric_threshold_50": 0.41201726948797046,
260
+ "scr_dir2_threshold_50": 0.41201726948797046,
261
+ "scr_dir1_threshold_100": 0.5627905725981688,
262
+ "scr_metric_threshold_100": 0.4206008506837011,
263
+ "scr_dir2_threshold_100": 0.4206008506837011,
264
+ "scr_dir1_threshold_500": 0.6744184112342817,
265
+ "scr_metric_threshold_500": 0.48497847709352376,
266
+ "scr_dir2_threshold_500": 0.48497847709352376
267
  },
268
  {
269
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
 
276
  "scr_dir1_threshold_10": 0.20465104802998307,
277
  "scr_metric_threshold_10": 0.20465104802998307,
278
  "scr_dir2_threshold_10": 0.1312216389118512,
279
+ "scr_dir1_threshold_20": 0.24651153377740184,
280
+ "scr_metric_threshold_20": 0.24651153377740184,
281
+ "scr_dir2_threshold_20": 0.16742080471658802,
282
+ "scr_dir1_threshold_50": 0.2511626372535729,
283
+ "scr_metric_threshold_50": 0.2511626372535729,
284
+ "scr_dir2_threshold_50": 0.208144798820853,
285
+ "scr_dir1_threshold_100": 0.31627891761278654,
286
+ "scr_metric_threshold_100": 0.31627891761278654,
287
+ "scr_dir2_threshold_100": 0.2714932041270145,
288
+ "scr_dir1_threshold_500": 0.4511628590383247,
289
+ "scr_metric_threshold_500": 0.4511628590383247,
290
+ "scr_dir2_threshold_500": 0.23529403832227772
291
  }
292
  ],
293
  "sae_bench_commit_hash": "Unknown",
gemma-2-2b-btk/k-100/seed-0/snapshots/step-146484-tokens-600000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "cb259007-e24c-44ba-8006-1acf43c62162",
69
- "datetime_epoch_millis": 1776884205111,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.027574996650218966,
73
- "tpp_threshold_2_intended_diff_only": 0.02940000295639038,
74
- "tpp_threshold_2_unintended_diff_only": 0.001825006306171417,
75
- "tpp_threshold_5_total_metric": 0.05815000534057617,
76
- "tpp_threshold_5_intended_diff_only": 0.06840000748634338,
77
- "tpp_threshold_5_unintended_diff_only": 0.010250002145767212,
78
- "tpp_threshold_10_total_metric": 0.11262500584125519,
79
- "tpp_threshold_10_intended_diff_only": 0.14600000977516173,
80
- "tpp_threshold_10_unintended_diff_only": 0.03337500393390656,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.044399994611740115,
99
- "tpp_threshold_2_intended_diff_only": 0.04780000448226929,
100
- "tpp_threshold_2_unintended_diff_only": 0.0034000098705291746,
101
- "tpp_threshold_5_total_metric": 0.09930000901222229,
102
- "tpp_threshold_5_intended_diff_only": 0.11780002117156982,
103
- "tpp_threshold_5_unintended_diff_only": 0.018500012159347535,
104
- "tpp_threshold_10_total_metric": 0.19675001502037048,
105
- "tpp_threshold_10_intended_diff_only": 0.2610000252723694,
106
- "tpp_threshold_10_unintended_diff_only": 0.0642500102519989,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.010749998688697814,
123
- "tpp_threshold_2_intended_diff_only": 0.011000001430511474,
124
- "tpp_threshold_2_unintended_diff_only": 0.0002500027418136597,
125
- "tpp_threshold_5_total_metric": 0.017000001668930054,
126
- "tpp_threshold_5_intended_diff_only": 0.018999993801116943,
127
- "tpp_threshold_5_unintended_diff_only": 0.00199999213218689,
128
- "tpp_threshold_10_total_metric": 0.02849999666213989,
129
- "tpp_threshold_10_intended_diff_only": 0.0309999942779541,
130
- "tpp_threshold_10_unintended_diff_only": 0.002499997615814209,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.011250019073486328,
190
  "tpp_threshold_10_total_metric": 0.1004999577999115,
191
  "tpp_threshold_10_intended_diff_only": 0.11299997568130493,
192
- "tpp_threshold_10_unintended_diff_only": 0.012500017881393433
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": 0.0017500072717666626,
196
- "tpp_threshold_2_intended_diff_only": 0.003000020980834961,
197
- "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983,
198
- "tpp_threshold_5_total_metric": 0.15300002694129944,
199
  "tpp_threshold_5_intended_diff_only": 0.17800003290176392,
200
- "tpp_threshold_5_unintended_diff_only": 0.025000005960464478,
201
- "tpp_threshold_10_total_metric": 0.20575003325939178,
202
- "tpp_threshold_10_intended_diff_only": 0.26600003242492676,
203
- "tpp_threshold_10_unintended_diff_only": 0.06024999916553497
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.1367499977350235,
207
- "tpp_threshold_2_intended_diff_only": 0.1470000147819519,
208
- "tpp_threshold_2_unintended_diff_only": 0.010250017046928406,
209
- "tpp_threshold_5_total_metric": 0.22850003838539124,
210
- "tpp_threshold_5_intended_diff_only": 0.2690000534057617,
211
- "tpp_threshold_5_unintended_diff_only": 0.04050001502037048,
212
- "tpp_threshold_10_total_metric": 0.21875004470348358,
213
- "tpp_threshold_10_intended_diff_only": 0.3340000510215759,
214
- "tpp_threshold_10_unintended_diff_only": 0.11525000631809235
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.011750012636184692,
218
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
219
- "tpp_threshold_2_unintended_diff_only": -0.002750009298324585,
220
- "tpp_threshold_5_total_metric": 0.005500003695487976,
221
- "tpp_threshold_5_intended_diff_only": 0.013000011444091797,
222
- "tpp_threshold_5_unintended_diff_only": 0.007500007748603821,
223
- "tpp_threshold_10_total_metric": 0.2847500443458557,
224
  "tpp_threshold_10_intended_diff_only": 0.362000048160553,
225
- "tpp_threshold_10_unintended_diff_only": 0.07725000381469727
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.003999993205070496,
229
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
230
- "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224,
231
- "tpp_threshold_5_total_metric": 0.02275000512599945,
232
- "tpp_threshold_5_intended_diff_only": 0.03100001811981201,
233
- "tpp_threshold_5_unintended_diff_only": 0.008250012993812561,
234
- "tpp_threshold_10_total_metric": 0.17399999499320984,
235
- "tpp_threshold_10_intended_diff_only": 0.23000001907348633,
236
- "tpp_threshold_10_unintended_diff_only": 0.05600002408027649
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.009249985218048096,
242
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
243
- "tpp_threshold_2_unintended_diff_only": -0.004249989986419678,
244
- "tpp_threshold_5_total_metric": 0.012250006198883057,
245
- "tpp_threshold_5_intended_diff_only": 0.009999990463256836,
246
- "tpp_threshold_5_unintended_diff_only": -0.0022500157356262207,
247
- "tpp_threshold_10_total_metric": 0.003250017762184143,
248
- "tpp_threshold_10_intended_diff_only": 0.008000016212463379,
249
- "tpp_threshold_10_unintended_diff_only": 0.004749998450279236
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": 0.002750024199485779,
253
- "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
254
- "tpp_threshold_2_unintended_diff_only": 0.0052499920129776,
255
- "tpp_threshold_5_total_metric": 0.012000024318695068,
256
- "tpp_threshold_5_intended_diff_only": 0.018000006675720215,
257
- "tpp_threshold_5_unintended_diff_only": 0.0059999823570251465,
258
- "tpp_threshold_10_total_metric": 0.04950001835823059,
259
- "tpp_threshold_10_intended_diff_only": 0.050999999046325684,
260
- "tpp_threshold_10_unintended_diff_only": 0.0014999806880950928
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.013500019907951355,
264
- "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
265
- "tpp_threshold_2_unintended_diff_only": -0.006499990820884705,
266
- "tpp_threshold_5_total_metric": 0.01900002360343933,
267
- "tpp_threshold_5_intended_diff_only": 0.017000019550323486,
268
- "tpp_threshold_5_unintended_diff_only": -0.0020000040531158447,
269
- "tpp_threshold_10_total_metric": 0.017250001430511475,
270
- "tpp_threshold_10_intended_diff_only": 0.018999993801116943,
271
- "tpp_threshold_10_unintended_diff_only": 0.0017499923706054688
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.006000012159347534,
275
- "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
276
- "tpp_threshold_2_unintended_diff_only": 0.006999999284744263,
277
- "tpp_threshold_5_total_metric": 0.0022499561309814453,
278
- "tpp_threshold_5_intended_diff_only": 0.003999948501586914,
279
- "tpp_threshold_5_unintended_diff_only": 0.0017499923706054688,
280
- "tpp_threshold_10_total_metric": 0.004499942064285278,
281
- "tpp_threshold_10_intended_diff_only": 0.007999956607818604,
282
- "tpp_threshold_10_unintended_diff_only": 0.003500014543533325
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.03424997627735138,
286
- "tpp_threshold_2_intended_diff_only": 0.0339999794960022,
287
- "tpp_threshold_2_unintended_diff_only": -0.00024999678134918213,
288
- "tpp_threshold_5_total_metric": 0.03949999809265137,
289
- "tpp_threshold_5_intended_diff_only": 0.046000003814697266,
290
- "tpp_threshold_5_unintended_diff_only": 0.0065000057220458984,
291
- "tpp_threshold_10_total_metric": 0.06800000369548798,
292
- "tpp_threshold_10_intended_diff_only": 0.0690000057220459,
293
- "tpp_threshold_10_unintended_diff_only": 0.0010000020265579224
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "482df7fa-eeb3-40f9-bb54-02dce228797d",
73
+ "datetime_epoch_millis": 1777316251617,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.02634999305009842,
77
+ "tpp_threshold_2_intended_diff_only": 0.029500001668930055,
78
+ "tpp_threshold_2_unintended_diff_only": 0.0031500086188316345,
79
+ "tpp_threshold_5_total_metric": 0.058674997091293334,
80
+ "tpp_threshold_5_intended_diff_only": 0.06740000247955322,
81
+ "tpp_threshold_5_unintended_diff_only": 0.008725005388259889,
82
+ "tpp_threshold_10_total_metric": 0.11385000199079515,
83
+ "tpp_threshold_10_intended_diff_only": 0.14680001139640808,
84
+ "tpp_threshold_10_unintended_diff_only": 0.03295000940561294,
85
+ "tpp_threshold_20_total_metric": 0.14422501027584075,
86
+ "tpp_threshold_20_intended_diff_only": 0.19700001478195192,
87
+ "tpp_threshold_20_unintended_diff_only": 0.052775004506111146,
88
+ "tpp_threshold_50_total_metric": 0.17449999749660491,
89
+ "tpp_threshold_50_intended_diff_only": 0.262800008058548,
90
+ "tpp_threshold_50_unintended_diff_only": 0.08830001056194305,
91
+ "tpp_threshold_100_total_metric": 0.1912250116467476,
92
+ "tpp_threshold_100_intended_diff_only": 0.3198000192642212,
93
+ "tpp_threshold_100_unintended_diff_only": 0.1285750076174736,
94
+ "tpp_threshold_500_total_metric": 0.24270001649856565,
95
+ "tpp_threshold_500_intended_diff_only": 0.4316000282764435,
96
+ "tpp_threshold_500_unintended_diff_only": 0.18890001177787782
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.043049997091293334,
103
+ "tpp_threshold_2_intended_diff_only": 0.048200011253356934,
104
+ "tpp_threshold_2_unintended_diff_only": 0.005150014162063598,
105
+ "tpp_threshold_5_total_metric": 0.10069999992847442,
106
+ "tpp_threshold_5_intended_diff_only": 0.11760001182556153,
107
+ "tpp_threshold_5_unintended_diff_only": 0.0169000118970871,
108
+ "tpp_threshold_10_total_metric": 0.19780000746250154,
109
+ "tpp_threshold_10_intended_diff_only": 0.26080002784729006,
110
+ "tpp_threshold_10_unintended_diff_only": 0.06300002038478851,
111
+ "tpp_threshold_20_total_metric": 0.23650000393390655,
112
+ "tpp_threshold_20_intended_diff_only": 0.3376000165939331,
113
+ "tpp_threshold_20_unintended_diff_only": 0.10110001266002655,
114
+ "tpp_threshold_50_total_metric": 0.23419999480247497,
115
+ "tpp_threshold_50_intended_diff_only": 0.4052000164985657,
116
+ "tpp_threshold_50_unintended_diff_only": 0.1710000216960907,
117
+ "tpp_threshold_100_total_metric": 0.19340000450611114,
118
+ "tpp_threshold_100_intended_diff_only": 0.4388000249862671,
119
+ "tpp_threshold_100_unintended_diff_only": 0.24540002048015594,
120
+ "tpp_threshold_500_total_metric": 0.11355001628398895,
121
+ "tpp_threshold_500_intended_diff_only": 0.4670000433921814,
122
+ "tpp_threshold_500_unintended_diff_only": 0.35345002710819245
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.009649989008903504,
127
+ "tpp_threshold_2_intended_diff_only": 0.010799992084503173,
128
+ "tpp_threshold_2_unintended_diff_only": 0.0011500030755996703,
129
+ "tpp_threshold_5_total_metric": 0.016649994254112243,
130
+ "tpp_threshold_5_intended_diff_only": 0.017199993133544922,
131
+ "tpp_threshold_5_unintended_diff_only": 0.0005499988794326783,
132
+ "tpp_threshold_10_total_metric": 0.029899996519088746,
133
+ "tpp_threshold_10_intended_diff_only": 0.03279999494552612,
134
+ "tpp_threshold_10_unintended_diff_only": 0.002899998426437378,
135
+ "tpp_threshold_20_total_metric": 0.051950016617774965,
136
+ "tpp_threshold_20_intended_diff_only": 0.0564000129699707,
137
+ "tpp_threshold_20_unintended_diff_only": 0.00444999635219574,
138
+ "tpp_threshold_50_total_metric": 0.11480000019073486,
139
+ "tpp_threshold_50_intended_diff_only": 0.12039999961853028,
140
+ "tpp_threshold_50_unintended_diff_only": 0.00559999942779541,
141
+ "tpp_threshold_100_total_metric": 0.18905001878738403,
142
+ "tpp_threshold_100_intended_diff_only": 0.2008000135421753,
143
+ "tpp_threshold_100_unintended_diff_only": 0.01174999475479126,
144
+ "tpp_threshold_500_total_metric": 0.3718500167131424,
145
+ "tpp_threshold_500_intended_diff_only": 0.3962000131607056,
146
+ "tpp_threshold_500_unintended_diff_only": 0.024349996447563173
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.011250019073486328,
194
  "tpp_threshold_10_total_metric": 0.1004999577999115,
195
  "tpp_threshold_10_intended_diff_only": 0.11299997568130493,
196
+ "tpp_threshold_10_unintended_diff_only": 0.012500017881393433,
197
+ "tpp_threshold_20_total_metric": 0.205999955534935,
198
+ "tpp_threshold_20_intended_diff_only": 0.3259999752044678,
199
+ "tpp_threshold_20_unintended_diff_only": 0.12000001966953278,
200
+ "tpp_threshold_50_total_metric": 0.23149995505809784,
201
+ "tpp_threshold_50_intended_diff_only": 0.39499998092651367,
202
+ "tpp_threshold_50_unintended_diff_only": 0.16350002586841583,
203
+ "tpp_threshold_100_total_metric": 0.18024997413158417,
204
+ "tpp_threshold_100_intended_diff_only": 0.4390000104904175,
205
+ "tpp_threshold_100_unintended_diff_only": 0.2587500363588333,
206
+ "tpp_threshold_500_total_metric": 0.11225000023841858,
207
+ "tpp_threshold_500_intended_diff_only": 0.45600003004074097,
208
+ "tpp_threshold_500_unintended_diff_only": 0.3437500298023224
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.0054999589920043945,
212
+ "tpp_threshold_2_intended_diff_only": 0.0059999823570251465,
213
+ "tpp_threshold_2_unintended_diff_only": 0.000500023365020752,
214
+ "tpp_threshold_5_total_metric": 0.15725000202655792,
215
  "tpp_threshold_5_intended_diff_only": 0.17800003290176392,
216
+ "tpp_threshold_5_unintended_diff_only": 0.020750030875205994,
217
+ "tpp_threshold_10_total_metric": 0.20500002801418304,
218
+ "tpp_threshold_10_intended_diff_only": 0.26500004529953003,
219
+ "tpp_threshold_10_unintended_diff_only": 0.060000017285346985,
220
+ "tpp_threshold_20_total_metric": 0.2487499862909317,
221
+ "tpp_threshold_20_intended_diff_only": 0.3140000104904175,
222
+ "tpp_threshold_20_unintended_diff_only": 0.06525002419948578,
223
+ "tpp_threshold_50_total_metric": 0.25425001978874207,
224
+ "tpp_threshold_50_intended_diff_only": 0.3850000500679016,
225
+ "tpp_threshold_50_unintended_diff_only": 0.13075003027915955,
226
+ "tpp_threshold_100_total_metric": 0.22874997556209564,
227
+ "tpp_threshold_100_intended_diff_only": 0.4259999990463257,
228
+ "tpp_threshold_100_unintended_diff_only": 0.19725002348423004,
229
+ "tpp_threshold_500_total_metric": 0.10125000774860382,
230
+ "tpp_threshold_500_intended_diff_only": 0.4590000510215759,
231
+ "tpp_threshold_500_unintended_diff_only": 0.3577500432729721
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.12625004351139069,
235
+ "tpp_threshold_2_intended_diff_only": 0.14400005340576172,
236
+ "tpp_threshold_2_unintended_diff_only": 0.017750009894371033,
237
+ "tpp_threshold_5_total_metric": 0.22750000655651093,
238
+ "tpp_threshold_5_intended_diff_only": 0.2680000066757202,
239
+ "tpp_threshold_5_unintended_diff_only": 0.04050000011920929,
240
+ "tpp_threshold_10_total_metric": 0.22300001978874207,
241
+ "tpp_threshold_10_intended_diff_only": 0.33500003814697266,
242
+ "tpp_threshold_10_unintended_diff_only": 0.11200001835823059,
243
+ "tpp_threshold_20_total_metric": 0.1977500468492508,
244
+ "tpp_threshold_20_intended_diff_only": 0.36100006103515625,
245
+ "tpp_threshold_20_unintended_diff_only": 0.16325001418590546,
246
+ "tpp_threshold_50_total_metric": 0.14774999022483826,
247
+ "tpp_threshold_50_intended_diff_only": 0.3830000162124634,
248
+ "tpp_threshold_50_unintended_diff_only": 0.23525002598762512,
249
+ "tpp_threshold_100_total_metric": 0.11075001955032349,
250
+ "tpp_threshold_100_intended_diff_only": 0.4140000343322754,
251
+ "tpp_threshold_100_unintended_diff_only": 0.3032500147819519,
252
+ "tpp_threshold_500_total_metric": 0.05900000035762787,
253
+ "tpp_threshold_500_intended_diff_only": 0.4520000219345093,
254
+ "tpp_threshold_500_unintended_diff_only": 0.3930000215768814
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.009499981999397278,
258
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704,
260
+ "tpp_threshold_5_total_metric": 0.013499990105628967,
261
+ "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
262
+ "tpp_threshold_5_unintended_diff_only": 0.0005000084638595581,
263
+ "tpp_threshold_10_total_metric": 0.28725002706050873,
264
  "tpp_threshold_10_intended_diff_only": 0.362000048160553,
265
+ "tpp_threshold_10_unintended_diff_only": 0.07475002110004425,
266
+ "tpp_threshold_20_total_metric": 0.30925001204013824,
267
+ "tpp_threshold_20_intended_diff_only": 0.41600000858306885,
268
+ "tpp_threshold_20_unintended_diff_only": 0.1067499965429306,
269
+ "tpp_threshold_50_total_metric": 0.27524998784065247,
270
+ "tpp_threshold_50_intended_diff_only": 0.48000001907348633,
271
+ "tpp_threshold_50_unintended_diff_only": 0.20475003123283386,
272
+ "tpp_threshold_100_total_metric": 0.21175001561641693,
273
+ "tpp_threshold_100_intended_diff_only": 0.4830000400543213,
274
+ "tpp_threshold_100_unintended_diff_only": 0.27125002443790436,
275
+ "tpp_threshold_500_total_metric": 0.14725002646446228,
276
+ "tpp_threshold_500_intended_diff_only": 0.4920000433921814,
277
+ "tpp_threshold_500_unintended_diff_only": 0.3447500169277191
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.006250038743019104,
281
+ "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464,
283
+ "tpp_threshold_5_total_metric": 0.018500030040740967,
284
+ "tpp_threshold_5_intended_diff_only": 0.030000030994415283,
285
+ "tpp_threshold_5_unintended_diff_only": 0.011500000953674316,
286
+ "tpp_threshold_10_total_metric": 0.1732500046491623,
287
+ "tpp_threshold_10_intended_diff_only": 0.2290000319480896,
288
+ "tpp_threshold_10_unintended_diff_only": 0.05575002729892731,
289
+ "tpp_threshold_20_total_metric": 0.22075001895427704,
290
+ "tpp_threshold_20_intended_diff_only": 0.2710000276565552,
291
+ "tpp_threshold_20_unintended_diff_only": 0.05025000870227814,
292
+ "tpp_threshold_50_total_metric": 0.26225002110004425,
293
+ "tpp_threshold_50_intended_diff_only": 0.3830000162124634,
294
+ "tpp_threshold_50_unintended_diff_only": 0.12074999511241913,
295
+ "tpp_threshold_100_total_metric": 0.2355000376701355,
296
+ "tpp_threshold_100_intended_diff_only": 0.4320000410079956,
297
+ "tpp_threshold_100_unintended_diff_only": 0.1965000033378601,
298
+ "tpp_threshold_500_total_metric": 0.14800004661083221,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.3280000239610672
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.0029999762773513794,
306
+ "tpp_threshold_2_intended_diff_only": 0.006999969482421875,
307
+ "tpp_threshold_2_unintended_diff_only": 0.003999993205070496,
308
+ "tpp_threshold_5_total_metric": 0.007250010967254639,
309
+ "tpp_threshold_5_intended_diff_only": 0.009000003337860107,
310
+ "tpp_threshold_5_unintended_diff_only": 0.0017499923706054688,
311
+ "tpp_threshold_10_total_metric": 0.004249975085258484,
312
+ "tpp_threshold_10_intended_diff_only": 0.006999969482421875,
313
+ "tpp_threshold_10_unintended_diff_only": 0.002749994397163391,
314
+ "tpp_threshold_20_total_metric": 0.019250020384788513,
315
+ "tpp_threshold_20_intended_diff_only": 0.022000014781951904,
316
+ "tpp_threshold_20_unintended_diff_only": 0.002749994397163391,
317
+ "tpp_threshold_50_total_metric": 0.06050001084804535,
318
+ "tpp_threshold_50_intended_diff_only": 0.06000000238418579,
319
+ "tpp_threshold_50_unintended_diff_only": -0.0005000084638595581,
320
+ "tpp_threshold_100_total_metric": 0.11225004494190216,
321
+ "tpp_threshold_100_intended_diff_only": 0.11800003051757812,
322
+ "tpp_threshold_100_unintended_diff_only": 0.005749985575675964,
323
+ "tpp_threshold_500_total_metric": 0.390500009059906,
324
+ "tpp_threshold_500_intended_diff_only": 0.40299999713897705,
325
+ "tpp_threshold_500_unintended_diff_only": 0.012499988079071045
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.013000041246414185,
329
+ "tpp_threshold_2_intended_diff_only": 0.01100003719329834,
330
+ "tpp_threshold_2_unintended_diff_only": -0.0020000040531158447,
331
+ "tpp_threshold_5_total_metric": 0.007750019431114197,
332
+ "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
333
+ "tpp_threshold_5_unintended_diff_only": 0.006249979138374329,
334
+ "tpp_threshold_10_total_metric": 0.048250049352645874,
335
+ "tpp_threshold_10_intended_diff_only": 0.05200004577636719,
336
+ "tpp_threshold_10_unintended_diff_only": 0.0037499964237213135,
337
+ "tpp_threshold_20_total_metric": 0.09850004315376282,
338
+ "tpp_threshold_20_intended_diff_only": 0.10300004482269287,
339
+ "tpp_threshold_20_unintended_diff_only": 0.004500001668930054,
340
+ "tpp_threshold_50_total_metric": 0.18325001001358032,
341
+ "tpp_threshold_50_intended_diff_only": 0.1940000057220459,
342
+ "tpp_threshold_50_unintended_diff_only": 0.010749995708465576,
343
+ "tpp_threshold_100_total_metric": 0.285750076174736,
344
+ "tpp_threshold_100_intended_diff_only": 0.30500006675720215,
345
+ "tpp_threshold_100_unintended_diff_only": 0.019249990582466125,
346
+ "tpp_threshold_500_total_metric": 0.41200003027915955,
347
+ "tpp_threshold_500_intended_diff_only": 0.43800002336502075,
348
+ "tpp_threshold_500_unintended_diff_only": 0.025999993085861206
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.005499988794326782,
352
+ "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
353
+ "tpp_threshold_2_unintended_diff_only": -0.0004999935626983643,
354
+ "tpp_threshold_5_total_metric": 0.012999996542930603,
355
+ "tpp_threshold_5_intended_diff_only": 0.008000016212463379,
356
+ "tpp_threshold_5_unintended_diff_only": -0.004999980330467224,
357
+ "tpp_threshold_10_total_metric": 0.016749992966651917,
358
+ "tpp_threshold_10_intended_diff_only": 0.014999985694885254,
359
+ "tpp_threshold_10_unintended_diff_only": -0.0017500072717666626,
360
+ "tpp_threshold_20_total_metric": 0.01524999737739563,
361
+ "tpp_threshold_20_intended_diff_only": 0.018999993801116943,
362
+ "tpp_threshold_20_unintended_diff_only": 0.0037499964237213135,
363
+ "tpp_threshold_50_total_metric": 0.04850001633167267,
364
+ "tpp_threshold_50_intended_diff_only": 0.04900002479553223,
365
+ "tpp_threshold_50_unintended_diff_only": 0.0005000084638595581,
366
+ "tpp_threshold_100_total_metric": 0.109250009059906,
367
+ "tpp_threshold_100_intended_diff_only": 0.12000000476837158,
368
+ "tpp_threshold_100_unintended_diff_only": 0.010749995708465576,
369
+ "tpp_threshold_500_total_metric": 0.34800003468990326,
370
+ "tpp_threshold_500_intended_diff_only": 0.37300002574920654,
371
+ "tpp_threshold_500_unintended_diff_only": 0.024999991059303284
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.001999959349632263,
375
+ "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
376
+ "tpp_threshold_2_unintended_diff_only": 1.4901161193847656e-08,
377
+ "tpp_threshold_5_total_metric": 0.012249961495399475,
378
+ "tpp_threshold_5_intended_diff_only": 0.011999964714050293,
379
+ "tpp_threshold_5_unintended_diff_only": -0.00024999678134918213,
380
+ "tpp_threshold_10_total_metric": 0.0007499605417251587,
381
+ "tpp_threshold_10_intended_diff_only": 0.006999969482421875,
382
+ "tpp_threshold_10_unintended_diff_only": 0.006250008940696716,
383
+ "tpp_threshold_20_total_metric": 0.012749999761581421,
384
+ "tpp_threshold_20_intended_diff_only": 0.013999998569488525,
385
+ "tpp_threshold_20_unintended_diff_only": 0.0012499988079071045,
386
+ "tpp_threshold_50_total_metric": 0.0782499760389328,
387
+ "tpp_threshold_50_intended_diff_only": 0.08499997854232788,
388
+ "tpp_threshold_50_unintended_diff_only": 0.006750002503395081,
389
+ "tpp_threshold_100_total_metric": 0.16099996864795685,
390
+ "tpp_threshold_100_intended_diff_only": 0.17299997806549072,
391
+ "tpp_threshold_100_unintended_diff_only": 0.012000009417533875,
392
+ "tpp_threshold_500_total_metric": 0.3669999837875366,
393
+ "tpp_threshold_500_intended_diff_only": 0.39399999380111694,
394
+ "tpp_threshold_500_unintended_diff_only": 0.027000010013580322
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.024749979376792908,
398
+ "tpp_threshold_2_intended_diff_only": 0.02899998426437378,
399
+ "tpp_threshold_2_unintended_diff_only": 0.004250004887580872,
400
+ "tpp_threshold_5_total_metric": 0.042999982833862305,
401
+ "tpp_threshold_5_intended_diff_only": 0.042999982833862305,
402
+ "tpp_threshold_5_unintended_diff_only": 0.0,
403
+ "tpp_threshold_10_total_metric": 0.07950000464916229,
404
+ "tpp_threshold_10_intended_diff_only": 0.08300000429153442,
405
+ "tpp_threshold_10_unintended_diff_only": 0.0034999996423721313,
406
+ "tpp_threshold_20_total_metric": 0.11400002241134644,
407
+ "tpp_threshold_20_intended_diff_only": 0.12400001287460327,
408
+ "tpp_threshold_20_unintended_diff_only": 0.009999990463256836,
409
+ "tpp_threshold_50_total_metric": 0.20349998772144318,
410
+ "tpp_threshold_50_intended_diff_only": 0.21399998664855957,
411
+ "tpp_threshold_50_unintended_diff_only": 0.010499998927116394,
412
+ "tpp_threshold_100_total_metric": 0.27699999511241913,
413
+ "tpp_threshold_100_intended_diff_only": 0.2879999876022339,
414
+ "tpp_threshold_100_unintended_diff_only": 0.010999992489814758,
415
+ "tpp_threshold_500_total_metric": 0.34175002574920654,
416
+ "tpp_threshold_500_intended_diff_only": 0.37300002574920654,
417
+ "tpp_threshold_500_unintended_diff_only": 0.03125
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-158691-tokens-650000000/scr/saebench_scr_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,8 +69,8 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "98c6e8bd-337e-442b-83a2-41a114ad753c",
69
- "datetime_epoch_millis": 1776884250692,
70
  "eval_result_metrics": {
71
  "scr_metrics": {
72
  "scr_dir1_threshold_2": 0.2936601455339623,
@@ -78,18 +82,18 @@
78
  "scr_dir1_threshold_10": 0.29843206934547256,
79
  "scr_metric_threshold_10": 0.25342062417805467,
80
  "scr_dir2_threshold_10": 0.23900938731986085,
81
- "scr_dir1_threshold_20": null,
82
- "scr_metric_threshold_20": null,
83
- "scr_dir2_threshold_20": null,
84
- "scr_dir1_threshold_50": null,
85
- "scr_metric_threshold_50": null,
86
- "scr_dir2_threshold_50": null,
87
- "scr_dir1_threshold_100": null,
88
- "scr_metric_threshold_100": null,
89
- "scr_dir2_threshold_100": null,
90
- "scr_dir1_threshold_500": null,
91
- "scr_metric_threshold_500": null,
92
- "scr_dir2_threshold_500": null
93
  }
94
  },
95
  "eval_result_details": [
@@ -104,18 +108,18 @@
104
  "scr_dir1_threshold_10": 0.38461573730554943,
105
  "scr_metric_threshold_10": 0.08290156000473131,
106
  "scr_dir2_threshold_10": 0.08290156000473131,
107
- "scr_dir1_threshold_20": null,
108
- "scr_metric_threshold_20": null,
109
- "scr_dir2_threshold_20": null,
110
- "scr_dir1_threshold_50": null,
111
- "scr_metric_threshold_50": null,
112
- "scr_dir2_threshold_50": null,
113
- "scr_dir1_threshold_100": null,
114
- "scr_metric_threshold_100": null,
115
- "scr_dir2_threshold_100": null,
116
- "scr_dir1_threshold_500": null,
117
- "scr_metric_threshold_500": null,
118
- "scr_dir2_threshold_500": null
119
  },
120
  {
121
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
@@ -128,18 +132,18 @@
128
  "scr_dir1_threshold_10": 0.41666662903748003,
129
  "scr_metric_threshold_10": 0.31192657874830254,
130
  "scr_dir2_threshold_10": 0.31192657874830254,
131
- "scr_dir1_threshold_20": null,
132
- "scr_metric_threshold_20": null,
133
- "scr_dir2_threshold_20": null,
134
- "scr_dir1_threshold_50": null,
135
- "scr_metric_threshold_50": null,
136
- "scr_dir2_threshold_50": null,
137
- "scr_dir1_threshold_100": null,
138
- "scr_metric_threshold_100": null,
139
- "scr_dir2_threshold_100": null,
140
- "scr_dir1_threshold_500": null,
141
- "scr_metric_threshold_500": null,
142
- "scr_dir2_threshold_500": null
143
  },
144
  {
145
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
@@ -152,18 +156,18 @@
152
  "scr_dir1_threshold_10": 0.31249965075409175,
153
  "scr_metric_threshold_10": 0.11306535109152112,
154
  "scr_dir2_threshold_10": 0.11306535109152112,
155
- "scr_dir1_threshold_20": null,
156
- "scr_metric_threshold_20": null,
157
- "scr_dir2_threshold_20": null,
158
- "scr_dir1_threshold_50": null,
159
- "scr_metric_threshold_50": null,
160
- "scr_dir2_threshold_50": null,
161
- "scr_dir1_threshold_100": null,
162
- "scr_metric_threshold_100": null,
163
- "scr_dir2_threshold_100": null,
164
- "scr_dir1_threshold_500": null,
165
- "scr_metric_threshold_500": null,
166
- "scr_dir2_threshold_500": null
167
  },
168
  {
169
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
@@ -176,18 +180,18 @@
176
  "scr_dir1_threshold_10": 0.1543624402703411,
177
  "scr_metric_threshold_10": 0.1337579436474169,
178
  "scr_dir2_threshold_10": 0.1337579436474169,
179
- "scr_dir1_threshold_20": null,
180
- "scr_metric_threshold_20": null,
181
- "scr_dir2_threshold_20": null,
182
- "scr_dir1_threshold_50": null,
183
- "scr_metric_threshold_50": null,
184
- "scr_dir2_threshold_50": null,
185
- "scr_dir1_threshold_100": null,
186
- "scr_metric_threshold_100": null,
187
- "scr_dir2_threshold_100": null,
188
- "scr_dir1_threshold_500": null,
189
- "scr_metric_threshold_500": null,
190
- "scr_dir2_threshold_500": null
191
  },
192
  {
193
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
@@ -200,18 +204,18 @@
200
  "scr_dir1_threshold_10": 0.08947378822971379,
201
  "scr_metric_threshold_10": 0.5277777712076017,
202
  "scr_dir2_threshold_10": 0.5277777712076017,
203
- "scr_dir1_threshold_20": null,
204
- "scr_metric_threshold_20": null,
205
- "scr_dir2_threshold_20": null,
206
- "scr_dir1_threshold_50": null,
207
- "scr_metric_threshold_50": null,
208
- "scr_dir2_threshold_50": null,
209
- "scr_dir1_threshold_100": null,
210
- "scr_metric_threshold_100": null,
211
- "scr_dir2_threshold_100": null,
212
- "scr_dir1_threshold_500": null,
213
- "scr_metric_threshold_500": null,
214
- "scr_dir2_threshold_500": null
215
  },
216
  {
217
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
@@ -224,18 +228,18 @@
224
  "scr_dir1_threshold_10": 0.3879780993606342,
225
  "scr_metric_threshold_10": 0.15648856785294307,
226
  "scr_dir2_threshold_10": 0.15648856785294307,
227
- "scr_dir1_threshold_20": null,
228
- "scr_metric_threshold_20": null,
229
- "scr_dir2_threshold_20": null,
230
- "scr_dir1_threshold_50": null,
231
- "scr_metric_threshold_50": null,
232
- "scr_dir2_threshold_50": null,
233
- "scr_dir1_threshold_100": null,
234
- "scr_metric_threshold_100": null,
235
- "scr_dir2_threshold_100": null,
236
- "scr_dir1_threshold_500": null,
237
- "scr_metric_threshold_500": null,
238
- "scr_dir2_threshold_500": null
239
  },
240
  {
241
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
@@ -248,18 +252,18 @@
248
  "scr_dir1_threshold_10": 0.3953486760285681,
249
  "scr_metric_threshold_10": 0.4549356870945189,
250
  "scr_dir2_threshold_10": 0.4549356870945189,
251
- "scr_dir1_threshold_20": null,
252
- "scr_metric_threshold_20": null,
253
- "scr_dir2_threshold_20": null,
254
- "scr_dir1_threshold_50": null,
255
- "scr_metric_threshold_50": null,
256
- "scr_dir2_threshold_50": null,
257
- "scr_dir1_threshold_100": null,
258
- "scr_metric_threshold_100": null,
259
- "scr_dir2_threshold_100": null,
260
- "scr_dir1_threshold_500": null,
261
- "scr_metric_threshold_500": null,
262
- "scr_dir2_threshold_500": null
263
  },
264
  {
265
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
@@ -272,18 +276,18 @@
272
  "scr_dir1_threshold_10": 0.24651153377740184,
273
  "scr_metric_threshold_10": 0.24651153377740184,
274
  "scr_dir2_threshold_10": 0.1312216389118512,
275
- "scr_dir1_threshold_20": null,
276
- "scr_metric_threshold_20": null,
277
- "scr_dir2_threshold_20": null,
278
- "scr_dir1_threshold_50": null,
279
- "scr_metric_threshold_50": null,
280
- "scr_dir2_threshold_50": null,
281
- "scr_dir1_threshold_100": null,
282
- "scr_metric_threshold_100": null,
283
- "scr_dir2_threshold_100": null,
284
- "scr_dir1_threshold_500": null,
285
- "scr_metric_threshold_500": null,
286
- "scr_dir2_threshold_500": null
287
  }
288
  ],
289
  "sae_bench_commit_hash": "Unknown",
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "d0b86c77-9df7-491a-b204-93d13ef5b918",
73
+ "datetime_epoch_millis": 1777316081630,
74
  "eval_result_metrics": {
75
  "scr_metrics": {
76
  "scr_dir1_threshold_2": 0.2936601455339623,
 
82
  "scr_dir1_threshold_10": 0.29843206934547256,
83
  "scr_metric_threshold_10": 0.25342062417805467,
84
  "scr_dir2_threshold_10": 0.23900938731986085,
85
+ "scr_dir1_threshold_20": 0.13807877076721764,
86
+ "scr_metric_threshold_20": 0.3056230421071651,
87
+ "scr_dir2_threshold_20": 0.2893886838653024,
88
+ "scr_dir1_threshold_50": 0.29747191129756534,
89
+ "scr_metric_threshold_50": 0.3545211424365467,
90
+ "scr_dir2_threshold_50": 0.339873136799641,
91
+ "scr_dir1_threshold_100": -0.14174953070081284,
92
+ "scr_metric_threshold_100": 0.37074884019934473,
93
+ "scr_dir2_threshold_100": 0.3525809025664471,
94
+ "scr_dir1_threshold_500": -1.0172816718021795,
95
+ "scr_metric_threshold_500": 0.2074486256017326,
96
+ "scr_dir2_threshold_500": 0.16098171357221602
97
  }
98
  },
99
  "eval_result_details": [
 
108
  "scr_dir1_threshold_10": 0.38461573730554943,
109
  "scr_metric_threshold_10": 0.08290156000473131,
110
  "scr_dir2_threshold_10": 0.08290156000473131,
111
+ "scr_dir1_threshold_20": 0.2820514388024664,
112
+ "scr_metric_threshold_20": 0.13471497710162714,
113
+ "scr_dir2_threshold_20": 0.13471497710162714,
114
+ "scr_dir1_threshold_50": 0.5384614208981502,
115
+ "scr_metric_threshold_50": 0.26683931423164214,
116
+ "scr_dir2_threshold_50": 0.26683931423164214,
117
+ "scr_dir1_threshold_100": 0.5256410746257707,
118
+ "scr_metric_threshold_100": 0.3264248056595377,
119
+ "scr_dir2_threshold_100": 0.3264248056595377,
120
+ "scr_dir1_threshold_500": -2.16666602986498,
121
+ "scr_metric_threshold_500": -0.0492227771252837,
122
+ "scr_dir2_threshold_500": -0.0492227771252837
123
  },
124
  {
125
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
 
132
  "scr_dir1_threshold_10": 0.41666662903748003,
133
  "scr_metric_threshold_10": 0.31192657874830254,
134
  "scr_dir2_threshold_10": 0.31192657874830254,
135
+ "scr_dir1_threshold_20": 0.12121234014557077,
136
+ "scr_metric_threshold_20": 0.31804288982255313,
137
+ "scr_dir2_threshold_20": 0.31804288982255313,
138
+ "scr_dir1_threshold_50": 0.5681820542194435,
139
+ "scr_metric_threshold_50": 0.47706424359523525,
140
+ "scr_dir2_threshold_50": 0.47706424359523525,
141
+ "scr_dir1_threshold_100": 0.10606057185225456,
142
+ "scr_metric_threshold_100": 0.5290520674790153,
143
+ "scr_dir2_threshold_100": 0.5290520674790153,
144
+ "scr_dir1_threshold_500": -1.9772721217849059,
145
+ "scr_metric_threshold_500": 0.09785933669330951,
146
+ "scr_dir2_threshold_500": 0.09785933669330951
147
  },
148
  {
149
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
 
156
  "scr_dir1_threshold_10": 0.31249965075409175,
157
  "scr_metric_threshold_10": 0.11306535109152112,
158
  "scr_dir2_threshold_10": 0.11306535109152112,
159
+ "scr_dir1_threshold_20": 0.0,
160
+ "scr_metric_threshold_20": 0.2060301620422405,
161
+ "scr_dir2_threshold_20": 0.2060301620422405,
162
+ "scr_dir1_threshold_50": 0.28125005820765137,
163
+ "scr_metric_threshold_50": 0.1859296219014388,
164
+ "scr_dir2_threshold_50": 0.1859296219014388,
165
+ "scr_dir1_threshold_100": -1.8124996507540918,
166
+ "scr_metric_threshold_100": 0.16080402160563206,
167
+ "scr_dir2_threshold_100": 0.16080402160563206,
168
+ "scr_dir1_threshold_500": -3.343749243300532,
169
+ "scr_metric_threshold_500": -0.10301508102112025,
170
+ "scr_dir2_threshold_500": -0.10301508102112025
171
  },
172
  {
173
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
 
180
  "scr_dir1_threshold_10": 0.1543624402703411,
181
  "scr_metric_threshold_10": 0.1337579436474169,
182
  "scr_dir2_threshold_10": 0.1337579436474169,
183
+ "scr_dir1_threshold_20": -0.5637587731843083,
184
+ "scr_metric_threshold_20": 0.1687899080607876,
185
+ "scr_dir2_threshold_20": 0.1687899080607876,
186
+ "scr_dir1_threshold_50": -0.0604027114114901,
187
+ "scr_metric_threshold_50": 0.21974523381154987,
188
+ "scr_dir2_threshold_50": 0.21974523381154987,
189
+ "scr_dir1_threshold_100": -1.1140940993397486,
190
+ "scr_metric_threshold_100": 0.124203926844982,
191
+ "scr_dir2_threshold_100": 0.124203926844982,
192
+ "scr_dir1_threshold_500": -1.912751994655583,
193
+ "scr_metric_threshold_500": -0.528662430054711,
194
+ "scr_dir2_threshold_500": -0.528662430054711
195
  },
196
  {
197
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
 
204
  "scr_dir1_threshold_10": 0.08947378822971379,
205
  "scr_metric_threshold_10": 0.5277777712076017,
206
  "scr_dir2_threshold_10": 0.5277777712076017,
207
+ "scr_dir1_threshold_20": 0.08947378822971379,
208
+ "scr_metric_threshold_20": 0.5992063595309115,
209
+ "scr_dir2_threshold_20": 0.5992063595309115,
210
+ "scr_dir1_threshold_50": -0.2578945568723999,
211
+ "scr_metric_threshold_50": 0.6944443984532124,
212
+ "scr_dir2_threshold_50": 0.6944443984532124,
213
+ "scr_dir1_threshold_100": -0.23684219607355966,
214
+ "scr_metric_threshold_100": 0.7222222879239824,
215
+ "scr_dir2_threshold_100": 0.7222222879239824,
216
+ "scr_dir1_threshold_500": -0.15263149804355594,
217
+ "scr_metric_threshold_500": 0.8055554832836195,
218
+ "scr_dir2_threshold_500": 0.8055554832836195
219
  },
220
  {
221
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
 
228
  "scr_dir1_threshold_10": 0.3879780993606342,
229
  "scr_metric_threshold_10": 0.15648856785294307,
230
  "scr_dir2_threshold_10": 0.15648856785294307,
231
+ "scr_dir1_threshold_20": 0.3989072479911928,
232
+ "scr_metric_threshold_20": 0.20229018313866656,
233
+ "scr_dir2_threshold_20": 0.20229018313866656,
234
+ "scr_dir1_threshold_50": 0.4590164254795243,
235
+ "scr_metric_threshold_50": 0.29389318621149046,
236
+ "scr_dir2_threshold_50": 0.29389318621149046,
237
+ "scr_dir1_threshold_100": 0.4535520140184937,
238
+ "scr_metric_threshold_100": 0.3206106245041952,
239
+ "scr_dir2_threshold_100": 0.3206106245041952,
240
+ "scr_dir1_threshold_500": 0.5683059575341262,
241
+ "scr_metric_threshold_500": 0.4007633943795555,
242
+ "scr_dir2_threshold_500": 0.4007633943795555
243
  },
244
  {
245
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
 
252
  "scr_dir1_threshold_10": 0.3953486760285681,
253
  "scr_metric_threshold_10": 0.4549356870945189,
254
  "scr_dir2_threshold_10": 0.4549356870945189,
255
+ "scr_dir1_threshold_20": 0.49302320810445616,
256
+ "scr_metric_threshold_20": 0.5321889411118851,
257
+ "scr_dir2_threshold_20": 0.5321889411118851,
258
+ "scr_dir1_threshold_50": 0.5348836822468563,
259
+ "scr_metric_threshold_50": 0.38197422367501804,
260
+ "scr_dir2_threshold_50": 0.38197422367501804,
261
+ "scr_dir1_threshold_100": 0.5906977401803443,
262
+ "scr_metric_threshold_100": 0.42918468769337936,
263
+ "scr_dir2_threshold_100": 0.42918468769337936,
264
+ "scr_dir1_threshold_500": 0.3209302093481138,
265
+ "scr_metric_threshold_500": 0.5107297323086109,
266
+ "scr_dir2_threshold_500": 0.5107297323086109
267
  },
268
  {
269
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
 
276
  "scr_dir1_threshold_10": 0.24651153377740184,
277
  "scr_metric_threshold_10": 0.24651153377740184,
278
  "scr_dir2_threshold_10": 0.1312216389118512,
279
+ "scr_dir1_threshold_20": 0.2837209160486496,
280
+ "scr_metric_threshold_20": 0.2837209160486496,
281
+ "scr_dir2_threshold_20": 0.15384605011374777,
282
+ "scr_dir1_threshold_50": 0.31627891761278654,
283
+ "scr_metric_threshold_50": 0.31627891761278654,
284
+ "scr_dir2_threshold_50": 0.19909487251754088,
285
+ "scr_dir1_threshold_100": 0.3534882998840343,
286
+ "scr_metric_threshold_100": 0.3534882998840343,
287
+ "scr_dir2_threshold_100": 0.208144798820853,
288
+ "scr_dir1_threshold_500": 0.5255813463498804,
289
+ "scr_metric_threshold_500": 0.5255813463498804,
290
+ "scr_dir2_threshold_500": 0.15384605011374777
291
  }
292
  ],
293
  "sae_bench_commit_hash": "Unknown",
gemma-2-2b-btk/k-100/seed-0/snapshots/step-158691-tokens-650000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "6a3f4e41-9281-4ef1-a4a0-7a787ea04fa5",
69
- "datetime_epoch_millis": 1776884414187,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.019324998557567596,
73
- "tpp_threshold_2_intended_diff_only": 0.020399999618530274,
74
- "tpp_threshold_2_unintended_diff_only": 0.001075001060962677,
75
- "tpp_threshold_5_total_metric": 0.06357499957084656,
76
- "tpp_threshold_5_intended_diff_only": 0.07430000305175781,
77
- "tpp_threshold_5_unintended_diff_only": 0.010725003480911254,
78
- "tpp_threshold_10_total_metric": 0.10377499908208848,
79
- "tpp_threshold_10_intended_diff_only": 0.13830000758171082,
80
- "tpp_threshold_10_unintended_diff_only": 0.034525008499622346,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.02819998562335968,
99
- "tpp_threshold_2_intended_diff_only": 0.02999999523162842,
100
- "tpp_threshold_2_unintended_diff_only": 0.0018000096082687379,
101
- "tpp_threshold_5_total_metric": 0.1095499962568283,
102
  "tpp_threshold_5_intended_diff_only": 0.1290000081062317,
103
- "tpp_threshold_5_unintended_diff_only": 0.01945001184940338,
104
- "tpp_threshold_10_total_metric": 0.18080000281333924,
105
- "tpp_threshold_10_intended_diff_only": 0.24680001735687257,
106
- "tpp_threshold_10_unintended_diff_only": 0.06600001454353333,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.010450011491775513,
123
- "tpp_threshold_2_intended_diff_only": 0.010800004005432129,
124
- "tpp_threshold_2_unintended_diff_only": 0.0003499925136566162,
125
- "tpp_threshold_5_total_metric": 0.017600002884864806,
126
- "tpp_threshold_5_intended_diff_only": 0.019599997997283937,
127
- "tpp_threshold_5_unintended_diff_only": 0.0019999951124191283,
128
- "tpp_threshold_10_total_metric": 0.026749995350837708,
129
- "tpp_threshold_10_intended_diff_only": 0.029799997806549072,
130
- "tpp_threshold_10_unintended_diff_only": 0.003050002455711365,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.003250017762184143,
190
  "tpp_threshold_10_total_metric": 0.019999966025352478,
191
  "tpp_threshold_10_intended_diff_only": 0.02399998903274536,
192
- "tpp_threshold_10_unintended_diff_only": 0.004000023007392883
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": -2.9802322387695312e-08,
196
- "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
197
- "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162,
198
- "tpp_threshold_5_total_metric": 0.15574999153614044,
199
  "tpp_threshold_5_intended_diff_only": 0.18400001525878906,
200
- "tpp_threshold_5_unintended_diff_only": 0.02825002372264862,
201
- "tpp_threshold_10_total_metric": 0.20650000870227814,
202
- "tpp_threshold_10_intended_diff_only": 0.2720000147819519,
203
- "tpp_threshold_10_unintended_diff_only": 0.06550000607967377
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.11574999988079071,
207
- "tpp_threshold_2_intended_diff_only": 0.12400001287460327,
208
- "tpp_threshold_2_unintended_diff_only": 0.008250012993812561,
209
- "tpp_threshold_5_total_metric": 0.242000013589859,
210
- "tpp_threshold_5_intended_diff_only": 0.281000018119812,
211
- "tpp_threshold_5_unintended_diff_only": 0.039000004529953,
212
- "tpp_threshold_10_total_metric": 0.2175000160932541,
213
- "tpp_threshold_10_intended_diff_only": 0.3360000252723694,
214
- "tpp_threshold_10_unintended_diff_only": 0.1185000091791153
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.010750025510787964,
218
  "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
219
- "tpp_threshold_2_unintended_diff_only": -0.002750009298324585,
220
- "tpp_threshold_5_total_metric": 0.1027500331401825,
221
- "tpp_threshold_5_intended_diff_only": 0.1170000433921814,
222
- "tpp_threshold_5_unintended_diff_only": 0.014250010251998901,
223
- "tpp_threshold_10_total_metric": 0.27524998784065247,
224
  "tpp_threshold_10_intended_diff_only": 0.35100001096725464,
225
- "tpp_threshold_10_unintended_diff_only": 0.07575002312660217
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.003999993205070496,
229
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
230
- "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224,
231
- "tpp_threshold_5_total_metric": 0.02449999749660492,
232
  "tpp_threshold_5_intended_diff_only": 0.03700000047683716,
233
- "tpp_threshold_5_unintended_diff_only": 0.012500002980232239,
234
- "tpp_threshold_10_total_metric": 0.184750035405159,
235
- "tpp_threshold_10_intended_diff_only": 0.2510000467300415,
236
- "tpp_threshold_10_unintended_diff_only": 0.06625001132488251
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.009499996900558472,
242
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
243
- "tpp_threshold_2_unintended_diff_only": -0.004500001668930054,
244
- "tpp_threshold_5_total_metric": 0.014000028371810913,
245
- "tpp_threshold_5_intended_diff_only": 0.012000024318695068,
246
- "tpp_threshold_5_unintended_diff_only": -0.0020000040531158447,
247
- "tpp_threshold_10_total_metric": -0.000250011682510376,
248
  "tpp_threshold_10_intended_diff_only": 0.004999995231628418,
249
- "tpp_threshold_10_unintended_diff_only": 0.005250006914138794
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": 0.0015000402927398682,
253
- "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
254
- "tpp_threshold_2_unintended_diff_only": 0.005499988794326782,
255
- "tpp_threshold_5_total_metric": 0.007750019431114197,
256
- "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
257
- "tpp_threshold_5_unintended_diff_only": 0.006249979138374329,
258
- "tpp_threshold_10_total_metric": 0.03450004756450653,
259
- "tpp_threshold_10_intended_diff_only": 0.03800004720687866,
260
- "tpp_threshold_10_unintended_diff_only": 0.0034999996423721313
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.012499988079071045,
264
- "tpp_threshold_2_intended_diff_only": 0.0059999823570251465,
265
- "tpp_threshold_2_unintended_diff_only": -0.0065000057220458984,
266
- "tpp_threshold_5_total_metric": 0.015750005841255188,
267
- "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
268
- "tpp_threshold_5_unintended_diff_only": -0.0017500072717666626,
269
- "tpp_threshold_10_total_metric": 0.016500011086463928,
270
- "tpp_threshold_10_intended_diff_only": 0.018000006675720215,
271
- "tpp_threshold_10_unintended_diff_only": 0.0014999955892562866
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.007499992847442627,
275
- "tpp_threshold_2_intended_diff_only": 0.0,
276
- "tpp_threshold_2_unintended_diff_only": 0.007499992847442627,
277
- "tpp_threshold_5_total_metric": -1.4901161193847656e-08,
278
- "tpp_threshold_5_intended_diff_only": 0.0009999871253967285,
279
- "tpp_threshold_5_unintended_diff_only": 0.0010000020265579224,
280
- "tpp_threshold_10_total_metric": 0.000999942421913147,
281
- "tpp_threshold_10_intended_diff_only": 0.003999948501586914,
282
- "tpp_threshold_10_unintended_diff_only": 0.003000006079673767
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.036250025033950806,
286
- "tpp_threshold_2_intended_diff_only": 0.03600001335144043,
287
- "tpp_threshold_2_unintended_diff_only": -0.000250011682510376,
288
- "tpp_threshold_5_total_metric": 0.05049997568130493,
289
- "tpp_threshold_5_intended_diff_only": 0.05699998140335083,
290
- "tpp_threshold_5_unintended_diff_only": 0.0065000057220458984,
291
- "tpp_threshold_10_total_metric": 0.08199998736381531,
292
- "tpp_threshold_10_intended_diff_only": 0.08399999141693115,
293
- "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "17179781-3ba6-4cf6-9aed-554cc99614ea",
73
+ "datetime_epoch_millis": 1777316353206,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.018099991977214812,
77
+ "tpp_threshold_2_intended_diff_only": 0.020499998331069948,
78
+ "tpp_threshold_2_unintended_diff_only": 0.0024000063538551332,
79
+ "tpp_threshold_5_total_metric": 0.06372499167919159,
80
+ "tpp_threshold_5_intended_diff_only": 0.07319999933242798,
81
+ "tpp_threshold_5_unintended_diff_only": 0.00947500765323639,
82
+ "tpp_threshold_10_total_metric": 0.10470000058412553,
83
+ "tpp_threshold_10_intended_diff_only": 0.139000004529953,
84
+ "tpp_threshold_10_unintended_diff_only": 0.03430000394582748,
85
+ "tpp_threshold_20_total_metric": 0.14527500718832015,
86
+ "tpp_threshold_20_intended_diff_only": 0.1982000172138214,
87
+ "tpp_threshold_20_unintended_diff_only": 0.05292501002550125,
88
+ "tpp_threshold_50_total_metric": 0.1659500002861023,
89
+ "tpp_threshold_50_intended_diff_only": 0.2554000079631805,
90
+ "tpp_threshold_50_unintended_diff_only": 0.08945000767707825,
91
+ "tpp_threshold_100_total_metric": 0.18922500908374784,
92
+ "tpp_threshold_100_intended_diff_only": 0.31510002017021177,
93
+ "tpp_threshold_100_unintended_diff_only": 0.12587501108646393,
94
+ "tpp_threshold_500_total_metric": 0.24060002863407134,
95
+ "tpp_threshold_500_intended_diff_only": 0.4327000379562378,
96
+ "tpp_threshold_500_unintended_diff_only": 0.19210000932216645
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.027099993824958802,
103
+ "tpp_threshold_2_intended_diff_only": 0.030800008773803712,
104
+ "tpp_threshold_2_unintended_diff_only": 0.0037000149488449095,
105
+ "tpp_threshold_5_total_metric": 0.11039999425411225,
106
  "tpp_threshold_5_intended_diff_only": 0.1290000081062317,
107
+ "tpp_threshold_5_unintended_diff_only": 0.018600013852119446,
108
+ "tpp_threshold_10_total_metric": 0.18130000531673432,
109
+ "tpp_threshold_10_intended_diff_only": 0.2466000199317932,
110
+ "tpp_threshold_10_unintended_diff_only": 0.0653000146150589,
111
+ "tpp_threshold_20_total_metric": 0.24280000627040862,
112
+ "tpp_threshold_20_intended_diff_only": 0.3440000295639038,
113
+ "tpp_threshold_20_unintended_diff_only": 0.10120002329349517,
114
+ "tpp_threshold_50_total_metric": 0.2319500058889389,
115
+ "tpp_threshold_50_intended_diff_only": 0.40540002584457396,
116
+ "tpp_threshold_50_unintended_diff_only": 0.17345001995563508,
117
+ "tpp_threshold_100_total_metric": 0.19920000731945037,
118
+ "tpp_threshold_100_intended_diff_only": 0.4392000317573547,
119
+ "tpp_threshold_100_unintended_diff_only": 0.24000002443790436,
120
+ "tpp_threshold_500_total_metric": 0.10850001871585846,
121
+ "tpp_threshold_500_intended_diff_only": 0.46620004177093505,
122
+ "tpp_threshold_500_unintended_diff_only": 0.3577000230550766
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.009099990129470825,
127
+ "tpp_threshold_2_intended_diff_only": 0.010199987888336181,
128
+ "tpp_threshold_2_unintended_diff_only": 0.0010999977588653565,
129
+ "tpp_threshold_5_total_metric": 0.017049989104270934,
130
+ "tpp_threshold_5_intended_diff_only": 0.01739999055862427,
131
+ "tpp_threshold_5_unintended_diff_only": 0.00035000145435333253,
132
+ "tpp_threshold_10_total_metric": 0.028099995851516724,
133
+ "tpp_threshold_10_intended_diff_only": 0.031399989128112794,
134
+ "tpp_threshold_10_unintended_diff_only": 0.003299993276596069,
135
+ "tpp_threshold_20_total_metric": 0.04775000810623169,
136
+ "tpp_threshold_20_intended_diff_only": 0.05240000486373901,
137
+ "tpp_threshold_20_unintended_diff_only": 0.004649996757507324,
138
+ "tpp_threshold_50_total_metric": 0.09994999468326568,
139
+ "tpp_threshold_50_intended_diff_only": 0.1053999900817871,
140
+ "tpp_threshold_50_unintended_diff_only": 0.005449995398521423,
141
+ "tpp_threshold_100_total_metric": 0.17925001084804534,
142
+ "tpp_threshold_100_intended_diff_only": 0.19100000858306884,
143
+ "tpp_threshold_100_unintended_diff_only": 0.011749997735023499,
144
+ "tpp_threshold_500_total_metric": 0.37270003855228423,
145
+ "tpp_threshold_500_intended_diff_only": 0.39920003414154054,
146
+ "tpp_threshold_500_unintended_diff_only": 0.026499995589256288
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.003250017762184143,
194
  "tpp_threshold_10_total_metric": 0.019999966025352478,
195
  "tpp_threshold_10_intended_diff_only": 0.02399998903274536,
196
+ "tpp_threshold_10_unintended_diff_only": 0.004000023007392883,
197
+ "tpp_threshold_20_total_metric": 0.22499997913837433,
198
+ "tpp_threshold_20_intended_diff_only": 0.30000001192092896,
199
+ "tpp_threshold_20_unintended_diff_only": 0.07500003278255463,
200
+ "tpp_threshold_50_total_metric": 0.25224997103214264,
201
+ "tpp_threshold_50_intended_diff_only": 0.39899998903274536,
202
+ "tpp_threshold_50_unintended_diff_only": 0.14675001800060272,
203
+ "tpp_threshold_100_total_metric": 0.22524996101856232,
204
+ "tpp_threshold_100_intended_diff_only": 0.44099998474121094,
205
+ "tpp_threshold_100_unintended_diff_only": 0.21575002372264862,
206
+ "tpp_threshold_500_total_metric": 0.125,
207
+ "tpp_threshold_500_intended_diff_only": 0.45600003004074097,
208
+ "tpp_threshold_500_unintended_diff_only": 0.33100003004074097
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.004499971866607666,
212
+ "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
213
+ "tpp_threshold_2_unintended_diff_only": 0.000500023365020752,
214
+ "tpp_threshold_5_total_metric": 0.16050000488758087,
215
  "tpp_threshold_5_intended_diff_only": 0.18400001525878906,
216
+ "tpp_threshold_5_unintended_diff_only": 0.02350001037120819,
217
+ "tpp_threshold_10_total_metric": 0.20725001394748688,
218
+ "tpp_threshold_10_intended_diff_only": 0.2710000276565552,
219
+ "tpp_threshold_10_unintended_diff_only": 0.0637500137090683,
220
+ "tpp_threshold_20_total_metric": 0.23374997079372406,
221
+ "tpp_threshold_20_intended_diff_only": 0.3149999976158142,
222
+ "tpp_threshold_20_unintended_diff_only": 0.08125002682209015,
223
+ "tpp_threshold_50_total_metric": 0.22200000286102295,
224
+ "tpp_threshold_50_intended_diff_only": 0.36500000953674316,
225
+ "tpp_threshold_50_unintended_diff_only": 0.14300000667572021,
226
+ "tpp_threshold_100_total_metric": 0.20224998891353607,
227
+ "tpp_threshold_100_intended_diff_only": 0.406000018119812,
228
+ "tpp_threshold_100_unintended_diff_only": 0.20375002920627594,
229
+ "tpp_threshold_500_total_metric": 0.09350000321865082,
230
+ "tpp_threshold_500_intended_diff_only": 0.4570000171661377,
231
+ "tpp_threshold_500_unintended_diff_only": 0.3635000139474869
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.10475003719329834,
235
+ "tpp_threshold_2_intended_diff_only": 0.12100005149841309,
236
+ "tpp_threshold_2_unintended_diff_only": 0.016250014305114746,
237
+ "tpp_threshold_5_total_metric": 0.23900000751018524,
238
+ "tpp_threshold_5_intended_diff_only": 0.2800000309944153,
239
+ "tpp_threshold_5_unintended_diff_only": 0.04100002348423004,
240
+ "tpp_threshold_10_total_metric": 0.2199999988079071,
241
+ "tpp_threshold_10_intended_diff_only": 0.3370000123977661,
242
+ "tpp_threshold_10_unintended_diff_only": 0.11700001358985901,
243
+ "tpp_threshold_20_total_metric": 0.19850000739097595,
244
+ "tpp_threshold_20_intended_diff_only": 0.3630000352859497,
245
+ "tpp_threshold_20_unintended_diff_only": 0.16450002789497375,
246
+ "tpp_threshold_50_total_metric": 0.15650004148483276,
247
+ "tpp_threshold_50_intended_diff_only": 0.3840000629425049,
248
+ "tpp_threshold_50_unintended_diff_only": 0.22750002145767212,
249
+ "tpp_threshold_100_total_metric": 0.11699999868869781,
250
+ "tpp_threshold_100_intended_diff_only": 0.4230000376701355,
251
+ "tpp_threshold_100_unintended_diff_only": 0.3060000389814377,
252
+ "tpp_threshold_500_total_metric": 0.056249991059303284,
253
+ "tpp_threshold_500_intended_diff_only": 0.4520000219345093,
254
+ "tpp_threshold_500_unintended_diff_only": 0.395750030875206
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.00849999487400055,
258
  "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704,
260
+ "tpp_threshold_5_total_metric": 0.10775002837181091,
261
+ "tpp_threshold_5_intended_diff_only": 0.11800003051757812,
262
+ "tpp_threshold_5_unintended_diff_only": 0.010250002145767212,
263
+ "tpp_threshold_10_total_metric": 0.2772499918937683,
264
  "tpp_threshold_10_intended_diff_only": 0.35100001096725464,
265
+ "tpp_threshold_10_unintended_diff_only": 0.07375001907348633,
266
+ "tpp_threshold_20_total_metric": 0.3137500435113907,
267
+ "tpp_threshold_20_intended_diff_only": 0.4310000538825989,
268
+ "tpp_threshold_20_unintended_diff_only": 0.11725001037120819,
269
+ "tpp_threshold_50_total_metric": 0.27125000953674316,
270
+ "tpp_threshold_50_intended_diff_only": 0.47800004482269287,
271
+ "tpp_threshold_50_unintended_diff_only": 0.2067500352859497,
272
+ "tpp_threshold_100_total_metric": 0.22150003910064697,
273
+ "tpp_threshold_100_intended_diff_only": 0.48200005292892456,
274
+ "tpp_threshold_100_unintended_diff_only": 0.2605000138282776,
275
+ "tpp_threshold_500_total_metric": 0.13700005412101746,
276
+ "tpp_threshold_500_intended_diff_only": 0.49000006914138794,
277
+ "tpp_threshold_500_unintended_diff_only": 0.3530000150203705
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.0072500258684158325,
281
+ "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464,
283
+ "tpp_threshold_5_total_metric": 0.021999984979629517,
284
  "tpp_threshold_5_intended_diff_only": 0.03700000047683716,
285
+ "tpp_threshold_5_unintended_diff_only": 0.015000015497207642,
286
+ "tpp_threshold_10_total_metric": 0.1820000559091568,
287
+ "tpp_threshold_10_intended_diff_only": 0.2500000596046448,
288
+ "tpp_threshold_10_unintended_diff_only": 0.06800000369548798,
289
+ "tpp_threshold_20_total_metric": 0.24300003051757812,
290
+ "tpp_threshold_20_intended_diff_only": 0.3110000491142273,
291
+ "tpp_threshold_20_unintended_diff_only": 0.06800001859664917,
292
+ "tpp_threshold_50_total_metric": 0.257750004529953,
293
+ "tpp_threshold_50_intended_diff_only": 0.4010000228881836,
294
+ "tpp_threshold_50_unintended_diff_only": 0.1432500183582306,
295
+ "tpp_threshold_100_total_metric": 0.23000004887580872,
296
+ "tpp_threshold_100_intended_diff_only": 0.4440000653266907,
297
+ "tpp_threshold_100_unintended_diff_only": 0.21400001645088196,
298
+ "tpp_threshold_500_total_metric": 0.13075004518032074,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.3452500253915787
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.0029999762773513794,
306
+ "tpp_threshold_2_intended_diff_only": 0.006999969482421875,
307
+ "tpp_threshold_2_unintended_diff_only": 0.003999993205070496,
308
+ "tpp_threshold_5_total_metric": 0.006999999284744263,
309
+ "tpp_threshold_5_intended_diff_only": 0.009000003337860107,
310
+ "tpp_threshold_5_unintended_diff_only": 0.0020000040531158447,
311
+ "tpp_threshold_10_total_metric": 0.0017500072717666626,
312
  "tpp_threshold_10_intended_diff_only": 0.004999995231628418,
313
+ "tpp_threshold_10_unintended_diff_only": 0.0032499879598617554,
314
+ "tpp_threshold_20_total_metric": 0.010000020265579224,
315
+ "tpp_threshold_20_intended_diff_only": 0.013000011444091797,
316
+ "tpp_threshold_20_unintended_diff_only": 0.0029999911785125732,
317
+ "tpp_threshold_50_total_metric": 0.05274999141693115,
318
+ "tpp_threshold_50_intended_diff_only": 0.05199998617172241,
319
+ "tpp_threshold_50_unintended_diff_only": -0.0007500052452087402,
320
+ "tpp_threshold_100_total_metric": 0.10674998164176941,
321
+ "tpp_threshold_100_intended_diff_only": 0.11299997568130493,
322
+ "tpp_threshold_100_unintended_diff_only": 0.0062499940395355225,
323
+ "tpp_threshold_500_total_metric": 0.39225003123283386,
324
+ "tpp_threshold_500_intended_diff_only": 0.406000018119812,
325
+ "tpp_threshold_500_unintended_diff_only": 0.01374998688697815
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.011000007390975952,
329
+ "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
330
+ "tpp_threshold_2_unintended_diff_only": -0.0020000040531158447,
331
+ "tpp_threshold_5_total_metric": 0.0022500157356262207,
332
+ "tpp_threshold_5_intended_diff_only": 0.009000003337860107,
333
+ "tpp_threshold_5_unintended_diff_only": 0.006749987602233887,
334
+ "tpp_threshold_10_total_metric": 0.034750014543533325,
335
+ "tpp_threshold_10_intended_diff_only": 0.04100000858306885,
336
+ "tpp_threshold_10_unintended_diff_only": 0.0062499940395355225,
337
+ "tpp_threshold_20_total_metric": 0.07700005173683167,
338
+ "tpp_threshold_20_intended_diff_only": 0.07900005578994751,
339
+ "tpp_threshold_20_unintended_diff_only": 0.0020000040531158447,
340
+ "tpp_threshold_50_total_metric": 0.13475003838539124,
341
+ "tpp_threshold_50_intended_diff_only": 0.14600002765655518,
342
+ "tpp_threshold_50_unintended_diff_only": 0.01124998927116394,
343
+ "tpp_threshold_100_total_metric": 0.22825007140636444,
344
+ "tpp_threshold_100_intended_diff_only": 0.24600005149841309,
345
+ "tpp_threshold_100_unintended_diff_only": 0.017749980092048645,
346
+ "tpp_threshold_500_total_metric": 0.40375009179115295,
347
+ "tpp_threshold_500_intended_diff_only": 0.43400007486343384,
348
+ "tpp_threshold_500_unintended_diff_only": 0.030249983072280884
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.003500029444694519,
352
+ "tpp_threshold_2_intended_diff_only": 0.003000020980834961,
353
+ "tpp_threshold_2_unintended_diff_only": -0.0005000084638595581,
354
+ "tpp_threshold_5_total_metric": 0.012999996542930603,
355
+ "tpp_threshold_5_intended_diff_only": 0.008000016212463379,
356
+ "tpp_threshold_5_unintended_diff_only": -0.004999980330467224,
357
+ "tpp_threshold_10_total_metric": 0.017499983310699463,
358
+ "tpp_threshold_10_intended_diff_only": 0.014999985694885254,
359
+ "tpp_threshold_10_unintended_diff_only": -0.002499997615814209,
360
+ "tpp_threshold_20_total_metric": 0.013499990105628967,
361
+ "tpp_threshold_20_intended_diff_only": 0.019999980926513672,
362
+ "tpp_threshold_20_unintended_diff_only": 0.006499990820884705,
363
+ "tpp_threshold_50_total_metric": 0.060499995946884155,
364
+ "tpp_threshold_50_intended_diff_only": 0.06199997663497925,
365
+ "tpp_threshold_50_unintended_diff_only": 0.0014999806880950928,
366
+ "tpp_threshold_100_total_metric": 0.13300003111362457,
367
+ "tpp_threshold_100_intended_diff_only": 0.14100003242492676,
368
+ "tpp_threshold_100_unintended_diff_only": 0.008000001311302185,
369
+ "tpp_threshold_500_total_metric": 0.36525003612041473,
370
+ "tpp_threshold_500_intended_diff_only": 0.39100003242492676,
371
+ "tpp_threshold_500_unintended_diff_only": 0.025749996304512024
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.002249971032142639,
375
+ "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
376
+ "tpp_threshold_2_unintended_diff_only": -0.00024999678134918213,
377
+ "tpp_threshold_5_total_metric": 0.00824996829032898,
378
+ "tpp_threshold_5_intended_diff_only": 0.006999969482421875,
379
+ "tpp_threshold_5_unintended_diff_only": -0.0012499988079071045,
380
+ "tpp_threshold_10_total_metric": -0.0007499903440475464,
381
+ "tpp_threshold_10_intended_diff_only": 0.004999995231628418,
382
+ "tpp_threshold_10_unintended_diff_only": 0.005749985575675964,
383
+ "tpp_threshold_20_total_metric": 0.012499988079071045,
384
+ "tpp_threshold_20_intended_diff_only": 0.013999998569488525,
385
+ "tpp_threshold_20_unintended_diff_only": 0.0015000104904174805,
386
+ "tpp_threshold_50_total_metric": 0.056499943137168884,
387
+ "tpp_threshold_50_intended_diff_only": 0.0639999508857727,
388
+ "tpp_threshold_50_unintended_diff_only": 0.007500007748603821,
389
+ "tpp_threshold_100_total_metric": 0.1487499624490738,
390
+ "tpp_threshold_100_intended_diff_only": 0.16399997472763062,
391
+ "tpp_threshold_100_unintended_diff_only": 0.015250012278556824,
392
+ "tpp_threshold_500_total_metric": 0.36150000989437103,
393
+ "tpp_threshold_500_intended_diff_only": 0.3920000195503235,
394
+ "tpp_threshold_500_unintended_diff_only": 0.030500009655952454
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.025749966502189636,
398
+ "tpp_threshold_2_intended_diff_only": 0.029999971389770508,
399
+ "tpp_threshold_2_unintended_diff_only": 0.004250004887580872,
400
+ "tpp_threshold_5_total_metric": 0.05474996566772461,
401
+ "tpp_threshold_5_intended_diff_only": 0.05399996042251587,
402
+ "tpp_threshold_5_unintended_diff_only": -0.0007500052452087402,
403
+ "tpp_threshold_10_total_metric": 0.08724996447563171,
404
+ "tpp_threshold_10_intended_diff_only": 0.09099996089935303,
405
+ "tpp_threshold_10_unintended_diff_only": 0.0037499964237213135,
406
+ "tpp_threshold_20_total_metric": 0.12574999034404755,
407
+ "tpp_threshold_20_intended_diff_only": 0.13599997758865356,
408
+ "tpp_threshold_20_unintended_diff_only": 0.010249987244606018,
409
+ "tpp_threshold_50_total_metric": 0.195250004529953,
410
+ "tpp_threshold_50_intended_diff_only": 0.203000009059906,
411
+ "tpp_threshold_50_unintended_diff_only": 0.007750004529953003,
412
+ "tpp_threshold_100_total_metric": 0.27950000762939453,
413
+ "tpp_threshold_100_intended_diff_only": 0.29100000858306885,
414
+ "tpp_threshold_100_unintended_diff_only": 0.011500000953674316,
415
+ "tpp_threshold_500_total_metric": 0.3407500237226486,
416
+ "tpp_threshold_500_intended_diff_only": 0.37300002574920654,
417
+ "tpp_threshold_500_unintended_diff_only": 0.03225000202655792
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-170898-tokens-700000000/scr/saebench_scr_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,8 +69,8 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "72f7ea16-cc54-4416-97a9-3b7d73da261b",
69
- "datetime_epoch_millis": 1776884203624,
70
  "eval_result_metrics": {
71
  "scr_metrics": {
72
  "scr_dir1_threshold_2": 0.2861429048424335,
@@ -78,18 +82,18 @@
78
  "scr_dir1_threshold_10": 0.3503226092500692,
79
  "scr_metric_threshold_10": 0.2318408624917077,
80
  "scr_dir2_threshold_10": 0.22440638480937283,
81
- "scr_dir1_threshold_20": null,
82
- "scr_metric_threshold_20": null,
83
- "scr_dir2_threshold_20": null,
84
- "scr_dir1_threshold_50": null,
85
- "scr_metric_threshold_50": null,
86
- "scr_dir2_threshold_50": null,
87
- "scr_dir1_threshold_100": null,
88
- "scr_metric_threshold_100": null,
89
- "scr_dir2_threshold_100": null,
90
- "scr_dir1_threshold_500": null,
91
- "scr_metric_threshold_500": null,
92
- "scr_dir2_threshold_500": null
93
  }
94
  },
95
  "eval_result_details": [
@@ -104,18 +108,18 @@
104
  "scr_dir1_threshold_10": 0.5641028776049328,
105
  "scr_metric_threshold_10": 0.06217613139950766,
106
  "scr_dir2_threshold_10": 0.06217613139950766,
107
- "scr_dir1_threshold_20": null,
108
- "scr_metric_threshold_20": null,
109
- "scr_dir2_threshold_20": null,
110
- "scr_dir1_threshold_50": null,
111
- "scr_metric_threshold_50": null,
112
- "scr_dir2_threshold_50": null,
113
- "scr_dir1_threshold_100": null,
114
- "scr_metric_threshold_100": null,
115
- "scr_dir2_threshold_100": null,
116
- "scr_dir1_threshold_500": null,
117
- "scr_metric_threshold_500": null,
118
- "scr_dir2_threshold_500": null
119
  },
120
  {
121
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
@@ -128,18 +132,18 @@
128
  "scr_dir1_threshold_10": 0.5000002257751198,
129
  "scr_metric_threshold_10": 0.321100954221084,
130
  "scr_dir2_threshold_10": 0.321100954221084,
131
- "scr_dir1_threshold_20": null,
132
- "scr_metric_threshold_20": null,
133
- "scr_dir2_threshold_20": null,
134
- "scr_dir1_threshold_50": null,
135
- "scr_metric_threshold_50": null,
136
- "scr_dir2_threshold_50": null,
137
- "scr_dir1_threshold_100": null,
138
- "scr_metric_threshold_100": null,
139
- "scr_dir2_threshold_100": null,
140
- "scr_dir1_threshold_500": null,
141
- "scr_metric_threshold_500": null,
142
- "scr_dir2_threshold_500": null
143
  },
144
  {
145
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
@@ -152,18 +156,18 @@
152
  "scr_dir1_threshold_10": 0.4687504074535596,
153
  "scr_metric_threshold_10": 0.12311562116192198,
154
  "scr_dir2_threshold_10": 0.12311562116192198,
155
- "scr_dir1_threshold_20": null,
156
- "scr_metric_threshold_20": null,
157
- "scr_dir2_threshold_20": null,
158
- "scr_dir1_threshold_50": null,
159
- "scr_metric_threshold_50": null,
160
- "scr_dir2_threshold_50": null,
161
- "scr_dir1_threshold_100": null,
162
- "scr_metric_threshold_100": null,
163
- "scr_dir2_threshold_100": null,
164
- "scr_dir1_threshold_500": null,
165
- "scr_metric_threshold_500": null,
166
- "scr_dir2_threshold_500": null
167
  },
168
  {
169
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
@@ -176,18 +180,18 @@
176
  "scr_dir1_threshold_10": 0.24161044561475795,
177
  "scr_metric_threshold_10": 0.12738859911246028,
178
  "scr_dir2_threshold_10": 0.12738859911246028,
179
- "scr_dir1_threshold_20": null,
180
- "scr_metric_threshold_20": null,
181
- "scr_dir2_threshold_20": null,
182
- "scr_dir1_threshold_50": null,
183
- "scr_metric_threshold_50": null,
184
- "scr_dir2_threshold_50": null,
185
- "scr_dir1_threshold_100": null,
186
- "scr_metric_threshold_100": null,
187
- "scr_dir2_threshold_100": null,
188
- "scr_dir1_threshold_500": null,
189
- "scr_metric_threshold_500": null,
190
- "scr_dir2_threshold_500": null
191
  },
192
  {
193
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
@@ -200,18 +204,18 @@
200
  "scr_dir1_threshold_10": 0.08421069803000372,
201
  "scr_metric_threshold_10": 0.4603173852297345,
202
  "scr_dir2_threshold_10": 0.4603173852297345,
203
- "scr_dir1_threshold_20": null,
204
- "scr_metric_threshold_20": null,
205
- "scr_dir2_threshold_20": null,
206
- "scr_dir1_threshold_50": null,
207
- "scr_metric_threshold_50": null,
208
- "scr_dir2_threshold_50": null,
209
- "scr_dir1_threshold_100": null,
210
- "scr_metric_threshold_100": null,
211
- "scr_dir2_threshold_100": null,
212
- "scr_dir1_threshold_500": null,
213
- "scr_metric_threshold_500": null,
214
- "scr_dir2_threshold_500": null
215
  },
216
  {
217
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
@@ -224,18 +228,18 @@
224
  "scr_dir1_threshold_10": 0.4043716594522235,
225
  "scr_metric_threshold_10": 0.14503810715685644,
226
  "scr_dir2_threshold_10": 0.14503810715685644,
227
- "scr_dir1_threshold_20": null,
228
- "scr_metric_threshold_20": null,
229
- "scr_dir2_threshold_20": null,
230
- "scr_dir1_threshold_50": null,
231
- "scr_metric_threshold_50": null,
232
- "scr_dir2_threshold_50": null,
233
- "scr_dir1_threshold_100": null,
234
- "scr_metric_threshold_100": null,
235
- "scr_dir2_threshold_100": null,
236
- "scr_dir1_threshold_500": null,
237
- "scr_metric_threshold_500": null,
238
- "scr_dir2_threshold_500": null
239
  },
240
  {
241
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
@@ -248,18 +252,18 @@
248
  "scr_dir1_threshold_10": 0.3488370996994263,
249
  "scr_metric_threshold_10": 0.4248926412815665,
250
  "scr_dir2_threshold_10": 0.4248926412815665,
251
- "scr_dir1_threshold_20": null,
252
- "scr_metric_threshold_20": null,
253
- "scr_dir2_threshold_20": null,
254
- "scr_dir1_threshold_50": null,
255
- "scr_metric_threshold_50": null,
256
- "scr_dir2_threshold_50": null,
257
- "scr_dir1_threshold_100": null,
258
- "scr_metric_threshold_100": null,
259
- "scr_dir2_threshold_100": null,
260
- "scr_dir1_threshold_500": null,
261
- "scr_metric_threshold_500": null,
262
- "scr_dir2_threshold_500": null
263
  },
264
  {
265
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
@@ -272,18 +276,18 @@
272
  "scr_dir1_threshold_10": 0.19069746037053023,
273
  "scr_metric_threshold_10": 0.19069746037053023,
274
  "scr_dir2_threshold_10": 0.1312216389118512,
275
- "scr_dir1_threshold_20": null,
276
- "scr_metric_threshold_20": null,
277
- "scr_dir2_threshold_20": null,
278
- "scr_dir1_threshold_50": null,
279
- "scr_metric_threshold_50": null,
280
- "scr_dir2_threshold_50": null,
281
- "scr_dir1_threshold_100": null,
282
- "scr_metric_threshold_100": null,
283
- "scr_dir2_threshold_100": null,
284
- "scr_dir1_threshold_500": null,
285
- "scr_metric_threshold_500": null,
286
- "scr_dir2_threshold_500": null
287
  }
288
  ],
289
  "sae_bench_commit_hash": "Unknown",
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "b8f25c94-279d-4df8-bf72-7932d42c1f31",
73
+ "datetime_epoch_millis": 1777316076815,
74
  "eval_result_metrics": {
75
  "scr_metrics": {
76
  "scr_dir1_threshold_2": 0.2861429048424335,
 
82
  "scr_dir1_threshold_10": 0.3503226092500692,
83
  "scr_metric_threshold_10": 0.2318408624917077,
84
  "scr_dir2_threshold_10": 0.22440638480937283,
85
+ "scr_dir1_threshold_20": 0.3109722239202221,
86
+ "scr_metric_threshold_20": 0.22861909642889994,
87
+ "scr_dir2_threshold_20": 0.21582578096982383,
88
+ "scr_dir1_threshold_50": 0.2613805711154749,
89
+ "scr_metric_threshold_50": 0.3253183183403778,
90
+ "scr_dir2_threshold_50": 0.3187309585351685,
91
+ "scr_dir1_threshold_100": 0.01770920368435549,
92
+ "scr_metric_threshold_100": 0.32543424271638177,
93
+ "scr_dir2_threshold_100": 0.30850803853263165,
94
+ "scr_dir1_threshold_500": -0.7972597500589058,
95
+ "scr_metric_threshold_500": 0.17700663121698362,
96
+ "scr_dir2_threshold_500": 0.16320574345720348
97
  }
98
  },
99
  "eval_result_details": [
 
108
  "scr_dir1_threshold_10": 0.5641028776049328,
109
  "scr_metric_threshold_10": 0.06217613139950766,
110
  "scr_dir2_threshold_10": 0.06217613139950766,
111
+ "scr_dir1_threshold_20": 0.5128207283533913,
112
+ "scr_metric_threshold_20": 0.11658034288417893,
113
+ "scr_dir2_threshold_20": 0.11658034288417893,
114
+ "scr_dir1_threshold_50": 0.5384614208981502,
115
+ "scr_metric_threshold_50": 0.2564765999290303,
116
+ "scr_dir2_threshold_50": 0.2564765999290303,
117
+ "scr_dir1_threshold_100": 0.5641028776049328,
118
+ "scr_metric_threshold_100": 0.2564765999290303,
119
+ "scr_dir2_threshold_100": 0.2564765999290303,
120
+ "scr_dir1_threshold_500": -2.3461531701643636,
121
+ "scr_metric_threshold_500": 0.010362714302611827,
122
+ "scr_dir2_threshold_500": 0.010362714302611827
123
  },
124
  {
125
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
 
132
  "scr_dir1_threshold_10": 0.5000002257751198,
133
  "scr_metric_threshold_10": 0.321100954221084,
134
  "scr_dir2_threshold_10": 0.321100954221084,
135
+ "scr_dir1_threshold_20": 0.5000002257751198,
136
+ "scr_metric_threshold_20": 0.03363925521540607,
137
+ "scr_dir2_threshold_20": 0.03363925521540607,
138
+ "scr_dir1_threshold_50": 0.2196972536262871,
139
+ "scr_metric_threshold_50": 0.3333333940923963,
140
+ "scr_dir2_threshold_50": 0.3333333940923963,
141
+ "scr_dir1_threshold_100": -0.12121188859533107,
142
+ "scr_metric_threshold_100": 0.5412845073503276,
143
+ "scr_dir2_threshold_100": 0.5412845073503276,
144
+ "scr_dir1_threshold_500": -1.0606057185225457,
145
+ "scr_metric_threshold_500": 0.06727832815362321,
146
+ "scr_dir2_threshold_500": 0.06727832815362321
147
  },
148
  {
149
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
 
156
  "scr_dir1_threshold_10": 0.4687504074535596,
157
  "scr_metric_threshold_10": 0.12311562116192198,
158
  "scr_dir2_threshold_10": 0.12311562116192198,
159
+ "scr_dir1_threshold_20": 0.3437501746229541,
160
+ "scr_metric_threshold_20": 0.17085429167603294,
161
+ "scr_dir2_threshold_20": 0.17085429167603294,
162
+ "scr_dir1_threshold_50": 0.09374970896174313,
163
+ "scr_metric_threshold_50": 0.22361817210553975,
164
+ "scr_dir2_threshold_50": 0.22361817210553975,
165
+ "scr_dir1_threshold_100": -1.6874994179234863,
166
+ "scr_metric_threshold_100": 0.04020108028160345,
167
+ "scr_dir2_threshold_100": 0.04020108028160345,
168
+ "scr_dir1_threshold_500": -2.2812500582076516,
169
+ "scr_metric_threshold_500": -0.06783921065491268,
170
+ "scr_dir2_threshold_500": -0.06783921065491268
171
  },
172
  {
173
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
 
180
  "scr_dir1_threshold_10": 0.24161044561475795,
181
  "scr_metric_threshold_10": 0.12738859911246028,
182
  "scr_dir2_threshold_10": 0.12738859911246028,
183
+ "scr_dir1_threshold_20": -0.1208054228229802,
184
+ "scr_metric_threshold_20": 0.035031964413370725,
185
+ "scr_dir2_threshold_20": 0.035031964413370725,
186
+ "scr_dir1_threshold_50": 0.16778508723680446,
187
+ "scr_metric_threshold_50": 0.18471345922188226,
188
+ "scr_dir2_threshold_50": 0.18471345922188226,
189
+ "scr_dir1_threshold_100": 0.10738237582531437,
190
+ "scr_metric_threshold_100": -0.03184710232218929,
191
+ "scr_dir2_threshold_100": -0.03184710232218929,
192
+ "scr_dir1_threshold_500": -1.7852352483493712,
193
+ "scr_metric_threshold_500": -0.5414013089483273,
194
+ "scr_dir2_threshold_500": -0.5414013089483273
195
  },
196
  {
197
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
 
204
  "scr_dir1_threshold_10": 0.08421069803000372,
205
  "scr_metric_threshold_10": 0.4603173852297345,
206
  "scr_dir2_threshold_10": 0.4603173852297345,
207
+ "scr_dir1_threshold_20": 0.09473687842942387,
208
+ "scr_metric_threshold_20": 0.5515872218065929,
209
+ "scr_dir2_threshold_20": 0.5515872218065929,
210
+ "scr_dir1_threshold_50": -0.17894726275076694,
211
+ "scr_metric_threshold_50": 0.6507936996006726,
212
+ "scr_dir2_threshold_50": 0.6507936996006726,
213
+ "scr_dir1_threshold_100": -0.17894726275076694,
214
+ "scr_metric_threshold_100": 0.6626983066369999,
215
+ "scr_dir2_threshold_100": 0.6626983066369999,
216
+ "scr_dir1_threshold_500": -0.17894726275076694,
217
+ "scr_metric_threshold_500": 0.742063536177531,
218
+ "scr_dir2_threshold_500": 0.742063536177531
219
  },
220
  {
221
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
 
228
  "scr_dir1_threshold_10": 0.4043716594522235,
229
  "scr_metric_threshold_10": 0.14503810715685644,
230
  "scr_dir2_threshold_10": 0.14503810715685644,
231
+ "scr_dir1_threshold_20": 0.4316940424658739,
232
+ "scr_metric_threshold_20": 0.17938926174649328,
233
+ "scr_dir2_threshold_20": 0.17938926174649328,
234
+ "scr_dir1_threshold_50": 0.4316940424658739,
235
+ "scr_metric_threshold_50": 0.2671755204201627,
236
+ "scr_dir2_threshold_50": 0.2671755204201627,
237
+ "scr_dir1_threshold_100": 0.5136611915068252,
238
+ "scr_metric_threshold_100": 0.3931296780826234,
239
+ "scr_dir2_threshold_100": 0.3931296780826234,
240
+ "scr_dir1_threshold_500": 0.622950723561427,
241
+ "scr_metric_threshold_500": 0.3206106245041952,
242
+ "scr_dir2_threshold_500": 0.3206106245041952
243
  },
244
  {
245
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
 
252
  "scr_dir1_threshold_10": 0.3488370996994263,
253
  "scr_metric_threshold_10": 0.4248926412815665,
254
  "scr_dir2_threshold_10": 0.4248926412815665,
255
+ "scr_dir1_threshold_20": 0.45581383614879756,
256
+ "scr_metric_threshold_20": 0.47210310529992777,
257
+ "scr_dir2_threshold_20": 0.47210310529992777,
258
+ "scr_dir1_threshold_50": 0.5441858866203395,
259
+ "scr_metric_threshold_50": 0.41201726948797046,
260
+ "scr_dir2_threshold_50": 0.41201726948797046,
261
+ "scr_dir1_threshold_100": 0.6232557327183983,
262
+ "scr_metric_threshold_100": 0.4206008506837011,
263
+ "scr_dir2_threshold_100": 0.4206008506837011,
264
+ "scr_dir1_threshold_500": 0.2511628448544011,
265
+ "scr_metric_threshold_500": 0.48497847709352376,
266
+ "scr_dir2_threshold_500": 0.48497847709352376
267
  },
268
  {
269
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
 
276
  "scr_dir1_threshold_10": 0.19069746037053023,
277
  "scr_metric_threshold_10": 0.19069746037053023,
278
  "scr_dir2_threshold_10": 0.1312216389118512,
279
+ "scr_dir1_threshold_20": 0.26976732838919676,
280
+ "scr_metric_threshold_20": 0.26976732838919676,
281
+ "scr_dir2_threshold_20": 0.16742080471658802,
282
+ "scr_dir1_threshold_50": 0.2744184318653678,
283
+ "scr_metric_threshold_50": 0.2744184318653678,
284
+ "scr_dir2_threshold_50": 0.22171955342369326,
285
+ "scr_dir1_threshold_100": 0.3209300210889576,
286
+ "scr_metric_threshold_100": 0.3209300210889576,
287
+ "scr_dir2_threshold_100": 0.18552038761895642,
288
+ "scr_dir1_threshold_500": 0.3999998891076241,
289
+ "scr_metric_threshold_500": 0.3999998891076241,
290
+ "scr_dir2_threshold_500": 0.2895927870293829
291
  }
292
  ],
293
  "sae_bench_commit_hash": "Unknown",
gemma-2-2b-btk/k-100/seed-0/snapshots/step-170898-tokens-700000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "a9f771d2-6cd5-486e-8f85-db7c9d52ffa2",
69
- "datetime_epoch_millis": 1776884352521,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.023974995315074923,
73
- "tpp_threshold_2_intended_diff_only": 0.02619999647140503,
74
- "tpp_threshold_2_unintended_diff_only": 0.0022250011563301085,
75
- "tpp_threshold_5_total_metric": 0.07752500176429748,
76
- "tpp_threshold_5_intended_diff_only": 0.09830000400543212,
77
- "tpp_threshold_5_unintended_diff_only": 0.02077500224113464,
78
- "tpp_threshold_10_total_metric": 0.09580000191926956,
79
- "tpp_threshold_10_intended_diff_only": 0.13700000643730165,
80
- "tpp_threshold_10_unintended_diff_only": 0.041200004518032074,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.037199997901916505,
99
- "tpp_threshold_2_intended_diff_only": 0.04140000343322754,
100
- "tpp_threshold_2_unintended_diff_only": 0.004200005531311035,
101
- "tpp_threshold_5_total_metric": 0.13485001623630524,
102
- "tpp_threshold_5_intended_diff_only": 0.17480002641677855,
103
- "tpp_threshold_5_unintended_diff_only": 0.039950010180473325,
104
- "tpp_threshold_10_total_metric": 0.1674000084400177,
105
  "tpp_threshold_10_intended_diff_only": 0.24740002155303956,
106
- "tpp_threshold_10_unintended_diff_only": 0.08000001311302185,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.010749992728233338,
123
- "tpp_threshold_2_intended_diff_only": 0.01099998950958252,
124
- "tpp_threshold_2_unintended_diff_only": 0.00024999678134918213,
125
- "tpp_threshold_5_total_metric": 0.020199987292289733,
126
- "tpp_threshold_5_intended_diff_only": 0.021799981594085693,
127
- "tpp_threshold_5_unintended_diff_only": 0.0015999943017959595,
128
- "tpp_threshold_10_total_metric": 0.024199995398521423,
129
- "tpp_threshold_10_intended_diff_only": 0.02659999132156372,
130
- "tpp_threshold_10_unintended_diff_only": 0.0023999959230422974,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.003250017762184143,
190
  "tpp_threshold_10_total_metric": 0.015749961137771606,
191
  "tpp_threshold_10_intended_diff_only": 0.018999993801116943,
192
- "tpp_threshold_10_unintended_diff_only": 0.003250032663345337
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": 0.05525003373622894,
196
- "tpp_threshold_2_intended_diff_only": 0.06700003147125244,
197
- "tpp_threshold_2_unintended_diff_only": 0.011749997735023499,
198
- "tpp_threshold_5_total_metric": 0.17875003814697266,
199
  "tpp_threshold_5_intended_diff_only": 0.2420000433921814,
200
- "tpp_threshold_5_unintended_diff_only": 0.06325000524520874,
201
- "tpp_threshold_10_total_metric": 0.19749999046325684,
202
  "tpp_threshold_10_intended_diff_only": 0.3100000023841858,
203
- "tpp_threshold_10_unintended_diff_only": 0.11250001192092896
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.10600000619888306,
207
- "tpp_threshold_2_intended_diff_only": 0.11500000953674316,
208
- "tpp_threshold_2_unintended_diff_only": 0.009000003337860107,
209
- "tpp_threshold_5_total_metric": 0.2092500627040863,
210
- "tpp_threshold_5_intended_diff_only": 0.2730000615119934,
211
- "tpp_threshold_5_unintended_diff_only": 0.0637499988079071,
212
- "tpp_threshold_10_total_metric": 0.17775003612041473,
213
- "tpp_threshold_10_intended_diff_only": 0.3250000476837158,
214
- "tpp_threshold_10_unintended_diff_only": 0.1472500115633011
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.010750025510787964,
218
  "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
219
- "tpp_threshold_2_unintended_diff_only": -0.002750009298324585,
220
- "tpp_threshold_5_total_metric": 0.24574998021125793,
221
- "tpp_threshold_5_intended_diff_only": 0.3050000071525574,
222
- "tpp_threshold_5_unintended_diff_only": 0.05925002694129944,
223
- "tpp_threshold_10_total_metric": 0.27025002241134644,
224
  "tpp_threshold_10_intended_diff_only": 0.3450000286102295,
225
- "tpp_threshold_10_unintended_diff_only": 0.07475000619888306
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.0037499815225601196,
229
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
230
- "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983,
231
- "tpp_threshold_5_total_metric": 0.0207500159740448,
232
- "tpp_threshold_5_intended_diff_only": 0.03100001811981201,
233
- "tpp_threshold_5_unintended_diff_only": 0.010250002145767212,
234
- "tpp_threshold_10_total_metric": 0.1757500320672989,
235
- "tpp_threshold_10_intended_diff_only": 0.2380000352859497,
236
- "tpp_threshold_10_unintended_diff_only": 0.06225000321865082
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.012249961495399475,
242
- "tpp_threshold_2_intended_diff_only": 0.006999969482421875,
243
- "tpp_threshold_2_unintended_diff_only": -0.0052499920129776,
244
- "tpp_threshold_5_total_metric": 0.013999983668327332,
245
- "tpp_threshold_5_intended_diff_only": 0.010999977588653564,
246
- "tpp_threshold_5_unintended_diff_only": -0.003000006079673767,
247
- "tpp_threshold_10_total_metric": 0.003250032663345337,
248
- "tpp_threshold_10_intended_diff_only": 0.008000016212463379,
249
- "tpp_threshold_10_unintended_diff_only": 0.004749983549118042
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": 0.003250017762184143,
253
- "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
254
- "tpp_threshold_2_unintended_diff_only": 0.005749985575675964,
255
- "tpp_threshold_5_total_metric": 0.021250024437904358,
256
- "tpp_threshold_5_intended_diff_only": 0.02799999713897705,
257
- "tpp_threshold_5_unintended_diff_only": 0.006749972701072693,
258
- "tpp_threshold_10_total_metric": 0.03425002098083496,
259
- "tpp_threshold_10_intended_diff_only": 0.03600001335144043,
260
- "tpp_threshold_10_unintended_diff_only": 0.0017499923706054688
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.013750031590461731,
264
- "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
265
- "tpp_threshold_2_unintended_diff_only": -0.006750002503395081,
266
- "tpp_threshold_5_total_metric": 0.01825001835823059,
267
- "tpp_threshold_5_intended_diff_only": 0.017000019550323486,
268
- "tpp_threshold_5_unintended_diff_only": -0.0012499988079071045,
269
- "tpp_threshold_10_total_metric": 0.018749982118606567,
270
- "tpp_threshold_10_intended_diff_only": 0.019999980926513672,
271
- "tpp_threshold_10_unintended_diff_only": 0.0012499988079071045
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.006750002503395081,
275
- "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
276
- "tpp_threshold_2_unintended_diff_only": 0.007749989628791809,
277
- "tpp_threshold_5_total_metric": 0.0032499581575393677,
278
- "tpp_threshold_5_intended_diff_only": 0.003999948501586914,
279
- "tpp_threshold_5_unintended_diff_only": 0.0007499903440475464,
280
- "tpp_threshold_10_total_metric": 0.005999937653541565,
281
- "tpp_threshold_10_intended_diff_only": 0.008999943733215332,
282
- "tpp_threshold_10_unintended_diff_only": 0.003000006079673767
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.03124995529651642,
286
- "tpp_threshold_2_intended_diff_only": 0.030999958515167236,
287
- "tpp_threshold_2_unintended_diff_only": -0.00024999678134918213,
288
- "tpp_threshold_5_total_metric": 0.04424995183944702,
289
- "tpp_threshold_5_intended_diff_only": 0.04899996519088745,
290
- "tpp_threshold_5_unintended_diff_only": 0.00475001335144043,
291
- "tpp_threshold_10_total_metric": 0.058750003576278687,
292
- "tpp_threshold_10_intended_diff_only": 0.06000000238418579,
293
- "tpp_threshold_10_unintended_diff_only": 0.0012499988079071045
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "4d27b65a-6125-410a-a362-4f2cb046f70a",
73
+ "datetime_epoch_millis": 1777316363997,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.022100001573562622,
77
+ "tpp_threshold_2_intended_diff_only": 0.0252000093460083,
78
+ "tpp_threshold_2_unintended_diff_only": 0.0031000077724456787,
79
+ "tpp_threshold_5_total_metric": 0.07795000076293945,
80
+ "tpp_threshold_5_intended_diff_only": 0.0971000075340271,
81
+ "tpp_threshold_5_unintended_diff_only": 0.019150006771087646,
82
+ "tpp_threshold_10_total_metric": 0.09620000272989274,
83
+ "tpp_threshold_10_intended_diff_only": 0.1386000096797943,
84
+ "tpp_threshold_10_unintended_diff_only": 0.04240000694990158,
85
+ "tpp_threshold_20_total_metric": 0.13112501055002213,
86
+ "tpp_threshold_20_intended_diff_only": 0.1983000159263611,
87
+ "tpp_threshold_20_unintended_diff_only": 0.06717500537633896,
88
+ "tpp_threshold_50_total_metric": 0.1575250133872032,
89
+ "tpp_threshold_50_intended_diff_only": 0.25490002036094667,
90
+ "tpp_threshold_50_unintended_diff_only": 0.09737500697374343,
91
+ "tpp_threshold_100_total_metric": 0.1825750067830086,
92
+ "tpp_threshold_100_intended_diff_only": 0.31490001678466795,
93
+ "tpp_threshold_100_unintended_diff_only": 0.1323250100016594,
94
+ "tpp_threshold_500_total_metric": 0.24650002270936966,
95
+ "tpp_threshold_500_intended_diff_only": 0.43620003461837764,
96
+ "tpp_threshold_500_unintended_diff_only": 0.18970001190900804
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.03554998934268951,
103
+ "tpp_threshold_2_intended_diff_only": 0.041200006008148195,
104
+ "tpp_threshold_2_unintended_diff_only": 0.005650016665458679,
105
+ "tpp_threshold_5_total_metric": 0.1366000145673752,
106
+ "tpp_threshold_5_intended_diff_only": 0.17500002384185792,
107
+ "tpp_threshold_5_unintended_diff_only": 0.03840000927448273,
108
+ "tpp_threshold_10_total_metric": 0.16530000269412995,
109
  "tpp_threshold_10_intended_diff_only": 0.24740002155303956,
110
+ "tpp_threshold_10_unintended_diff_only": 0.0821000188589096,
111
+ "tpp_threshold_20_total_metric": 0.216100013256073,
112
+ "tpp_threshold_20_intended_diff_only": 0.3470000267028809,
113
+ "tpp_threshold_20_unintended_diff_only": 0.13090001344680785,
114
+ "tpp_threshold_50_total_metric": 0.21220001578330994,
115
+ "tpp_threshold_50_intended_diff_only": 0.40200003385543825,
116
+ "tpp_threshold_50_unintended_diff_only": 0.1898000180721283,
117
+ "tpp_threshold_100_total_metric": 0.1813500016927719,
118
+ "tpp_threshold_100_intended_diff_only": 0.4350000262260437,
119
+ "tpp_threshold_100_unintended_diff_only": 0.2536500245332718,
120
+ "tpp_threshold_500_total_metric": 0.11185001730918884,
121
+ "tpp_threshold_500_intended_diff_only": 0.4670000433921814,
122
+ "tpp_threshold_500_unintended_diff_only": 0.35515002608299256
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.00865001380443573,
127
+ "tpp_threshold_2_intended_diff_only": 0.009200012683868409,
128
+ "tpp_threshold_2_unintended_diff_only": 0.0005499988794326783,
129
+ "tpp_threshold_5_total_metric": 0.019299986958503722,
130
+ "tpp_threshold_5_intended_diff_only": 0.01919999122619629,
131
+ "tpp_threshold_5_unintended_diff_only": -9.999573230743409e-05,
132
+ "tpp_threshold_10_total_metric": 0.027100002765655516,
133
+ "tpp_threshold_10_intended_diff_only": 0.029799997806549072,
134
+ "tpp_threshold_10_unintended_diff_only": 0.0026999950408935548,
135
+ "tpp_threshold_20_total_metric": 0.046150007843971254,
136
+ "tpp_threshold_20_intended_diff_only": 0.04960000514984131,
137
+ "tpp_threshold_20_unintended_diff_only": 0.003449997305870056,
138
+ "tpp_threshold_50_total_metric": 0.1028500109910965,
139
+ "tpp_threshold_50_intended_diff_only": 0.10780000686645508,
140
+ "tpp_threshold_50_unintended_diff_only": 0.004949995875358581,
141
+ "tpp_threshold_100_total_metric": 0.18380001187324524,
142
+ "tpp_threshold_100_intended_diff_only": 0.19480000734329223,
143
+ "tpp_threshold_100_unintended_diff_only": 0.010999995470046996,
144
+ "tpp_threshold_500_total_metric": 0.38115002810955045,
145
+ "tpp_threshold_500_intended_diff_only": 0.40540002584457396,
146
+ "tpp_threshold_500_unintended_diff_only": 0.0242499977350235
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.003250017762184143,
194
  "tpp_threshold_10_total_metric": 0.015749961137771606,
195
  "tpp_threshold_10_intended_diff_only": 0.018999993801116943,
196
+ "tpp_threshold_10_unintended_diff_only": 0.003250032663345337,
197
+ "tpp_threshold_20_total_metric": 0.21974997222423553,
198
+ "tpp_threshold_20_intended_diff_only": 0.32999998331069946,
199
+ "tpp_threshold_20_unintended_diff_only": 0.11025001108646393,
200
+ "tpp_threshold_50_total_metric": 0.2395000010728836,
201
+ "tpp_threshold_50_intended_diff_only": 0.3920000195503235,
202
+ "tpp_threshold_50_unintended_diff_only": 0.15250001847743988,
203
+ "tpp_threshold_100_total_metric": 0.2202499508857727,
204
+ "tpp_threshold_100_intended_diff_only": 0.43699997663497925,
205
+ "tpp_threshold_100_unintended_diff_only": 0.21675002574920654,
206
+ "tpp_threshold_500_total_metric": 0.1302500069141388,
207
+ "tpp_threshold_500_intended_diff_only": 0.45600003004074097,
208
+ "tpp_threshold_500_unintended_diff_only": 0.3257500231266022
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.057999953627586365,
212
+ "tpp_threshold_2_intended_diff_only": 0.06599998474121094,
213
+ "tpp_threshold_2_unintended_diff_only": 0.008000031113624573,
214
+ "tpp_threshold_5_total_metric": 0.18100003898143768,
215
  "tpp_threshold_5_intended_diff_only": 0.2420000433921814,
216
+ "tpp_threshold_5_unintended_diff_only": 0.06100000441074371,
217
+ "tpp_threshold_10_total_metric": 0.19049999117851257,
218
  "tpp_threshold_10_intended_diff_only": 0.3100000023841858,
219
+ "tpp_threshold_10_unintended_diff_only": 0.11950001120567322,
220
+ "tpp_threshold_20_total_metric": 0.21650002896785736,
221
+ "tpp_threshold_20_intended_diff_only": 0.34400004148483276,
222
+ "tpp_threshold_20_unintended_diff_only": 0.1275000125169754,
223
+ "tpp_threshold_50_total_metric": 0.2004999965429306,
224
+ "tpp_threshold_50_intended_diff_only": 0.37700003385543823,
225
+ "tpp_threshold_50_unintended_diff_only": 0.17650003731250763,
226
+ "tpp_threshold_100_total_metric": 0.16725000739097595,
227
+ "tpp_threshold_100_intended_diff_only": 0.4150000214576721,
228
+ "tpp_threshold_100_unintended_diff_only": 0.24775001406669617,
229
+ "tpp_threshold_500_total_metric": 0.09400002658367157,
230
+ "tpp_threshold_500_intended_diff_only": 0.4590000510215759,
231
+ "tpp_threshold_500_unintended_diff_only": 0.36500002443790436
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.09425003826618195,
235
+ "tpp_threshold_2_intended_diff_only": 0.11200004816055298,
236
+ "tpp_threshold_2_unintended_diff_only": 0.017750009894371033,
237
+ "tpp_threshold_5_total_metric": 0.2120000123977661,
238
+ "tpp_threshold_5_intended_diff_only": 0.2720000147819519,
239
+ "tpp_threshold_5_unintended_diff_only": 0.06000000238418579,
240
+ "tpp_threshold_10_total_metric": 0.17725001275539398,
241
+ "tpp_threshold_10_intended_diff_only": 0.32600003480911255,
242
+ "tpp_threshold_10_unintended_diff_only": 0.14875002205371857,
243
+ "tpp_threshold_20_total_metric": 0.138000026345253,
244
+ "tpp_threshold_20_intended_diff_only": 0.359000027179718,
245
+ "tpp_threshold_20_unintended_diff_only": 0.22100000083446503,
246
+ "tpp_threshold_50_total_metric": 0.11350005865097046,
247
+ "tpp_threshold_50_intended_diff_only": 0.37900006771087646,
248
+ "tpp_threshold_50_unintended_diff_only": 0.265500009059906,
249
+ "tpp_threshold_100_total_metric": 0.08675001561641693,
250
+ "tpp_threshold_100_intended_diff_only": 0.409000039100647,
251
+ "tpp_threshold_100_unintended_diff_only": 0.32225002348423004,
252
+ "tpp_threshold_500_total_metric": 0.054749995470047,
253
+ "tpp_threshold_500_intended_diff_only": 0.4520000219345093,
254
+ "tpp_threshold_500_unintended_diff_only": 0.3972500264644623
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.008249998092651367,
258
  "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0002499818801879883,
260
+ "tpp_threshold_5_total_metric": 0.2525000423192978,
261
+ "tpp_threshold_5_intended_diff_only": 0.3060000538825989,
262
+ "tpp_threshold_5_unintended_diff_only": 0.053500011563301086,
263
+ "tpp_threshold_10_total_metric": 0.2680000215768814,
264
  "tpp_threshold_10_intended_diff_only": 0.3450000286102295,
265
+ "tpp_threshold_10_unintended_diff_only": 0.07700000703334808,
266
+ "tpp_threshold_20_total_metric": 0.28675003349781036,
267
+ "tpp_threshold_20_intended_diff_only": 0.41200006008148193,
268
+ "tpp_threshold_20_unintended_diff_only": 0.12525002658367157,
269
+ "tpp_threshold_50_total_metric": 0.2657500058412552,
270
+ "tpp_threshold_50_intended_diff_only": 0.47600001096725464,
271
+ "tpp_threshold_50_unintended_diff_only": 0.21025000512599945,
272
+ "tpp_threshold_100_total_metric": 0.2095000147819519,
273
+ "tpp_threshold_100_intended_diff_only": 0.4830000400543213,
274
+ "tpp_threshold_100_unintended_diff_only": 0.2735000252723694,
275
+ "tpp_threshold_500_total_metric": 0.14275000989437103,
276
+ "tpp_threshold_500_intended_diff_only": 0.4920000433921814,
277
+ "tpp_threshold_500_unintended_diff_only": 0.34925003349781036
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.0070000141859054565,
281
+ "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224,
283
+ "tpp_threshold_5_total_metric": 0.01774999499320984,
284
+ "tpp_threshold_5_intended_diff_only": 0.03200000524520874,
285
+ "tpp_threshold_5_unintended_diff_only": 0.014250010251998901,
286
+ "tpp_threshold_10_total_metric": 0.17500002682209015,
287
+ "tpp_threshold_10_intended_diff_only": 0.23700004816055298,
288
+ "tpp_threshold_10_unintended_diff_only": 0.06200002133846283,
289
+ "tpp_threshold_20_total_metric": 0.21950000524520874,
290
+ "tpp_threshold_20_intended_diff_only": 0.2900000214576721,
291
+ "tpp_threshold_20_unintended_diff_only": 0.07050001621246338,
292
+ "tpp_threshold_50_total_metric": 0.24175001680850983,
293
+ "tpp_threshold_50_intended_diff_only": 0.38600003719329834,
294
+ "tpp_threshold_50_unintended_diff_only": 0.1442500203847885,
295
+ "tpp_threshold_100_total_metric": 0.22300001978874207,
296
+ "tpp_threshold_100_intended_diff_only": 0.4310000538825989,
297
+ "tpp_threshold_100_unintended_diff_only": 0.2080000340938568,
298
+ "tpp_threshold_500_total_metric": 0.13750004768371582,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.3385000228881836
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.005250021815299988,
306
+ "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
307
+ "tpp_threshold_2_unintended_diff_only": 0.002749994397163391,
308
+ "tpp_threshold_5_total_metric": 0.0065000057220458984,
309
+ "tpp_threshold_5_intended_diff_only": 0.008000016212463379,
310
+ "tpp_threshold_5_unintended_diff_only": 0.0015000104904174805,
311
+ "tpp_threshold_10_total_metric": 0.004749968647956848,
312
+ "tpp_threshold_10_intended_diff_only": 0.006999969482421875,
313
+ "tpp_threshold_10_unintended_diff_only": 0.002250000834465027,
314
+ "tpp_threshold_20_total_metric": 0.0169999897480011,
315
+ "tpp_threshold_20_intended_diff_only": 0.018999993801116943,
316
+ "tpp_threshold_20_unintended_diff_only": 0.0020000040531158447,
317
+ "tpp_threshold_50_total_metric": 0.058500006794929504,
318
+ "tpp_threshold_50_intended_diff_only": 0.0559999942779541,
319
+ "tpp_threshold_50_unintended_diff_only": -0.002500012516975403,
320
+ "tpp_threshold_100_total_metric": 0.1300000250339508,
321
+ "tpp_threshold_100_intended_diff_only": 0.13700002431869507,
322
+ "tpp_threshold_100_unintended_diff_only": 0.006999999284744263,
323
+ "tpp_threshold_500_total_metric": 0.4007500261068344,
324
+ "tpp_threshold_500_intended_diff_only": 0.4150000214576721,
325
+ "tpp_threshold_500_unintended_diff_only": 0.014249995350837708
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.012000054121017456,
329
+ "tpp_threshold_2_intended_diff_only": 0.010000050067901611,
330
+ "tpp_threshold_2_unintended_diff_only": -0.0020000040531158447,
331
+ "tpp_threshold_5_total_metric": 0.017000064253807068,
332
+ "tpp_threshold_5_intended_diff_only": 0.024000048637390137,
333
+ "tpp_threshold_5_unintended_diff_only": 0.006999984383583069,
334
+ "tpp_threshold_10_total_metric": 0.034000054001808167,
335
+ "tpp_threshold_10_intended_diff_only": 0.03900003433227539,
336
+ "tpp_threshold_10_unintended_diff_only": 0.004999980330467224,
337
+ "tpp_threshold_20_total_metric": 0.07200005650520325,
338
+ "tpp_threshold_20_intended_diff_only": 0.07500004768371582,
339
+ "tpp_threshold_20_unintended_diff_only": 0.0029999911785125732,
340
+ "tpp_threshold_50_total_metric": 0.1272500604391098,
341
+ "tpp_threshold_50_intended_diff_only": 0.1390000581741333,
342
+ "tpp_threshold_50_unintended_diff_only": 0.011749997735023499,
343
+ "tpp_threshold_100_total_metric": 0.20150001347064972,
344
+ "tpp_threshold_100_intended_diff_only": 0.2160000205039978,
345
+ "tpp_threshold_100_unintended_diff_only": 0.014500007033348083,
346
+ "tpp_threshold_500_total_metric": 0.4147500544786453,
347
+ "tpp_threshold_500_intended_diff_only": 0.43300002813339233,
348
+ "tpp_threshold_500_unintended_diff_only": 0.01824997365474701
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.0045000165700912476,
352
+ "tpp_threshold_2_intended_diff_only": 0.003000020980834961,
353
+ "tpp_threshold_2_unintended_diff_only": -0.0014999955892562866,
354
+ "tpp_threshold_5_total_metric": 0.014499977231025696,
355
+ "tpp_threshold_5_intended_diff_only": 0.009999990463256836,
356
+ "tpp_threshold_5_unintended_diff_only": -0.00449998676776886,
357
+ "tpp_threshold_10_total_metric": 0.019250020384788513,
358
+ "tpp_threshold_10_intended_diff_only": 0.017000019550323486,
359
+ "tpp_threshold_10_unintended_diff_only": -0.002250000834465027,
360
+ "tpp_threshold_20_total_metric": 0.00875002145767212,
361
+ "tpp_threshold_20_intended_diff_only": 0.012000024318695068,
362
+ "tpp_threshold_20_unintended_diff_only": 0.0032500028610229492,
363
+ "tpp_threshold_50_total_metric": 0.05500000715255737,
364
+ "tpp_threshold_50_intended_diff_only": 0.0559999942779541,
365
+ "tpp_threshold_50_unintended_diff_only": 0.0009999871253967285,
366
+ "tpp_threshold_100_total_metric": 0.14850005507469177,
367
+ "tpp_threshold_100_intended_diff_only": 0.15900003910064697,
368
+ "tpp_threshold_100_unintended_diff_only": 0.0104999840259552,
369
+ "tpp_threshold_500_total_metric": 0.36250004172325134,
370
+ "tpp_threshold_500_intended_diff_only": 0.39000004529953003,
371
+ "tpp_threshold_500_unintended_diff_only": 0.027500003576278687
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.001999974250793457,
375
+ "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
376
+ "tpp_threshold_2_unintended_diff_only": 0.0,
377
+ "tpp_threshold_5_total_metric": 0.011749938130378723,
378
+ "tpp_threshold_5_intended_diff_only": 0.008999943733215332,
379
+ "tpp_threshold_5_unintended_diff_only": -0.002749994397163391,
380
+ "tpp_threshold_10_total_metric": 0.005499988794326782,
381
+ "tpp_threshold_10_intended_diff_only": 0.010999977588653564,
382
+ "tpp_threshold_10_unintended_diff_only": 0.005499988794326782,
383
+ "tpp_threshold_20_total_metric": 0.025499969720840454,
384
+ "tpp_threshold_20_intended_diff_only": 0.02599996328353882,
385
+ "tpp_threshold_20_unintended_diff_only": 0.0004999935626983643,
386
+ "tpp_threshold_50_total_metric": 0.07799996435642242,
387
+ "tpp_threshold_50_intended_diff_only": 0.08599996566772461,
388
+ "tpp_threshold_50_unintended_diff_only": 0.008000001311302185,
389
+ "tpp_threshold_100_total_metric": 0.16124998033046722,
390
+ "tpp_threshold_100_intended_diff_only": 0.17299997806549072,
391
+ "tpp_threshold_100_unintended_diff_only": 0.011749997735023499,
392
+ "tpp_threshold_500_total_metric": 0.38324999809265137,
393
+ "tpp_threshold_500_intended_diff_only": 0.41600000858306885,
394
+ "tpp_threshold_500_unintended_diff_only": 0.03275001049041748
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.0195000022649765,
398
+ "tpp_threshold_2_intended_diff_only": 0.023000001907348633,
399
+ "tpp_threshold_2_unintended_diff_only": 0.0034999996423721313,
400
+ "tpp_threshold_5_total_metric": 0.04674994945526123,
401
+ "tpp_threshold_5_intended_diff_only": 0.04499995708465576,
402
+ "tpp_threshold_5_unintended_diff_only": -0.0017499923706054688,
403
+ "tpp_threshold_10_total_metric": 0.07199998199939728,
404
+ "tpp_threshold_10_intended_diff_only": 0.07499998807907104,
405
+ "tpp_threshold_10_unintended_diff_only": 0.003000006079673767,
406
+ "tpp_threshold_20_total_metric": 0.10750000178813934,
407
+ "tpp_threshold_20_intended_diff_only": 0.11599999666213989,
408
+ "tpp_threshold_20_unintended_diff_only": 0.00849999487400055,
409
+ "tpp_threshold_50_total_metric": 0.19550001621246338,
410
+ "tpp_threshold_50_intended_diff_only": 0.20200002193450928,
411
+ "tpp_threshold_50_unintended_diff_only": 0.0065000057220458984,
412
+ "tpp_threshold_100_total_metric": 0.2777499854564667,
413
+ "tpp_threshold_100_intended_diff_only": 0.2889999747276306,
414
+ "tpp_threshold_100_unintended_diff_only": 0.01124998927116394,
415
+ "tpp_threshold_500_total_metric": 0.34450002014636993,
416
+ "tpp_threshold_500_intended_diff_only": 0.37300002574920654,
417
+ "tpp_threshold_500_unintended_diff_only": 0.02850000560283661
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-183105-tokens-750000000/scr/saebench_scr_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,8 +69,8 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "1b2531f8-5985-405b-b248-aadde8f689d2",
69
- "datetime_epoch_millis": 1776884434552,
70
  "eval_result_metrics": {
71
  "scr_metrics": {
72
  "scr_dir1_threshold_2": 0.29073645881291527,
@@ -78,18 +82,18 @@
78
  "scr_dir1_threshold_10": 0.24672639960801526,
79
  "scr_metric_threshold_10": 0.21722385233565517,
80
  "scr_dir2_threshold_10": 0.20410166840481794,
81
- "scr_dir1_threshold_20": null,
82
- "scr_metric_threshold_20": null,
83
- "scr_dir2_threshold_20": null,
84
- "scr_dir1_threshold_50": null,
85
- "scr_metric_threshold_50": null,
86
- "scr_dir2_threshold_50": null,
87
- "scr_dir1_threshold_100": null,
88
- "scr_metric_threshold_100": null,
89
- "scr_dir2_threshold_100": null,
90
- "scr_dir1_threshold_500": null,
91
- "scr_metric_threshold_500": null,
92
- "scr_dir2_threshold_500": null
93
  }
94
  },
95
  "eval_result_details": [
@@ -104,18 +108,18 @@
104
  "scr_dir1_threshold_10": 0.25641074625770754,
105
  "scr_metric_threshold_10": 0.05440405706850797,
106
  "scr_dir2_threshold_10": 0.05440405706850797,
107
- "scr_dir1_threshold_20": null,
108
- "scr_metric_threshold_20": null,
109
- "scr_dir2_threshold_20": null,
110
- "scr_dir1_threshold_50": null,
111
- "scr_metric_threshold_50": null,
112
- "scr_dir2_threshold_50": null,
113
- "scr_dir1_threshold_100": null,
114
- "scr_metric_threshold_100": null,
115
- "scr_dir2_threshold_100": null,
116
- "scr_dir1_threshold_500": null,
117
- "scr_metric_threshold_500": null,
118
- "scr_dir2_threshold_500": null
119
  },
120
  {
121
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
@@ -128,18 +132,18 @@
128
  "scr_dir1_threshold_10": 0.26515165540575625,
129
  "scr_metric_threshold_10": 0.2966360744784594,
130
  "scr_dir2_threshold_10": 0.2966360744784594,
131
- "scr_dir1_threshold_20": null,
132
- "scr_metric_threshold_20": null,
133
- "scr_dir2_threshold_20": null,
134
- "scr_dir1_threshold_50": null,
135
- "scr_metric_threshold_50": null,
136
- "scr_dir2_threshold_50": null,
137
- "scr_dir1_threshold_100": null,
138
- "scr_metric_threshold_100": null,
139
- "scr_dir2_threshold_100": null,
140
- "scr_dir1_threshold_500": null,
141
- "scr_metric_threshold_500": null,
142
- "scr_dir2_threshold_500": null
143
  },
144
  {
145
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
@@ -152,18 +156,18 @@
152
  "scr_dir1_threshold_10": 0.4843752037267798,
153
  "scr_metric_threshold_10": 0.09547734102822189,
154
  "scr_dir2_threshold_10": 0.09547734102822189,
155
- "scr_dir1_threshold_20": null,
156
- "scr_metric_threshold_20": null,
157
- "scr_dir2_threshold_20": null,
158
- "scr_dir1_threshold_50": null,
159
- "scr_metric_threshold_50": null,
160
- "scr_dir2_threshold_50": null,
161
- "scr_dir1_threshold_100": null,
162
- "scr_metric_threshold_100": null,
163
- "scr_dir2_threshold_100": null,
164
- "scr_dir1_threshold_500": null,
165
- "scr_metric_threshold_500": null,
166
- "scr_dir2_threshold_500": null
167
  },
168
  {
169
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
@@ -176,18 +180,18 @@
176
  "scr_dir1_threshold_10": -0.1409397933038777,
177
  "scr_metric_threshold_10": 0.08917196243161127,
178
  "scr_dir2_threshold_10": 0.08917196243161127,
179
- "scr_dir1_threshold_20": null,
180
- "scr_metric_threshold_20": null,
181
- "scr_dir2_threshold_20": null,
182
- "scr_dir1_threshold_50": null,
183
- "scr_metric_threshold_50": null,
184
- "scr_dir2_threshold_50": null,
185
- "scr_dir1_threshold_100": null,
186
- "scr_metric_threshold_100": null,
187
- "scr_dir2_threshold_100": null,
188
- "scr_dir1_threshold_500": null,
189
- "scr_metric_threshold_500": null,
190
- "scr_dir2_threshold_500": null
191
  },
192
  {
193
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
@@ -200,18 +204,18 @@
200
  "scr_dir1_threshold_10": 0.08421069803000372,
201
  "scr_metric_threshold_10": 0.4722222287923982,
202
  "scr_dir2_threshold_10": 0.4722222287923982,
203
- "scr_dir1_threshold_20": null,
204
- "scr_metric_threshold_20": null,
205
- "scr_dir2_threshold_20": null,
206
- "scr_dir1_threshold_50": null,
207
- "scr_metric_threshold_50": null,
208
- "scr_dir2_threshold_50": null,
209
- "scr_dir1_threshold_100": null,
210
- "scr_metric_threshold_100": null,
211
- "scr_dir2_threshold_100": null,
212
- "scr_dir1_threshold_500": null,
213
- "scr_metric_threshold_500": null,
214
- "scr_dir2_threshold_500": null
215
  },
216
  {
217
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
@@ -224,18 +228,18 @@
224
  "scr_dir1_threshold_10": 0.4153004823742847,
225
  "scr_metric_threshold_10": 0.15648856785294307,
226
  "scr_dir2_threshold_10": 0.15648856785294307,
227
- "scr_dir1_threshold_20": null,
228
- "scr_metric_threshold_20": null,
229
- "scr_dir2_threshold_20": null,
230
- "scr_dir1_threshold_50": null,
231
- "scr_metric_threshold_50": null,
232
- "scr_dir2_threshold_50": null,
233
- "scr_dir1_threshold_100": null,
234
- "scr_metric_threshold_100": null,
235
- "scr_dir2_threshold_100": null,
236
- "scr_dir1_threshold_500": null,
237
- "scr_metric_threshold_500": null,
238
- "scr_dir2_threshold_500": null
239
  },
240
  {
241
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
@@ -248,18 +252,18 @@
248
  "scr_dir1_threshold_10": 0.4093022598196558,
249
  "scr_metric_threshold_10": 0.3733906424792874,
250
  "scr_dir2_threshold_10": 0.3733906424792874,
251
- "scr_dir1_threshold_20": null,
252
- "scr_metric_threshold_20": null,
253
- "scr_dir2_threshold_20": null,
254
- "scr_dir1_threshold_50": null,
255
- "scr_metric_threshold_50": null,
256
- "scr_dir2_threshold_50": null,
257
- "scr_dir1_threshold_100": null,
258
- "scr_metric_threshold_100": null,
259
- "scr_dir2_threshold_100": null,
260
- "scr_dir1_threshold_500": null,
261
- "scr_metric_threshold_500": null,
262
- "scr_dir2_threshold_500": null
263
  },
264
  {
265
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
@@ -272,18 +276,18 @@
272
  "scr_dir1_threshold_10": 0.19999994455381204,
273
  "scr_metric_threshold_10": 0.19999994455381204,
274
  "scr_dir2_threshold_10": 0.0950224731071144,
275
- "scr_dir1_threshold_20": null,
276
- "scr_metric_threshold_20": null,
277
- "scr_dir2_threshold_20": null,
278
- "scr_dir1_threshold_50": null,
279
- "scr_metric_threshold_50": null,
280
- "scr_dir2_threshold_50": null,
281
- "scr_dir1_threshold_100": null,
282
- "scr_metric_threshold_100": null,
283
- "scr_dir2_threshold_100": null,
284
- "scr_dir1_threshold_500": null,
285
- "scr_metric_threshold_500": null,
286
- "scr_dir2_threshold_500": null
287
  }
288
  ],
289
  "sae_bench_commit_hash": "Unknown",
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "dea99386-a55b-43a6-b020-692d06d95f2b",
73
+ "datetime_epoch_millis": 1777316345064,
74
  "eval_result_metrics": {
75
  "scr_metrics": {
76
  "scr_dir1_threshold_2": 0.29073645881291527,
 
82
  "scr_dir1_threshold_10": 0.24672639960801526,
83
  "scr_metric_threshold_10": 0.21722385233565517,
84
  "scr_dir2_threshold_10": 0.20410166840481794,
85
+ "scr_dir1_threshold_20": 0.12416420833329343,
86
+ "scr_metric_threshold_20": 0.2561844355577999,
87
+ "scr_dir2_threshold_20": 0.23883465463813552,
88
+ "scr_dir1_threshold_50": 0.10869257614240875,
89
+ "scr_metric_threshold_50": 0.30557380609423856,
90
+ "scr_dir2_threshold_50": 0.29326717124636625,
91
+ "scr_dir1_threshold_100": 0.0319769995665256,
92
+ "scr_metric_threshold_100": 0.3304550201245743,
93
+ "scr_dir2_threshold_100": 0.3197189525437431,
94
+ "scr_dir1_threshold_500": -0.8822934937256783,
95
+ "scr_metric_threshold_500": 0.1986650188748059,
96
+ "scr_dir2_threshold_500": 0.17055020921041797
97
  }
98
  },
99
  "eval_result_details": [
 
108
  "scr_dir1_threshold_10": 0.25641074625770754,
109
  "scr_metric_threshold_10": 0.05440405706850797,
110
  "scr_dir2_threshold_10": 0.05440405706850797,
111
+ "scr_dir1_threshold_20": 0.16666679402700396,
112
+ "scr_metric_threshold_20": 0.1062176285815671,
113
+ "scr_dir2_threshold_20": 0.1062176285815671,
114
+ "scr_dir1_threshold_50": 0.5897435701496917,
115
+ "scr_metric_threshold_50": 0.22797925140897027,
116
+ "scr_dir2_threshold_50": 0.22797925140897027,
117
+ "scr_dir1_threshold_100": 0.5000003820810118,
118
+ "scr_metric_threshold_100": 0.33937815993376164,
119
+ "scr_dir2_threshold_100": 0.33937815993376164,
120
+ "scr_dir1_threshold_500": -2.217948179116522,
121
+ "scr_metric_threshold_500": 0.020725428605223653,
122
+ "scr_dir2_threshold_500": 0.020725428605223653
123
  },
124
  {
125
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
 
132
  "scr_dir1_threshold_10": 0.26515165540575625,
133
  "scr_metric_threshold_10": 0.2966360744784594,
134
  "scr_dir2_threshold_10": 0.2966360744784594,
135
+ "scr_dir1_threshold_20": 0.31060605718522544,
136
+ "scr_metric_threshold_20": 0.22018355312924365,
137
+ "scr_dir2_threshold_20": 0.22018355312924365,
138
+ "scr_dir1_threshold_50": 0.46969714073872715,
139
+ "scr_metric_threshold_50": 0.38532103569898746,
140
+ "scr_dir2_threshold_50": 0.38532103569898746,
141
+ "scr_dir1_threshold_100": 0.37878788562954907,
142
+ "scr_metric_threshold_100": 0.6391436617664482,
143
+ "scr_dir2_threshold_100": 0.6391436617664482,
144
+ "scr_dir1_threshold_500": -1.446969262523633,
145
+ "scr_metric_threshold_500": 0.11620790536168352,
146
+ "scr_dir2_threshold_500": 0.11620790536168352
147
  },
148
  {
149
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
 
156
  "scr_dir1_threshold_10": 0.4843752037267798,
157
  "scr_metric_threshold_10": 0.09547734102822189,
158
  "scr_dir2_threshold_10": 0.09547734102822189,
159
+ "scr_dir1_threshold_20": 0.1093754365573853,
160
+ "scr_metric_threshold_20": 0.1783920316689313,
161
+ "scr_dir2_threshold_20": 0.1783920316689313,
162
+ "scr_dir1_threshold_50": -0.921875087311477,
163
+ "scr_metric_threshold_50": 0.20854269211974302,
164
+ "scr_dir2_threshold_50": 0.20854269211974302,
165
+ "scr_dir1_threshold_100": -1.1250002328306055,
166
+ "scr_metric_threshold_100": 0.017588010063299225,
167
+ "scr_dir2_threshold_100": 0.017588010063299225,
168
+ "scr_dir1_threshold_500": -2.9218741559890553,
169
+ "scr_metric_threshold_500": 0.0,
170
+ "scr_dir2_threshold_500": 0.0
171
  },
172
  {
173
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
 
180
  "scr_dir1_threshold_10": -0.1409397933038777,
181
  "scr_metric_threshold_10": 0.08917196243161127,
182
  "scr_dir2_threshold_10": 0.08917196243161127,
183
+ "scr_dir1_threshold_20": -0.9060406711723514,
184
+ "scr_metric_threshold_20": 0.08917196243161127,
185
+ "scr_dir2_threshold_20": 0.08917196243161127,
186
+ "scr_dir1_threshold_50": -0.40939633291396726,
187
+ "scr_metric_threshold_50": 0.01910841325227608,
188
+ "scr_dir2_threshold_50": 0.01910841325227608,
189
+ "scr_dir1_threshold_100": -0.6510067785287252,
190
+ "scr_metric_threshold_100": -0.09872616905774931,
191
+ "scr_dir2_threshold_100": -0.09872616905774931,
192
+ "scr_dir1_threshold_500": -1.8389266362776298,
193
+ "scr_metric_threshold_500": -0.5350317745896676,
194
+ "scr_dir2_threshold_500": -0.5350317745896676
195
  },
196
  {
197
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
 
204
  "scr_dir1_threshold_10": 0.08421069803000372,
205
  "scr_metric_threshold_10": 0.4722222287923982,
206
  "scr_dir2_threshold_10": 0.4722222287923982,
207
+ "scr_dir1_threshold_20": 0.09473687842942387,
208
+ "scr_metric_threshold_20": 0.515872927644938,
209
+ "scr_dir2_threshold_20": 0.515872927644938,
210
+ "scr_dir1_threshold_50": -0.11578955293692479,
211
+ "scr_metric_threshold_50": 0.6428570583834513,
212
+ "scr_dir2_threshold_50": 0.6428570583834513,
213
+ "scr_dir1_threshold_100": -0.18947375685884774,
214
+ "scr_metric_threshold_100": 0.654761901946115,
215
+ "scr_dir2_threshold_100": 0.654761901946115,
216
+ "scr_dir1_threshold_500": -0.12105264313663487,
217
+ "scr_metric_threshold_500": 0.7301586926148672,
218
+ "scr_dir2_threshold_500": 0.7301586926148672
219
  },
220
  {
221
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
 
228
  "scr_dir1_threshold_10": 0.4153004823742847,
229
  "scr_metric_threshold_10": 0.15648856785294307,
230
  "scr_dir2_threshold_10": 0.15648856785294307,
231
+ "scr_dir1_threshold_20": 0.4644808369405549,
232
+ "scr_metric_threshold_20": 0.20992367193697561,
233
+ "scr_dir2_threshold_20": 0.20992367193697561,
234
+ "scr_dir1_threshold_50": 0.4153004823742847,
235
+ "scr_metric_threshold_50": 0.29007644181233594,
236
+ "scr_dir2_threshold_50": 0.29007644181233594,
237
+ "scr_dir1_threshold_100": 0.4262296310048433,
238
+ "scr_metric_threshold_100": 0.36641223978991866,
239
+ "scr_dir2_threshold_100": 0.36641223978991866,
240
+ "scr_dir1_threshold_500": 0.5628415460730956,
241
+ "scr_metric_threshold_500": 0.38549618928431434,
242
+ "scr_dir2_threshold_500": 0.38549618928431434
243
  },
244
  {
245
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
 
252
  "scr_dir1_threshold_10": 0.4093022598196558,
253
  "scr_metric_threshold_10": 0.3733906424792874,
254
  "scr_dir2_threshold_10": 0.3733906424792874,
255
+ "scr_dir1_threshold_20": 0.4744185221266269,
256
+ "scr_metric_threshold_20": 0.4506438964966536,
257
+ "scr_dir2_threshold_20": 0.4506438964966536,
258
+ "scr_dir1_threshold_50": 0.548837266037944,
259
+ "scr_metric_threshold_50": 0.3776824330771527,
260
+ "scr_dir2_threshold_50": 0.3776824330771527,
261
+ "scr_dir1_threshold_100": 0.5860463607627397,
262
+ "scr_metric_threshold_100": 0.3948498512825616,
263
+ "scr_dir2_threshold_100": 0.3948498512825616,
264
+ "scr_dir1_threshold_500": 0.4744185221266269,
265
+ "scr_metric_threshold_500": 0.4206008506837011,
266
+ "scr_dir2_threshold_500": 0.4206008506837011
267
  },
268
  {
269
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
 
276
  "scr_dir1_threshold_10": 0.19999994455381204,
277
  "scr_metric_threshold_10": 0.19999994455381204,
278
  "scr_dir2_threshold_10": 0.0950224731071144,
279
+ "scr_dir1_threshold_20": 0.27906981257247854,
280
+ "scr_metric_threshold_20": 0.27906981257247854,
281
+ "scr_dir2_threshold_20": 0.14027156521516332,
282
+ "scr_dir1_threshold_50": 0.29302312300099165,
283
+ "scr_metric_threshold_50": 0.29302312300099165,
284
+ "scr_dir2_threshold_50": 0.19457004421801274,
285
+ "scr_dir1_threshold_100": 0.3302325052722394,
286
+ "scr_metric_threshold_100": 0.3302325052722394,
287
+ "scr_dir2_threshold_100": 0.2443439646255898,
288
+ "scr_dir1_threshold_500": 0.4511628590383247,
289
+ "scr_metric_threshold_500": 0.4511628590383247,
290
+ "scr_dir2_threshold_500": 0.2262443817232214
291
  }
292
  ],
293
  "sae_bench_commit_hash": "Unknown",
gemma-2-2b-btk/k-100/seed-0/snapshots/step-183105-tokens-750000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "1a07b655-3fb7-411e-9cba-1797a4142eee",
69
- "datetime_epoch_millis": 1776884595301,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.035474997758865354,
73
- "tpp_threshold_2_intended_diff_only": 0.038699996471405027,
74
- "tpp_threshold_2_unintended_diff_only": 0.003224998712539673,
75
- "tpp_threshold_5_total_metric": 0.08667500615119933,
76
- "tpp_threshold_5_intended_diff_only": 0.10940001010894776,
77
- "tpp_threshold_5_unintended_diff_only": 0.022725003957748412,
78
- "tpp_threshold_10_total_metric": 0.10779999345541001,
79
- "tpp_threshold_10_intended_diff_only": 0.15280000567436217,
80
- "tpp_threshold_10_unintended_diff_only": 0.04500001221895218,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.06124999523162842,
99
- "tpp_threshold_2_intended_diff_only": 0.06740000247955322,
100
- "tpp_threshold_2_unintended_diff_only": 0.006150007247924805,
101
- "tpp_threshold_5_total_metric": 0.1564000129699707,
102
- "tpp_threshold_5_intended_diff_only": 0.19980002641677858,
103
- "tpp_threshold_5_unintended_diff_only": 0.04340001344680786,
104
- "tpp_threshold_10_total_metric": 0.19074999988079072,
105
- "tpp_threshold_10_intended_diff_only": 0.2782000184059143,
106
- "tpp_threshold_10_unintended_diff_only": 0.0874500185251236,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.009700000286102295,
123
- "tpp_threshold_2_intended_diff_only": 0.009999990463256836,
124
- "tpp_threshold_2_unintended_diff_only": 0.000299990177154541,
125
- "tpp_threshold_5_total_metric": 0.016949999332427978,
126
- "tpp_threshold_5_intended_diff_only": 0.018999993801116943,
127
- "tpp_threshold_5_unintended_diff_only": 0.0020499944686889647,
128
- "tpp_threshold_10_total_metric": 0.024849987030029295,
129
- "tpp_threshold_10_intended_diff_only": 0.027399992942810057,
130
- "tpp_threshold_10_unintended_diff_only": 0.0025500059127807617,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.01150001585483551,
190
  "tpp_threshold_10_total_metric": 0.11299999058246613,
191
  "tpp_threshold_10_intended_diff_only": 0.125,
192
- "tpp_threshold_10_unintended_diff_only": 0.012000009417533875
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": 0.08724997937679291,
196
- "tpp_threshold_2_intended_diff_only": 0.10199999809265137,
197
- "tpp_threshold_2_unintended_diff_only": 0.01475001871585846,
198
- "tpp_threshold_5_total_metric": 0.19999998807907104,
199
  "tpp_threshold_5_intended_diff_only": 0.2680000066757202,
200
- "tpp_threshold_5_unintended_diff_only": 0.06800001859664917,
201
- "tpp_threshold_10_total_metric": 0.22450001537799835,
202
  "tpp_threshold_10_intended_diff_only": 0.33000004291534424,
203
- "tpp_threshold_10_unintended_diff_only": 0.10550002753734589
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.12700000405311584,
207
- "tpp_threshold_2_intended_diff_only": 0.1380000114440918,
208
- "tpp_threshold_2_unintended_diff_only": 0.011000007390975952,
209
- "tpp_threshold_5_total_metric": 0.2275000363588333,
210
- "tpp_threshold_5_intended_diff_only": 0.29200005531311035,
211
- "tpp_threshold_5_unintended_diff_only": 0.06450001895427704,
212
- "tpp_threshold_10_total_metric": 0.17399999499320984,
213
- "tpp_threshold_10_intended_diff_only": 0.3370000123977661,
214
- "tpp_threshold_10_unintended_diff_only": 0.16300001740455627
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.012000009417533875,
218
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
219
- "tpp_threshold_2_unintended_diff_only": -0.003000006079673767,
220
- "tpp_threshold_5_total_metric": 0.23975002765655518,
221
- "tpp_threshold_5_intended_diff_only": 0.3030000329017639,
222
- "tpp_threshold_5_unintended_diff_only": 0.06325000524520874,
223
- "tpp_threshold_10_total_metric": 0.2722500115633011,
224
  "tpp_threshold_10_intended_diff_only": 0.3680000305175781,
225
- "tpp_threshold_10_unintended_diff_only": 0.09575001895427704
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.003000006079673767,
229
- "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
230
- "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224,
231
- "tpp_threshold_5_total_metric": 0.019250035285949707,
232
  "tpp_threshold_5_intended_diff_only": 0.029000043869018555,
233
- "tpp_threshold_5_unintended_diff_only": 0.009750008583068848,
234
- "tpp_threshold_10_total_metric": 0.16999998688697815,
235
- "tpp_threshold_10_intended_diff_only": 0.23100000619888306,
236
- "tpp_threshold_10_unintended_diff_only": 0.06100001931190491
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.008750006556510925,
242
- "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
243
- "tpp_threshold_2_unintended_diff_only": -0.004749998450279236,
244
- "tpp_threshold_5_total_metric": 0.01199999451637268,
245
- "tpp_threshold_5_intended_diff_only": 0.009999990463256836,
246
- "tpp_threshold_5_unintended_diff_only": -0.0020000040531158447,
247
- "tpp_threshold_10_total_metric": -0.0014999955892562866,
248
- "tpp_threshold_10_intended_diff_only": 0.003000020980834961,
249
- "tpp_threshold_10_unintended_diff_only": 0.0045000165700912476
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": 0.003000020980834961,
253
- "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
254
- "tpp_threshold_2_unintended_diff_only": 0.004999995231628418,
255
- "tpp_threshold_5_total_metric": 0.01325005292892456,
256
- "tpp_threshold_5_intended_diff_only": 0.020000040531158447,
257
- "tpp_threshold_5_unintended_diff_only": 0.006749987602233887,
258
- "tpp_threshold_10_total_metric": 0.04275001585483551,
259
- "tpp_threshold_10_intended_diff_only": 0.046000003814697266,
260
- "tpp_threshold_10_unintended_diff_only": 0.0032499879598617554
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.012250006198883057,
264
- "tpp_threshold_2_intended_diff_only": 0.0059999823570251465,
265
- "tpp_threshold_2_unintended_diff_only": -0.00625002384185791,
266
- "tpp_threshold_5_total_metric": 0.017749980092048645,
267
- "tpp_threshold_5_intended_diff_only": 0.015999972820281982,
268
- "tpp_threshold_5_unintended_diff_only": -0.0017500072717666626,
269
- "tpp_threshold_10_total_metric": 0.01724998652935028,
270
- "tpp_threshold_10_intended_diff_only": 0.018999993801116943,
271
- "tpp_threshold_10_unintended_diff_only": 0.0017500072717666626
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.007249996066093445,
275
- "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
276
- "tpp_threshold_2_unintended_diff_only": 0.008249983191490173,
277
- "tpp_threshold_5_total_metric": 0.0037499964237213135,
278
- "tpp_threshold_5_intended_diff_only": 0.004999995231628418,
279
- "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045,
280
- "tpp_threshold_10_total_metric": 0.006499946117401123,
281
  "tpp_threshold_10_intended_diff_only": 0.007999956607818604,
282
- "tpp_threshold_10_unintended_diff_only": 0.0015000104904174805
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.03174996376037598,
286
- "tpp_threshold_2_intended_diff_only": 0.030999958515167236,
287
- "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402,
288
- "tpp_threshold_5_total_metric": 0.03799997270107269,
289
- "tpp_threshold_5_intended_diff_only": 0.04399996995925903,
290
- "tpp_threshold_5_unintended_diff_only": 0.00599999725818634,
291
- "tpp_threshold_10_total_metric": 0.05924998223781586,
292
- "tpp_threshold_10_intended_diff_only": 0.06099998950958252,
293
- "tpp_threshold_10_unintended_diff_only": 0.0017500072717666626
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "df5a6daf-328f-4f93-b99e-dae43f128b94",
73
+ "datetime_epoch_millis": 1777316627848,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.034175002574920656,
77
+ "tpp_threshold_2_intended_diff_only": 0.038300007581710815,
78
+ "tpp_threshold_2_unintended_diff_only": 0.0041250050067901615,
79
+ "tpp_threshold_5_total_metric": 0.08677499294281006,
80
+ "tpp_threshold_5_intended_diff_only": 0.1081000030040741,
81
+ "tpp_threshold_5_unintended_diff_only": 0.02132501006126404,
82
+ "tpp_threshold_10_total_metric": 0.10925000607967378,
83
+ "tpp_threshold_10_intended_diff_only": 0.15460001230239867,
84
+ "tpp_threshold_10_unintended_diff_only": 0.04535000622272491,
85
+ "tpp_threshold_20_total_metric": 0.13745000660419465,
86
+ "tpp_threshold_20_intended_diff_only": 0.2049000084400177,
87
+ "tpp_threshold_20_unintended_diff_only": 0.06745000183582306,
88
+ "tpp_threshold_50_total_metric": 0.16090001314878463,
89
+ "tpp_threshold_50_intended_diff_only": 0.2610000193119049,
90
+ "tpp_threshold_50_unintended_diff_only": 0.10010000616312027,
91
+ "tpp_threshold_100_total_metric": 0.18385001718997956,
92
+ "tpp_threshold_100_intended_diff_only": 0.32000002861022947,
93
+ "tpp_threshold_100_unintended_diff_only": 0.13615001142024993,
94
+ "tpp_threshold_500_total_metric": 0.2407000184059143,
95
+ "tpp_threshold_500_intended_diff_only": 0.4353000283241272,
96
+ "tpp_threshold_500_unintended_diff_only": 0.19460000991821289
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.05995000600814819,
103
+ "tpp_threshold_2_intended_diff_only": 0.06720001697540283,
104
+ "tpp_threshold_2_unintended_diff_only": 0.007250010967254639,
105
+ "tpp_threshold_5_total_metric": 0.15739999413490297,
106
+ "tpp_threshold_5_intended_diff_only": 0.1998000144958496,
107
+ "tpp_threshold_5_unintended_diff_only": 0.04240002036094666,
108
+ "tpp_threshold_10_total_metric": 0.1895000159740448,
109
+ "tpp_threshold_10_intended_diff_only": 0.27820003032684326,
110
+ "tpp_threshold_10_unintended_diff_only": 0.08870001435279846,
111
+ "tpp_threshold_20_total_metric": 0.2262500047683716,
112
+ "tpp_threshold_20_intended_diff_only": 0.35780001878738404,
113
+ "tpp_threshold_20_unintended_diff_only": 0.13155001401901245,
114
+ "tpp_threshold_50_total_metric": 0.21665000915527344,
115
+ "tpp_threshold_50_intended_diff_only": 0.41160002946853635,
116
+ "tpp_threshold_50_unintended_diff_only": 0.19495002031326295,
117
+ "tpp_threshold_100_total_metric": 0.18435002267360687,
118
+ "tpp_threshold_100_intended_diff_only": 0.44460004568099976,
119
+ "tpp_threshold_100_unintended_diff_only": 0.26025002300739286,
120
+ "tpp_threshold_500_total_metric": 0.10345001816749573,
121
+ "tpp_threshold_500_intended_diff_only": 0.4670000433921814,
122
+ "tpp_threshold_500_unintended_diff_only": 0.36355002522468566
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.008399999141693116,
127
+ "tpp_threshold_2_intended_diff_only": 0.009399998188018798,
128
+ "tpp_threshold_2_unintended_diff_only": 0.0009999990463256836,
129
+ "tpp_threshold_5_total_metric": 0.016149991750717164,
130
+ "tpp_threshold_5_intended_diff_only": 0.016399991512298585,
131
+ "tpp_threshold_5_unintended_diff_only": 0.0002499997615814209,
132
+ "tpp_threshold_10_total_metric": 0.028999996185302735,
133
+ "tpp_threshold_10_intended_diff_only": 0.0309999942779541,
134
+ "tpp_threshold_10_unintended_diff_only": 0.0019999980926513673,
135
+ "tpp_threshold_20_total_metric": 0.0486500084400177,
136
+ "tpp_threshold_20_intended_diff_only": 0.051999998092651364,
137
+ "tpp_threshold_20_unintended_diff_only": 0.003349989652633667,
138
+ "tpp_threshold_50_total_metric": 0.10515001714229584,
139
+ "tpp_threshold_50_intended_diff_only": 0.11040000915527344,
140
+ "tpp_threshold_50_unintended_diff_only": 0.0052499920129776,
141
+ "tpp_threshold_100_total_metric": 0.18335001170635223,
142
+ "tpp_threshold_100_intended_diff_only": 0.19540001153945924,
143
+ "tpp_threshold_100_unintended_diff_only": 0.012049999833106995,
144
+ "tpp_threshold_500_total_metric": 0.37795001864433286,
145
+ "tpp_threshold_500_intended_diff_only": 0.403600013256073,
146
+ "tpp_threshold_500_unintended_diff_only": 0.025649994611740112
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.01150001585483551,
194
  "tpp_threshold_10_total_metric": 0.11299999058246613,
195
  "tpp_threshold_10_intended_diff_only": 0.125,
196
+ "tpp_threshold_10_unintended_diff_only": 0.012000009417533875,
197
+ "tpp_threshold_20_total_metric": 0.23149995505809784,
198
+ "tpp_threshold_20_intended_diff_only": 0.3399999737739563,
199
+ "tpp_threshold_20_unintended_diff_only": 0.10850001871585846,
200
+ "tpp_threshold_50_total_metric": 0.22649997472763062,
201
+ "tpp_threshold_50_intended_diff_only": 0.40299999713897705,
202
+ "tpp_threshold_50_unintended_diff_only": 0.17650002241134644,
203
+ "tpp_threshold_100_total_metric": 0.20749999582767487,
204
+ "tpp_threshold_100_intended_diff_only": 0.43800002336502075,
205
+ "tpp_threshold_100_unintended_diff_only": 0.2305000275373459,
206
+ "tpp_threshold_500_total_metric": 0.11124999821186066,
207
+ "tpp_threshold_500_intended_diff_only": 0.45600003004074097,
208
+ "tpp_threshold_500_unintended_diff_only": 0.3447500318288803
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.09000000357627869,
212
+ "tpp_threshold_2_intended_diff_only": 0.10000002384185791,
213
+ "tpp_threshold_2_unintended_diff_only": 0.010000020265579224,
214
+ "tpp_threshold_5_total_metric": 0.20249998569488525,
215
  "tpp_threshold_5_intended_diff_only": 0.2680000066757202,
216
+ "tpp_threshold_5_unintended_diff_only": 0.06550002098083496,
217
+ "tpp_threshold_10_total_metric": 0.2185000330209732,
218
  "tpp_threshold_10_intended_diff_only": 0.33000004291534424,
219
+ "tpp_threshold_10_unintended_diff_only": 0.11150000989437103,
220
+ "tpp_threshold_20_total_metric": 0.236999973654747,
221
+ "tpp_threshold_20_intended_diff_only": 0.3569999933242798,
222
+ "tpp_threshold_20_unintended_diff_only": 0.12000001966953278,
223
+ "tpp_threshold_50_total_metric": 0.23200002312660217,
224
+ "tpp_threshold_50_intended_diff_only": 0.3940000534057617,
225
+ "tpp_threshold_50_unintended_diff_only": 0.16200003027915955,
226
+ "tpp_threshold_100_total_metric": 0.19675002992153168,
227
+ "tpp_threshold_100_intended_diff_only": 0.437000036239624,
228
+ "tpp_threshold_100_unintended_diff_only": 0.24025000631809235,
229
+ "tpp_threshold_500_total_metric": 0.09050002694129944,
230
+ "tpp_threshold_500_intended_diff_only": 0.4590000510215759,
231
+ "tpp_threshold_500_unintended_diff_only": 0.3685000240802765
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.11675003170967102,
235
+ "tpp_threshold_2_intended_diff_only": 0.13600003719329834,
236
+ "tpp_threshold_2_unintended_diff_only": 0.01925000548362732,
237
+ "tpp_threshold_5_total_metric": 0.2292499840259552,
238
+ "tpp_threshold_5_intended_diff_only": 0.29100000858306885,
239
+ "tpp_threshold_5_unintended_diff_only": 0.06175002455711365,
240
+ "tpp_threshold_10_total_metric": 0.1717500388622284,
241
+ "tpp_threshold_10_intended_diff_only": 0.3380000591278076,
242
+ "tpp_threshold_10_unintended_diff_only": 0.16625002026557922,
243
+ "tpp_threshold_20_total_metric": 0.1365000307559967,
244
+ "tpp_threshold_20_intended_diff_only": 0.36600005626678467,
245
+ "tpp_threshold_20_unintended_diff_only": 0.22950002551078796,
246
+ "tpp_threshold_50_total_metric": 0.12025004625320435,
247
+ "tpp_threshold_50_intended_diff_only": 0.3840000629425049,
248
+ "tpp_threshold_50_unintended_diff_only": 0.26375001668930054,
249
+ "tpp_threshold_100_total_metric": 0.08700002729892731,
250
+ "tpp_threshold_100_intended_diff_only": 0.41700005531311035,
251
+ "tpp_threshold_100_unintended_diff_only": 0.33000002801418304,
252
+ "tpp_threshold_500_total_metric": 0.05349999666213989,
253
+ "tpp_threshold_500_intended_diff_only": 0.4520000219345093,
254
+ "tpp_threshold_500_unintended_diff_only": 0.3985000252723694
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.009749993681907654,
258
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0007499903440475464,
260
+ "tpp_threshold_5_total_metric": 0.24324999749660492,
261
+ "tpp_threshold_5_intended_diff_only": 0.30400002002716064,
262
+ "tpp_threshold_5_unintended_diff_only": 0.060750022530555725,
263
+ "tpp_threshold_10_total_metric": 0.27525001764297485,
264
  "tpp_threshold_10_intended_diff_only": 0.3680000305175781,
265
+ "tpp_threshold_10_unintended_diff_only": 0.09275001287460327,
266
+ "tpp_threshold_20_total_metric": 0.30025002360343933,
267
+ "tpp_threshold_20_intended_diff_only": 0.4150000214576721,
268
+ "tpp_threshold_20_unintended_diff_only": 0.11474999785423279,
269
+ "tpp_threshold_50_total_metric": 0.26474998891353607,
270
+ "tpp_threshold_50_intended_diff_only": 0.47600001096725464,
271
+ "tpp_threshold_50_unintended_diff_only": 0.21125002205371857,
272
+ "tpp_threshold_100_total_metric": 0.21375001966953278,
273
+ "tpp_threshold_100_intended_diff_only": 0.48200005292892456,
274
+ "tpp_threshold_100_unintended_diff_only": 0.2682500332593918,
275
+ "tpp_threshold_500_total_metric": 0.1302500218153,
276
+ "tpp_threshold_500_intended_diff_only": 0.4920000433921814,
277
+ "tpp_threshold_500_unintended_diff_only": 0.3617500215768814
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.00625002384185791,
281
+ "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0007500052452087402,
283
+ "tpp_threshold_5_total_metric": 0.016500025987625122,
284
  "tpp_threshold_5_intended_diff_only": 0.029000043869018555,
285
+ "tpp_threshold_5_unintended_diff_only": 0.012500017881393433,
286
+ "tpp_threshold_10_total_metric": 0.16899999976158142,
287
+ "tpp_threshold_10_intended_diff_only": 0.23000001907348633,
288
+ "tpp_threshold_10_unintended_diff_only": 0.06100001931190491,
289
+ "tpp_threshold_20_total_metric": 0.22600004076957703,
290
+ "tpp_threshold_20_intended_diff_only": 0.3110000491142273,
291
+ "tpp_threshold_20_unintended_diff_only": 0.08500000834465027,
292
+ "tpp_threshold_50_total_metric": 0.23975001275539398,
293
+ "tpp_threshold_50_intended_diff_only": 0.4010000228881836,
294
+ "tpp_threshold_50_unintended_diff_only": 0.1612500101327896,
295
+ "tpp_threshold_100_total_metric": 0.21675004065036774,
296
+ "tpp_threshold_100_intended_diff_only": 0.4490000605583191,
297
+ "tpp_threshold_100_unintended_diff_only": 0.23225001990795135,
298
+ "tpp_threshold_500_total_metric": 0.13175004720687866,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.34425002336502075
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.0014999955892562866,
306
+ "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
307
+ "tpp_threshold_2_unintended_diff_only": 0.0034999996423721313,
308
+ "tpp_threshold_5_total_metric": 0.0037499964237213135,
309
+ "tpp_threshold_5_intended_diff_only": 0.004999995231628418,
310
+ "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045,
311
+ "tpp_threshold_10_total_metric": 0.0032500028610229492,
312
+ "tpp_threshold_10_intended_diff_only": 0.004999995231628418,
313
+ "tpp_threshold_10_unintended_diff_only": 0.0017499923706054688,
314
+ "tpp_threshold_20_total_metric": 0.007500007748603821,
315
+ "tpp_threshold_20_intended_diff_only": 0.009999990463256836,
316
+ "tpp_threshold_20_unintended_diff_only": 0.002499982714653015,
317
+ "tpp_threshold_50_total_metric": 0.046750009059906006,
318
+ "tpp_threshold_50_intended_diff_only": 0.046999990940093994,
319
+ "tpp_threshold_50_unintended_diff_only": 0.0002499818801879883,
320
+ "tpp_threshold_100_total_metric": 0.10199998319149017,
321
+ "tpp_threshold_100_intended_diff_only": 0.10799998044967651,
322
+ "tpp_threshold_100_unintended_diff_only": 0.00599999725818634,
323
+ "tpp_threshold_500_total_metric": 0.38975000381469727,
324
+ "tpp_threshold_500_intended_diff_only": 0.40799999237060547,
325
+ "tpp_threshold_500_unintended_diff_only": 0.018249988555908203
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.013250038027763367,
329
+ "tpp_threshold_2_intended_diff_only": 0.01100003719329834,
330
+ "tpp_threshold_2_unintended_diff_only": -0.002250000834465027,
331
+ "tpp_threshold_5_total_metric": 0.009750038385391235,
332
+ "tpp_threshold_5_intended_diff_only": 0.016000032424926758,
333
+ "tpp_threshold_5_unintended_diff_only": 0.0062499940395355225,
334
+ "tpp_threshold_10_total_metric": 0.04600003361701965,
335
+ "tpp_threshold_10_intended_diff_only": 0.04900002479553223,
336
+ "tpp_threshold_10_unintended_diff_only": 0.0029999911785125732,
337
+ "tpp_threshold_20_total_metric": 0.07525007426738739,
338
+ "tpp_threshold_20_intended_diff_only": 0.07900005578994751,
339
+ "tpp_threshold_20_unintended_diff_only": 0.0037499815225601196,
340
+ "tpp_threshold_50_total_metric": 0.13650007545948029,
341
+ "tpp_threshold_50_intended_diff_only": 0.1480000615119934,
342
+ "tpp_threshold_50_unintended_diff_only": 0.011499986052513123,
343
+ "tpp_threshold_100_total_metric": 0.22400006651878357,
344
+ "tpp_threshold_100_intended_diff_only": 0.24100005626678467,
345
+ "tpp_threshold_100_unintended_diff_only": 0.0169999897480011,
346
+ "tpp_threshold_500_total_metric": 0.4162500500679016,
347
+ "tpp_threshold_500_intended_diff_only": 0.4360000491142273,
348
+ "tpp_threshold_500_unintended_diff_only": 0.019749999046325684
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.003750026226043701,
352
+ "tpp_threshold_2_intended_diff_only": 0.003000020980834961,
353
+ "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402,
354
+ "tpp_threshold_5_total_metric": 0.012750014662742615,
355
+ "tpp_threshold_5_intended_diff_only": 0.008000016212463379,
356
+ "tpp_threshold_5_unintended_diff_only": -0.004749998450279236,
357
+ "tpp_threshold_10_total_metric": 0.019250020384788513,
358
+ "tpp_threshold_10_intended_diff_only": 0.017000019550323486,
359
+ "tpp_threshold_10_unintended_diff_only": -0.002250000834465027,
360
+ "tpp_threshold_20_total_metric": 0.012499988079071045,
361
+ "tpp_threshold_20_intended_diff_only": 0.014999985694885254,
362
+ "tpp_threshold_20_unintended_diff_only": 0.002499997615814209,
363
+ "tpp_threshold_50_total_metric": 0.05125001072883606,
364
+ "tpp_threshold_50_intended_diff_only": 0.050000011920928955,
365
+ "tpp_threshold_50_unintended_diff_only": -0.0012499988079071045,
366
+ "tpp_threshold_100_total_metric": 0.10474999248981476,
367
+ "tpp_threshold_100_intended_diff_only": 0.11599999666213989,
368
+ "tpp_threshold_100_unintended_diff_only": 0.011250004172325134,
369
+ "tpp_threshold_500_total_metric": 0.35875003039836884,
370
+ "tpp_threshold_500_intended_diff_only": 0.38700002431869507,
371
+ "tpp_threshold_500_unintended_diff_only": 0.028249993920326233
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.0014999806880950928,
375
+ "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
376
+ "tpp_threshold_2_unintended_diff_only": 0.0004999935626983643,
377
+ "tpp_threshold_5_total_metric": 0.013499945402145386,
378
+ "tpp_threshold_5_intended_diff_only": 0.012999951839447021,
379
+ "tpp_threshold_5_unintended_diff_only": -0.0004999935626983643,
380
+ "tpp_threshold_10_total_metric": 0.003999948501586914,
381
  "tpp_threshold_10_intended_diff_only": 0.007999956607818604,
382
+ "tpp_threshold_10_unintended_diff_only": 0.0040000081062316895,
383
+ "tpp_threshold_20_total_metric": 0.04649999737739563,
384
+ "tpp_threshold_20_intended_diff_only": 0.046999990940093994,
385
+ "tpp_threshold_20_unintended_diff_only": 0.0004999935626983643,
386
+ "tpp_threshold_50_total_metric": 0.0949999988079071,
387
+ "tpp_threshold_50_intended_diff_only": 0.10199999809265137,
388
+ "tpp_threshold_50_unintended_diff_only": 0.006999999284744263,
389
+ "tpp_threshold_100_total_metric": 0.2212499976158142,
390
+ "tpp_threshold_100_intended_diff_only": 0.23600000143051147,
391
+ "tpp_threshold_100_unintended_diff_only": 0.014750003814697266,
392
+ "tpp_threshold_500_total_metric": 0.3804999738931656,
393
+ "tpp_threshold_500_intended_diff_only": 0.4139999747276306,
394
+ "tpp_threshold_500_unintended_diff_only": 0.03350000083446503
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.02199995517730713,
398
+ "tpp_threshold_2_intended_diff_only": 0.02599996328353882,
399
+ "tpp_threshold_2_unintended_diff_only": 0.0040000081062316895,
400
+ "tpp_threshold_5_total_metric": 0.040999963879585266,
401
+ "tpp_threshold_5_intended_diff_only": 0.039999961853027344,
402
+ "tpp_threshold_5_unintended_diff_only": -0.0010000020265579224,
403
+ "tpp_threshold_10_total_metric": 0.07249997556209564,
404
+ "tpp_threshold_10_intended_diff_only": 0.07599997520446777,
405
+ "tpp_threshold_10_unintended_diff_only": 0.0034999996423721313,
406
+ "tpp_threshold_20_total_metric": 0.10149997472763062,
407
+ "tpp_threshold_20_intended_diff_only": 0.10899996757507324,
408
+ "tpp_threshold_20_unintended_diff_only": 0.007499992847442627,
409
+ "tpp_threshold_50_total_metric": 0.19624999165534973,
410
+ "tpp_threshold_50_intended_diff_only": 0.20499998331069946,
411
+ "tpp_threshold_50_unintended_diff_only": 0.008749991655349731,
412
+ "tpp_threshold_100_total_metric": 0.26475001871585846,
413
+ "tpp_threshold_100_intended_diff_only": 0.2760000228881836,
414
+ "tpp_threshold_100_unintended_diff_only": 0.011250004172325134,
415
+ "tpp_threshold_500_total_metric": 0.34450003504753113,
416
+ "tpp_threshold_500_intended_diff_only": 0.37300002574920654,
417
+ "tpp_threshold_500_unintended_diff_only": 0.028499990701675415
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-195312-tokens-800000000/scr/saebench_scr_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,8 +69,8 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "c9b01a50-123e-4537-87b5-41a091d4564a",
69
- "datetime_epoch_millis": 1776884615448,
70
  "eval_result_metrics": {
71
  "scr_metrics": {
72
  "scr_dir1_threshold_2": 0.2947372238881211,
@@ -78,18 +82,18 @@
78
  "scr_dir1_threshold_10": 0.35524561939979316,
79
  "scr_metric_threshold_10": 0.21637876785214583,
80
  "scr_dir2_threshold_10": 0.2019990997881127,
81
- "scr_dir1_threshold_20": null,
82
- "scr_metric_threshold_20": null,
83
- "scr_dir2_threshold_20": null,
84
- "scr_dir1_threshold_50": null,
85
- "scr_metric_threshold_50": null,
86
- "scr_dir2_threshold_50": null,
87
- "scr_dir1_threshold_100": null,
88
- "scr_metric_threshold_100": null,
89
- "scr_dir2_threshold_100": null,
90
- "scr_dir1_threshold_500": null,
91
- "scr_metric_threshold_500": null,
92
- "scr_dir2_threshold_500": null
93
  }
94
  },
95
  "eval_result_details": [
@@ -104,18 +108,18 @@
104
  "scr_dir1_threshold_10": 0.5769232238773122,
105
  "scr_metric_threshold_10": 0.036269422851059746,
106
  "scr_dir2_threshold_10": 0.036269422851059746,
107
- "scr_dir1_threshold_20": null,
108
- "scr_metric_threshold_20": null,
109
- "scr_dir2_threshold_20": null,
110
- "scr_dir1_threshold_50": null,
111
- "scr_metric_threshold_50": null,
112
- "scr_dir2_threshold_50": null,
113
- "scr_dir1_threshold_100": null,
114
- "scr_metric_threshold_100": null,
115
- "scr_dir2_threshold_100": null,
116
- "scr_dir1_threshold_500": null,
117
- "scr_metric_threshold_500": null,
118
- "scr_dir2_threshold_500": null
119
  },
120
  {
121
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
@@ -128,18 +132,18 @@
128
  "scr_dir1_threshold_10": 0.4469697140738727,
129
  "scr_metric_threshold_10": 0.1743120403197142,
130
  "scr_dir2_threshold_10": 0.1743120403197142,
131
- "scr_dir1_threshold_20": null,
132
- "scr_metric_threshold_20": null,
133
- "scr_dir2_threshold_20": null,
134
- "scr_dir1_threshold_50": null,
135
- "scr_metric_threshold_50": null,
136
- "scr_dir2_threshold_50": null,
137
- "scr_dir1_threshold_100": null,
138
- "scr_metric_threshold_100": null,
139
- "scr_dir2_threshold_100": null,
140
- "scr_dir1_threshold_500": null,
141
- "scr_metric_threshold_500": null,
142
- "scr_dir2_threshold_500": null
143
  },
144
  {
145
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
@@ -152,18 +156,18 @@
152
  "scr_dir1_threshold_10": 0.4062502910382569,
153
  "scr_metric_threshold_10": 0.13316589123232284,
154
  "scr_dir2_threshold_10": 0.13316589123232284,
155
- "scr_dir1_threshold_20": null,
156
- "scr_metric_threshold_20": null,
157
- "scr_dir2_threshold_20": null,
158
- "scr_dir1_threshold_50": null,
159
- "scr_metric_threshold_50": null,
160
- "scr_dir2_threshold_50": null,
161
- "scr_dir1_threshold_100": null,
162
- "scr_metric_threshold_100": null,
163
- "scr_dir2_threshold_100": null,
164
- "scr_dir1_threshold_500": null,
165
- "scr_metric_threshold_500": null,
166
- "scr_dir2_threshold_500": null
167
  },
168
  {
169
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
@@ -176,18 +180,18 @@
176
  "scr_dir1_threshold_10": 0.30201315702624804,
177
  "scr_metric_threshold_10": 0.124203926844982,
178
  "scr_dir2_threshold_10": 0.124203926844982,
179
- "scr_dir1_threshold_20": null,
180
- "scr_metric_threshold_20": null,
181
- "scr_dir2_threshold_20": null,
182
- "scr_dir1_threshold_50": null,
183
- "scr_metric_threshold_50": null,
184
- "scr_dir2_threshold_50": null,
185
- "scr_dir1_threshold_100": null,
186
- "scr_metric_threshold_100": null,
187
- "scr_dir2_threshold_100": null,
188
- "scr_dir1_threshold_500": null,
189
- "scr_metric_threshold_500": null,
190
- "scr_dir2_threshold_500": null
191
  },
192
  {
193
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
@@ -200,18 +204,18 @@
200
  "scr_dir1_threshold_10": 0.09473687842942387,
201
  "scr_metric_threshold_10": 0.5198413665167169,
202
  "scr_dir2_threshold_10": 0.5198413665167169,
203
- "scr_dir1_threshold_20": null,
204
- "scr_metric_threshold_20": null,
205
- "scr_dir2_threshold_20": null,
206
- "scr_dir1_threshold_50": null,
207
- "scr_metric_threshold_50": null,
208
- "scr_dir2_threshold_50": null,
209
- "scr_dir1_threshold_100": null,
210
- "scr_metric_threshold_100": null,
211
- "scr_dir2_threshold_100": null,
212
- "scr_dir1_threshold_500": null,
213
- "scr_metric_threshold_500": null,
214
- "scr_dir2_threshold_500": null
215
  },
216
  {
217
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
@@ -224,18 +228,18 @@
224
  "scr_dir1_threshold_10": 0.3825136878996036,
225
  "scr_metric_threshold_10": 0.14122136275770192,
226
  "scr_dir2_threshold_10": 0.14122136275770192,
227
- "scr_dir1_threshold_20": null,
228
- "scr_metric_threshold_20": null,
229
- "scr_dir2_threshold_20": null,
230
- "scr_dir1_threshold_50": null,
231
- "scr_metric_threshold_50": null,
232
- "scr_dir2_threshold_50": null,
233
- "scr_dir1_threshold_100": null,
234
- "scr_metric_threshold_100": null,
235
- "scr_dir2_threshold_100": null,
236
- "scr_dir1_threshold_500": null,
237
- "scr_metric_threshold_500": null,
238
- "scr_dir2_threshold_500": null
239
  },
240
  {
241
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
@@ -248,18 +252,18 @@
248
  "scr_dir1_threshold_10": 0.3953486760285681,
249
  "scr_metric_threshold_10": 0.3648068054696092,
250
  "scr_dir2_threshold_10": 0.3648068054696092,
251
- "scr_dir1_threshold_20": null,
252
- "scr_metric_threshold_20": null,
253
- "scr_dir2_threshold_20": null,
254
- "scr_dir1_threshold_50": null,
255
- "scr_metric_threshold_50": null,
256
- "scr_dir2_threshold_50": null,
257
- "scr_dir1_threshold_100": null,
258
- "scr_metric_threshold_100": null,
259
- "scr_dir2_threshold_100": null,
260
- "scr_dir1_threshold_500": null,
261
- "scr_metric_threshold_500": null,
262
- "scr_dir2_threshold_500": null
263
  },
264
  {
265
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
@@ -272,18 +276,18 @@
272
  "scr_dir1_threshold_10": 0.2372093268250598,
273
  "scr_metric_threshold_10": 0.2372093268250598,
274
  "scr_dir2_threshold_10": 0.1221719823127949,
275
- "scr_dir1_threshold_20": null,
276
- "scr_metric_threshold_20": null,
277
- "scr_dir2_threshold_20": null,
278
- "scr_dir1_threshold_50": null,
279
- "scr_metric_threshold_50": null,
280
- "scr_dir2_threshold_50": null,
281
- "scr_dir1_threshold_100": null,
282
- "scr_metric_threshold_100": null,
283
- "scr_dir2_threshold_100": null,
284
- "scr_dir1_threshold_500": null,
285
- "scr_metric_threshold_500": null,
286
- "scr_dir2_threshold_500": null
287
  }
288
  ],
289
  "sae_bench_commit_hash": "Unknown",
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "6c5700ba-083a-4d96-a7b9-a05419f3d808",
73
+ "datetime_epoch_millis": 1777316489620,
74
  "eval_result_metrics": {
75
  "scr_metrics": {
76
  "scr_dir1_threshold_2": 0.2947372238881211,
 
82
  "scr_dir1_threshold_10": 0.35524561939979316,
83
  "scr_metric_threshold_10": 0.21637876785214583,
84
  "scr_dir2_threshold_10": 0.2019990997881127,
85
+ "scr_dir1_threshold_20": 0.31020072750215694,
86
+ "scr_metric_threshold_20": 0.23214022507635007,
87
+ "scr_dir2_threshold_20": 0.21760271115984237,
88
+ "scr_dir1_threshold_50": 0.32320885874957717,
89
+ "scr_metric_threshold_50": 0.30508886941015756,
90
+ "scr_dir2_threshold_50": 0.28692093177725986,
91
+ "scr_dir1_threshold_100": -0.12096203532666253,
92
+ "scr_metric_threshold_100": 0.30685690316563835,
93
+ "scr_dir2_threshold_100": 0.28652122499326227,
94
+ "scr_dir1_threshold_500": -0.8702652497939531,
95
+ "scr_metric_threshold_500": 0.1961407921858782,
96
+ "scr_dir2_threshold_500": 0.17429507670451402
97
  }
98
  },
99
  "eval_result_details": [
 
108
  "scr_dir1_threshold_10": 0.5769232238773122,
109
  "scr_metric_threshold_10": 0.036269422851059746,
110
  "scr_dir2_threshold_10": 0.036269422851059746,
111
+ "scr_dir1_threshold_20": 0.5128207283533913,
112
+ "scr_metric_threshold_20": 0.10362698860995496,
113
+ "scr_dir2_threshold_20": 0.10362698860995496,
114
+ "scr_dir1_threshold_50": 0.5256410746257707,
115
+ "scr_metric_threshold_50": 0.15803104567846293,
116
+ "scr_dir2_threshold_50": 0.15803104567846293,
117
+ "scr_dir1_threshold_100": 0.30769289550924905,
118
+ "scr_metric_threshold_100": 0.2927460227800901,
119
+ "scr_dir2_threshold_100": 0.2927460227800901,
120
+ "scr_dir1_threshold_500": -2.8717942447901343,
121
+ "scr_metric_threshold_500": 0.05958549142789553,
122
+ "scr_dir2_threshold_500": 0.05958549142789553
123
  },
124
  {
125
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
 
132
  "scr_dir1_threshold_10": 0.4469697140738727,
133
  "scr_metric_threshold_10": 0.1743120403197142,
134
  "scr_dir2_threshold_10": 0.1743120403197142,
135
+ "scr_dir1_threshold_20": 0.43181839733079624,
136
+ "scr_metric_threshold_20": 0.11009177656462182,
137
+ "scr_dir2_threshold_20": 0.11009177656462182,
138
+ "scr_dir1_threshold_50": 0.4090909706659418,
139
+ "scr_metric_threshold_50": 0.32721708301814567,
140
+ "scr_dir2_threshold_50": 0.32721708301814567,
141
+ "scr_dir1_threshold_100": 0.1742424002965782,
142
+ "scr_metric_threshold_100": 0.4678898681224538,
143
+ "scr_dir2_threshold_100": 0.4678898681224538,
144
+ "scr_dir1_threshold_500": -0.6818178328929966,
145
+ "scr_metric_threshold_500": 0.14984716057708958,
146
+ "scr_dir2_threshold_500": 0.14984716057708958
147
  },
148
  {
149
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
 
156
  "scr_dir1_threshold_10": 0.4062502910382569,
157
  "scr_metric_threshold_10": 0.13316589123232284,
158
  "scr_dir2_threshold_10": 0.13316589123232284,
159
+ "scr_dir1_threshold_20": 0.06250011641530274,
160
+ "scr_metric_threshold_20": 0.1507537515352312,
161
+ "scr_dir2_threshold_20": 0.1507537515352312,
162
+ "scr_dir1_threshold_50": 0.31249965075409175,
163
+ "scr_metric_threshold_50": 0.19346736189433716,
164
+ "scr_dir2_threshold_50": 0.19346736189433716,
165
+ "scr_dir1_threshold_100": -1.9374998835846973,
166
+ "scr_metric_threshold_100": 0.03266334028870509,
167
+ "scr_dir2_threshold_100": 0.03266334028870509,
168
+ "scr_dir1_threshold_500": -3.1249993015081836,
169
+ "scr_metric_threshold_500": -0.05527641050700931,
170
+ "scr_dir2_threshold_500": -0.05527641050700931
171
  },
172
  {
173
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
 
180
  "scr_dir1_threshold_10": 0.30201315702624804,
181
  "scr_metric_threshold_10": 0.124203926844982,
182
  "scr_dir2_threshold_10": 0.124203926844982,
183
+ "scr_dir1_threshold_20": 0.43624162684689405,
184
+ "scr_metric_threshold_20": 0.006369534358659739,
185
+ "scr_dir2_threshold_20": 0.006369534358659739,
186
+ "scr_dir1_threshold_50": 0.25503349261242375,
187
+ "scr_metric_threshold_50": 0.11464972021884395,
188
+ "scr_dir2_threshold_50": 0.11464972021884395,
189
+ "scr_dir1_threshold_100": -0.5436244027034108,
190
+ "scr_metric_threshold_100": -0.07006373900303832,
191
+ "scr_dir2_threshold_100": -0.07006373900303832,
192
+ "scr_dir1_threshold_500": -1.583893143665206,
193
+ "scr_metric_threshold_500": -0.5254777577872327,
194
+ "scr_dir2_threshold_500": -0.5254777577872327
195
  },
196
  {
197
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
 
204
  "scr_dir1_threshold_10": 0.09473687842942387,
205
  "scr_metric_threshold_10": 0.5198413665167169,
206
  "scr_dir2_threshold_10": 0.5198413665167169,
207
+ "scr_dir1_threshold_20": -0.13684191373576507,
208
+ "scr_metric_threshold_20": 0.5873015159682478,
209
+ "scr_dir2_threshold_20": 0.5873015159682478,
210
+ "scr_dir1_threshold_50": -0.22631570196547887,
211
+ "scr_metric_threshold_50": 0.6904761961077699,
212
+ "scr_dir2_threshold_50": 0.6904761961077699,
213
+ "scr_dir1_threshold_100": -0.1947368470585578,
214
+ "scr_metric_threshold_100": 0.742063536177531,
215
+ "scr_dir2_threshold_100": 0.742063536177531,
216
+ "scr_dir1_threshold_500": -0.16315799215163673,
217
+ "scr_metric_threshold_500": 0.801587280938177,
218
+ "scr_dir2_threshold_500": 0.801587280938177
219
  },
220
  {
221
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
 
228
  "scr_dir1_threshold_10": 0.3825136878996036,
229
  "scr_metric_threshold_10": 0.14122136275770192,
230
  "scr_dir2_threshold_10": 0.14122136275770192,
231
+ "scr_dir1_threshold_20": 0.4262296310048433,
232
+ "scr_metric_threshold_20": 0.19465646684173443,
233
+ "scr_dir2_threshold_20": 0.19465646684173443,
234
+ "scr_dir1_threshold_50": 0.3934425108216648,
235
+ "scr_metric_threshold_50": 0.29007644181233594,
236
+ "scr_dir2_threshold_50": 0.29007644181233594,
237
+ "scr_dir1_threshold_100": 0.4262296310048433,
238
+ "scr_metric_threshold_100": 0.35496177909383203,
239
+ "scr_dir2_threshold_100": 0.35496177909383203,
240
+ "scr_dir1_threshold_500": 0.5519127231510342,
241
+ "scr_metric_threshold_500": 0.36259549539076413,
242
+ "scr_dir2_threshold_500": 0.36259549539076413
243
  },
244
  {
245
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
 
252
  "scr_dir1_threshold_10": 0.3953486760285681,
253
  "scr_metric_threshold_10": 0.3648068054696092,
254
  "scr_dir2_threshold_10": 0.3648068054696092,
255
+ "scr_dir1_threshold_20": 0.46511631775314366,
256
+ "scr_metric_threshold_20": 0.4206008506837011,
257
+ "scr_dir2_threshold_20": 0.4206008506837011,
258
+ "scr_dir1_threshold_50": 0.5627905725981688,
259
+ "scr_metric_threshold_50": 0.31330480666733007,
260
+ "scr_dir2_threshold_50": 0.31330480666733007,
261
+ "scr_dir1_threshold_100": 0.4744185221266269,
262
+ "scr_metric_threshold_100": 0.30901301606946474,
263
+ "scr_dir2_threshold_100": 0.30901301606946474,
264
+ "scr_dir1_threshold_500": 0.4744185221266269,
265
+ "scr_metric_threshold_500": 0.33905580606846963,
266
+ "scr_dir2_threshold_500": 0.33905580606846963
267
  },
268
  {
269
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
 
276
  "scr_dir1_threshold_10": 0.2372093268250598,
277
  "scr_metric_threshold_10": 0.2372093268250598,
278
  "scr_dir2_threshold_10": 0.1221719823127949,
279
+ "scr_dir1_threshold_20": 0.2837209160486496,
280
+ "scr_metric_threshold_20": 0.2837209160486496,
281
+ "scr_dir2_threshold_20": 0.16742080471658802,
282
+ "scr_dir1_threshold_50": 0.3534882998840343,
283
+ "scr_metric_threshold_50": 0.3534882998840343,
284
+ "scr_dir2_threshold_50": 0.208144798820853,
285
+ "scr_dir1_threshold_100": 0.32558140179606837,
286
+ "scr_metric_threshold_100": 0.32558140179606837,
287
+ "scr_dir2_threshold_100": 0.16289597641705988,
288
+ "scr_dir1_threshold_500": 0.43720927137887183,
289
+ "scr_metric_threshold_500": 0.43720927137887183,
290
+ "scr_dir2_threshold_500": 0.26244354752795823
291
  }
292
  ],
293
  "sae_bench_commit_hash": "Unknown",
gemma-2-2b-btk/k-100/seed-0/snapshots/step-195312-tokens-800000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "9c47242c-3ddc-42b2-994c-e801ae56458b",
69
- "datetime_epoch_millis": 1776884779578,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.033774998784065244,
73
- "tpp_threshold_2_intended_diff_only": 0.03669999837875366,
74
- "tpp_threshold_2_unintended_diff_only": 0.002924999594688416,
75
- "tpp_threshold_5_total_metric": 0.09659999907016754,
76
- "tpp_threshold_5_intended_diff_only": 0.12450000643730164,
77
- "tpp_threshold_5_unintended_diff_only": 0.027900007367134095,
78
- "tpp_threshold_10_total_metric": 0.1063500002026558,
79
- "tpp_threshold_10_intended_diff_only": 0.15390000939369203,
80
- "tpp_threshold_10_unintended_diff_only": 0.047550009191036226,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.05894999802112579,
99
- "tpp_threshold_2_intended_diff_only": 0.06460000276565551,
100
- "tpp_threshold_2_unintended_diff_only": 0.0056500047445297245,
101
- "tpp_threshold_5_total_metric": 0.17910000085830688,
102
- "tpp_threshold_5_intended_diff_only": 0.23280001878738404,
103
- "tpp_threshold_5_unintended_diff_only": 0.05370001792907715,
104
- "tpp_threshold_10_total_metric": 0.18945000767707826,
105
  "tpp_threshold_10_intended_diff_only": 0.28240002393722535,
106
- "tpp_threshold_10_unintended_diff_only": 0.0929500162601471,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.008599999547004699,
123
- "tpp_threshold_2_intended_diff_only": 0.008799993991851806,
124
- "tpp_threshold_2_unintended_diff_only": 0.00019999444484710693,
125
- "tpp_threshold_5_total_metric": 0.014099997282028199,
126
- "tpp_threshold_5_intended_diff_only": 0.016199994087219238,
127
- "tpp_threshold_5_unintended_diff_only": 0.00209999680519104,
128
- "tpp_threshold_10_total_metric": 0.02324999272823334,
129
- "tpp_threshold_10_intended_diff_only": 0.025399994850158692,
130
- "tpp_threshold_10_unintended_diff_only": 0.002150002121925354,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.013250008225440979,
190
  "tpp_threshold_10_total_metric": 0.11374999582767487,
191
  "tpp_threshold_10_intended_diff_only": 0.1290000081062317,
192
- "tpp_threshold_10_unintended_diff_only": 0.015250012278556824
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": 0.04924999177455902,
196
- "tpp_threshold_2_intended_diff_only": 0.06000000238418579,
197
- "tpp_threshold_2_unintended_diff_only": 0.01075001060962677,
198
- "tpp_threshold_5_total_metric": 0.18025003373622894,
199
  "tpp_threshold_5_intended_diff_only": 0.2420000433921814,
200
- "tpp_threshold_5_unintended_diff_only": 0.061750009655952454,
201
- "tpp_threshold_10_total_metric": 0.19750000536441803,
202
  "tpp_threshold_10_intended_diff_only": 0.2940000295639038,
203
- "tpp_threshold_10_unintended_diff_only": 0.09650002419948578
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.14550001919269562,
207
- "tpp_threshold_2_intended_diff_only": 0.15500003099441528,
208
- "tpp_threshold_2_unintended_diff_only": 0.009500011801719666,
209
- "tpp_threshold_5_total_metric": 0.2265000343322754,
210
- "tpp_threshold_5_intended_diff_only": 0.29700005054473877,
211
- "tpp_threshold_5_unintended_diff_only": 0.07050001621246338,
212
- "tpp_threshold_10_total_metric": 0.179500013589859,
213
- "tpp_threshold_10_intended_diff_only": 0.3540000319480896,
214
- "tpp_threshold_10_unintended_diff_only": 0.1745000183582306
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.010750025510787964,
218
  "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
219
- "tpp_threshold_2_unintended_diff_only": -0.002750009298324585,
220
- "tpp_threshold_5_total_metric": 0.24124997854232788,
221
- "tpp_threshold_5_intended_diff_only": 0.3050000071525574,
222
- "tpp_threshold_5_unintended_diff_only": 0.06375002861022949,
223
- "tpp_threshold_10_total_metric": 0.2695000171661377,
224
  "tpp_threshold_10_intended_diff_only": 0.37800002098083496,
225
- "tpp_threshold_10_unintended_diff_only": 0.10850000381469727
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.0037499964237213135,
229
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
230
- "tpp_threshold_2_unintended_diff_only": 0.0012499988079071045,
231
- "tpp_threshold_5_total_metric": 0.14374998211860657,
232
- "tpp_threshold_5_intended_diff_only": 0.203000009059906,
233
- "tpp_threshold_5_unintended_diff_only": 0.05925002694129944,
234
- "tpp_threshold_10_total_metric": 0.18700000643730164,
235
- "tpp_threshold_10_intended_diff_only": 0.25700002908706665,
236
- "tpp_threshold_10_unintended_diff_only": 0.07000002264976501
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.009000003337860107,
242
- "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
243
- "tpp_threshold_2_unintended_diff_only": -0.004999995231628418,
244
- "tpp_threshold_5_total_metric": 0.013249963521957397,
245
- "tpp_threshold_5_intended_diff_only": 0.010999977588653564,
246
- "tpp_threshold_5_unintended_diff_only": -0.002249985933303833,
247
- "tpp_threshold_10_total_metric": 0.0007499903440475464,
248
  "tpp_threshold_10_intended_diff_only": 0.004999995231628418,
249
- "tpp_threshold_10_unintended_diff_only": 0.004250004887580872
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": 0.0037500113248825073,
253
- "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
254
- "tpp_threshold_2_unintended_diff_only": 0.0052499920129776,
255
- "tpp_threshold_5_total_metric": 0.007750064134597778,
256
- "tpp_threshold_5_intended_diff_only": 0.01500004529953003,
257
- "tpp_threshold_5_unintended_diff_only": 0.007249981164932251,
258
- "tpp_threshold_10_total_metric": 0.0310000479221344,
259
- "tpp_threshold_10_intended_diff_only": 0.033000051975250244,
260
- "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.012000009417533875,
264
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
265
- "tpp_threshold_2_unintended_diff_only": -0.0070000141859054565,
266
- "tpp_threshold_5_total_metric": 0.01724998652935028,
267
- "tpp_threshold_5_intended_diff_only": 0.014999985694885254,
268
- "tpp_threshold_5_unintended_diff_only": -0.002250000834465027,
269
- "tpp_threshold_10_total_metric": 0.01799999177455902,
270
- "tpp_threshold_10_intended_diff_only": 0.018999993801116943,
271
- "tpp_threshold_10_unintended_diff_only": 0.0010000020265579224
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.008250042796134949,
275
- "tpp_threshold_2_intended_diff_only": -0.001000046730041504,
276
- "tpp_threshold_2_unintended_diff_only": 0.007249996066093445,
277
- "tpp_threshold_5_total_metric": 0.002249971032142639,
278
- "tpp_threshold_5_intended_diff_only": 0.0029999613761901855,
279
- "tpp_threshold_5_unintended_diff_only": 0.0007499903440475464,
280
- "tpp_threshold_10_total_metric": 0.007499948143959045,
281
- "tpp_threshold_10_intended_diff_only": 0.008999943733215332,
282
- "tpp_threshold_10_unintended_diff_only": 0.0014999955892562866
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.026500016450881958,
286
- "tpp_threshold_2_intended_diff_only": 0.027000010013580322,
287
- "tpp_threshold_2_unintended_diff_only": 0.0004999935626983643,
288
- "tpp_threshold_5_total_metric": 0.030000001192092896,
289
- "tpp_threshold_5_intended_diff_only": 0.03700000047683716,
290
- "tpp_threshold_5_unintended_diff_only": 0.006999999284744263,
291
- "tpp_threshold_10_total_metric": 0.058999985456466675,
292
- "tpp_threshold_10_intended_diff_only": 0.06099998950958252,
293
- "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "05b75ee8-70ef-4f88-803f-f67922902635",
73
+ "datetime_epoch_millis": 1777316762706,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.03249999582767486,
77
+ "tpp_threshold_2_intended_diff_only": 0.03650000095367432,
78
+ "tpp_threshold_2_unintended_diff_only": 0.0040000051259994505,
79
+ "tpp_threshold_5_total_metric": 0.09695000648498535,
80
+ "tpp_threshold_5_intended_diff_only": 0.12340001463890075,
81
+ "tpp_threshold_5_unintended_diff_only": 0.026450008153915405,
82
+ "tpp_threshold_10_total_metric": 0.10630000531673432,
83
+ "tpp_threshold_10_intended_diff_only": 0.1552000105381012,
84
+ "tpp_threshold_10_unintended_diff_only": 0.04890000522136688,
85
+ "tpp_threshold_20_total_metric": 0.1335500106215477,
86
+ "tpp_threshold_20_intended_diff_only": 0.20650001764297488,
87
+ "tpp_threshold_20_unintended_diff_only": 0.07295000702142715,
88
+ "tpp_threshold_50_total_metric": 0.15557500422000886,
89
+ "tpp_threshold_50_intended_diff_only": 0.2589000165462494,
90
+ "tpp_threshold_50_unintended_diff_only": 0.10332501232624054,
91
+ "tpp_threshold_100_total_metric": 0.17610000520944596,
92
+ "tpp_threshold_100_intended_diff_only": 0.3164000153541565,
93
+ "tpp_threshold_100_unintended_diff_only": 0.14030001014471055,
94
+ "tpp_threshold_500_total_metric": 0.23960001170635223,
95
+ "tpp_threshold_500_intended_diff_only": 0.43610002398490905,
96
+ "tpp_threshold_500_unintended_diff_only": 0.19650001227855682
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.05729999244213104,
103
+ "tpp_threshold_2_intended_diff_only": 0.06440000534057617,
104
+ "tpp_threshold_2_unintended_diff_only": 0.007100012898445129,
105
+ "tpp_threshold_5_total_metric": 0.18060002028942107,
106
+ "tpp_threshold_5_intended_diff_only": 0.2330000400543213,
107
+ "tpp_threshold_5_unintended_diff_only": 0.052400019764900205,
108
+ "tpp_threshold_10_total_metric": 0.18715001344680787,
109
  "tpp_threshold_10_intended_diff_only": 0.28240002393722535,
110
+ "tpp_threshold_10_unintended_diff_only": 0.09525001049041748,
111
+ "tpp_threshold_20_total_metric": 0.21870001554489135,
112
+ "tpp_threshold_20_intended_diff_only": 0.36140003204345705,
113
+ "tpp_threshold_20_unintended_diff_only": 0.14270001649856567,
114
+ "tpp_threshold_50_total_metric": 0.20645001232624055,
115
+ "tpp_threshold_50_intended_diff_only": 0.40660003423690794,
116
+ "tpp_threshold_50_unintended_diff_only": 0.20015002191066741,
117
+ "tpp_threshold_100_total_metric": 0.16890000998973848,
118
+ "tpp_threshold_100_intended_diff_only": 0.4372000336647034,
119
+ "tpp_threshold_100_unintended_diff_only": 0.2683000236749649,
120
+ "tpp_threshold_500_total_metric": 0.10315001010894775,
121
+ "tpp_threshold_500_intended_diff_only": 0.46720004081726074,
122
+ "tpp_threshold_500_unintended_diff_only": 0.364050030708313
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.007699999213218689,
127
+ "tpp_threshold_2_intended_diff_only": 0.008599996566772461,
128
+ "tpp_threshold_2_unintended_diff_only": 0.000899997353553772,
129
+ "tpp_threshold_5_total_metric": 0.013299992680549622,
130
+ "tpp_threshold_5_intended_diff_only": 0.013799989223480224,
131
+ "tpp_threshold_5_unintended_diff_only": 0.000499996542930603,
132
+ "tpp_threshold_10_total_metric": 0.025449997186660765,
133
+ "tpp_threshold_10_intended_diff_only": 0.02799999713897705,
134
+ "tpp_threshold_10_unintended_diff_only": 0.0025499999523162843,
135
+ "tpp_threshold_20_total_metric": 0.04840000569820404,
136
+ "tpp_threshold_20_intended_diff_only": 0.05160000324249268,
137
+ "tpp_threshold_20_unintended_diff_only": 0.0031999975442886354,
138
+ "tpp_threshold_50_total_metric": 0.10469999611377716,
139
+ "tpp_threshold_50_intended_diff_only": 0.11119999885559081,
140
+ "tpp_threshold_50_unintended_diff_only": 0.0065000027418136595,
141
+ "tpp_threshold_100_total_metric": 0.18330000042915345,
142
+ "tpp_threshold_100_intended_diff_only": 0.1955999970436096,
143
+ "tpp_threshold_100_unintended_diff_only": 0.012299996614456177,
144
+ "tpp_threshold_500_total_metric": 0.3760500133037567,
145
+ "tpp_threshold_500_intended_diff_only": 0.40500000715255735,
146
+ "tpp_threshold_500_unintended_diff_only": 0.02894999384880066
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.013250008225440979,
194
  "tpp_threshold_10_total_metric": 0.11374999582767487,
195
  "tpp_threshold_10_intended_diff_only": 0.1290000081062317,
196
+ "tpp_threshold_10_unintended_diff_only": 0.015250012278556824,
197
+ "tpp_threshold_20_total_metric": 0.21850000321865082,
198
+ "tpp_threshold_20_intended_diff_only": 0.3500000238418579,
199
+ "tpp_threshold_20_unintended_diff_only": 0.1315000206232071,
200
+ "tpp_threshold_50_total_metric": 0.23600000143051147,
201
+ "tpp_threshold_50_intended_diff_only": 0.4020000100135803,
202
+ "tpp_threshold_50_unintended_diff_only": 0.16600000858306885,
203
+ "tpp_threshold_100_total_metric": 0.1992499977350235,
204
+ "tpp_threshold_100_intended_diff_only": 0.4390000104904175,
205
+ "tpp_threshold_100_unintended_diff_only": 0.23975001275539398,
206
+ "tpp_threshold_500_total_metric": 0.10874998569488525,
207
+ "tpp_threshold_500_intended_diff_only": 0.45600003004074097,
208
+ "tpp_threshold_500_unintended_diff_only": 0.3472500443458557
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.05149999260902405,
212
+ "tpp_threshold_2_intended_diff_only": 0.05900001525878906,
213
+ "tpp_threshold_2_unintended_diff_only": 0.007500022649765015,
214
+ "tpp_threshold_5_total_metric": 0.1810000091791153,
215
  "tpp_threshold_5_intended_diff_only": 0.2420000433921814,
216
+ "tpp_threshold_5_unintended_diff_only": 0.0610000342130661,
217
+ "tpp_threshold_10_total_metric": 0.19075001776218414,
218
  "tpp_threshold_10_intended_diff_only": 0.2940000295639038,
219
+ "tpp_threshold_10_unintended_diff_only": 0.10325001180171967,
220
+ "tpp_threshold_20_total_metric": 0.21824999153614044,
221
+ "tpp_threshold_20_intended_diff_only": 0.3370000123977661,
222
+ "tpp_threshold_20_unintended_diff_only": 0.11875002086162567,
223
+ "tpp_threshold_50_total_metric": 0.1939999908208847,
224
+ "tpp_threshold_50_intended_diff_only": 0.36900001764297485,
225
+ "tpp_threshold_50_unintended_diff_only": 0.17500002682209015,
226
+ "tpp_threshold_100_total_metric": 0.17275001108646393,
227
+ "tpp_threshold_100_intended_diff_only": 0.41300004720687866,
228
+ "tpp_threshold_100_unintended_diff_only": 0.24025003612041473,
229
+ "tpp_threshold_500_total_metric": 0.09325002133846283,
230
+ "tpp_threshold_500_intended_diff_only": 0.4590000510215759,
231
+ "tpp_threshold_500_unintended_diff_only": 0.3657500296831131
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.13374999165534973,
235
+ "tpp_threshold_2_intended_diff_only": 0.15200001001358032,
236
+ "tpp_threshold_2_unintended_diff_only": 0.01825001835823059,
237
+ "tpp_threshold_5_total_metric": 0.23100005090236664,
238
+ "tpp_threshold_5_intended_diff_only": 0.29600006341934204,
239
+ "tpp_threshold_5_unintended_diff_only": 0.0650000125169754,
240
+ "tpp_threshold_10_total_metric": 0.17475000023841858,
241
+ "tpp_threshold_10_intended_diff_only": 0.35500001907348633,
242
+ "tpp_threshold_10_unintended_diff_only": 0.18025001883506775,
243
+ "tpp_threshold_20_total_metric": 0.14325003325939178,
244
+ "tpp_threshold_20_intended_diff_only": 0.3720000386238098,
245
+ "tpp_threshold_20_unintended_diff_only": 0.22875000536441803,
246
+ "tpp_threshold_50_total_metric": 0.10474999248981476,
247
+ "tpp_threshold_50_intended_diff_only": 0.3920000195503235,
248
+ "tpp_threshold_50_unintended_diff_only": 0.28725002706050873,
249
+ "tpp_threshold_100_total_metric": 0.09550002217292786,
250
+ "tpp_threshold_100_intended_diff_only": 0.42200005054473877,
251
+ "tpp_threshold_100_unintended_diff_only": 0.3265000283718109,
252
+ "tpp_threshold_500_total_metric": 0.05274999141693115,
253
+ "tpp_threshold_500_intended_diff_only": 0.4520000219345093,
254
+ "tpp_threshold_500_unintended_diff_only": 0.3992500305175781
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.00849999487400055,
258
  "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704,
260
+ "tpp_threshold_5_total_metric": 0.2460000365972519,
261
+ "tpp_threshold_5_intended_diff_only": 0.3060000538825989,
262
+ "tpp_threshold_5_unintended_diff_only": 0.060000017285346985,
263
+ "tpp_threshold_10_total_metric": 0.2737500071525574,
264
  "tpp_threshold_10_intended_diff_only": 0.37800002098083496,
265
+ "tpp_threshold_10_unintended_diff_only": 0.10425001382827759,
266
+ "tpp_threshold_20_total_metric": 0.2937500327825546,
267
+ "tpp_threshold_20_intended_diff_only": 0.4360000491142273,
268
+ "tpp_threshold_20_unintended_diff_only": 0.14225001633167267,
269
+ "tpp_threshold_50_total_metric": 0.26350003480911255,
270
+ "tpp_threshold_50_intended_diff_only": 0.47700005769729614,
271
+ "tpp_threshold_50_unintended_diff_only": 0.2135000228881836,
272
+ "tpp_threshold_100_total_metric": 0.19550001621246338,
273
+ "tpp_threshold_100_intended_diff_only": 0.4830000400543213,
274
+ "tpp_threshold_100_unintended_diff_only": 0.2875000238418579,
275
+ "tpp_threshold_500_total_metric": 0.13250000774860382,
276
+ "tpp_threshold_500_intended_diff_only": 0.4930000305175781,
277
+ "tpp_threshold_500_unintended_diff_only": 0.3605000227689743
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.0072500258684158325,
281
+ "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464,
283
+ "tpp_threshold_5_total_metric": 0.14125002920627594,
284
+ "tpp_threshold_5_intended_diff_only": 0.2040000557899475,
285
+ "tpp_threshold_5_unintended_diff_only": 0.06275002658367157,
286
+ "tpp_threshold_10_total_metric": 0.18275004625320435,
287
+ "tpp_threshold_10_intended_diff_only": 0.2560000419616699,
288
+ "tpp_threshold_10_unintended_diff_only": 0.07324999570846558,
289
+ "tpp_threshold_20_total_metric": 0.21975001692771912,
290
+ "tpp_threshold_20_intended_diff_only": 0.312000036239624,
291
+ "tpp_threshold_20_unintended_diff_only": 0.09225001931190491,
292
+ "tpp_threshold_50_total_metric": 0.2340000420808792,
293
+ "tpp_threshold_50_intended_diff_only": 0.393000066280365,
294
+ "tpp_threshold_50_unintended_diff_only": 0.15900002419948578,
295
+ "tpp_threshold_100_total_metric": 0.18150000274181366,
296
+ "tpp_threshold_100_intended_diff_only": 0.42900002002716064,
297
+ "tpp_threshold_100_unintended_diff_only": 0.24750001728534698,
298
+ "tpp_threshold_500_total_metric": 0.1285000443458557,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.3475000262260437
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.002749994397163391,
306
+ "tpp_threshold_2_intended_diff_only": 0.0059999823570251465,
307
+ "tpp_threshold_2_unintended_diff_only": 0.0032499879598617554,
308
+ "tpp_threshold_5_total_metric": 0.007500007748603821,
309
+ "tpp_threshold_5_intended_diff_only": 0.009000003337860107,
310
+ "tpp_threshold_5_unintended_diff_only": 0.0014999955892562866,
311
+ "tpp_threshold_10_total_metric": 0.0032500028610229492,
312
  "tpp_threshold_10_intended_diff_only": 0.004999995231628418,
313
+ "tpp_threshold_10_unintended_diff_only": 0.0017499923706054688,
314
+ "tpp_threshold_20_total_metric": 0.018499985337257385,
315
+ "tpp_threshold_20_intended_diff_only": 0.019999980926513672,
316
+ "tpp_threshold_20_unintended_diff_only": 0.0014999955892562866,
317
+ "tpp_threshold_50_total_metric": 0.05600002408027649,
318
+ "tpp_threshold_50_intended_diff_only": 0.058000028133392334,
319
+ "tpp_threshold_50_unintended_diff_only": 0.0020000040531158447,
320
+ "tpp_threshold_100_total_metric": 0.1354999840259552,
321
+ "tpp_threshold_100_intended_diff_only": 0.1499999761581421,
322
+ "tpp_threshold_100_unintended_diff_only": 0.01449999213218689,
323
+ "tpp_threshold_500_total_metric": 0.4037500023841858,
324
+ "tpp_threshold_500_intended_diff_only": 0.4259999990463257,
325
+ "tpp_threshold_500_unintended_diff_only": 0.022249996662139893
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.013250038027763367,
329
+ "tpp_threshold_2_intended_diff_only": 0.01100003719329834,
330
+ "tpp_threshold_2_unintended_diff_only": -0.002250000834465027,
331
+ "tpp_threshold_5_total_metric": 0.004000052809715271,
332
+ "tpp_threshold_5_intended_diff_only": 0.01100003719329834,
333
+ "tpp_threshold_5_unintended_diff_only": 0.006999984383583069,
334
+ "tpp_threshold_10_total_metric": 0.03100001811981201,
335
+ "tpp_threshold_10_intended_diff_only": 0.03600001335144043,
336
+ "tpp_threshold_10_unintended_diff_only": 0.004999995231628418,
337
+ "tpp_threshold_20_total_metric": 0.0792500227689743,
338
+ "tpp_threshold_20_intended_diff_only": 0.0820000171661377,
339
+ "tpp_threshold_20_unintended_diff_only": 0.002749994397163391,
340
+ "tpp_threshold_50_total_metric": 0.15000002086162567,
341
+ "tpp_threshold_50_intended_diff_only": 0.16100001335144043,
342
+ "tpp_threshold_50_unintended_diff_only": 0.010999992489814758,
343
+ "tpp_threshold_100_total_metric": 0.24075005948543549,
344
+ "tpp_threshold_100_intended_diff_only": 0.2600000500679016,
345
+ "tpp_threshold_100_unintended_diff_only": 0.019249990582466125,
346
+ "tpp_threshold_500_total_metric": 0.40950004756450653,
347
+ "tpp_threshold_500_intended_diff_only": 0.437000036239624,
348
+ "tpp_threshold_500_unintended_diff_only": 0.027499988675117493
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.0032499730587005615,
352
+ "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
353
+ "tpp_threshold_2_unintended_diff_only": -0.0012499988079071045,
354
+ "tpp_threshold_5_total_metric": 0.011499986052513123,
355
+ "tpp_threshold_5_intended_diff_only": 0.0059999823570251465,
356
+ "tpp_threshold_5_unintended_diff_only": -0.005500003695487976,
357
+ "tpp_threshold_10_total_metric": 0.016749978065490723,
358
+ "tpp_threshold_10_intended_diff_only": 0.014999985694885254,
359
+ "tpp_threshold_10_unintended_diff_only": -0.0017499923706054688,
360
+ "tpp_threshold_20_total_metric": 0.00950002670288086,
361
+ "tpp_threshold_20_intended_diff_only": 0.012000024318695068,
362
+ "tpp_threshold_20_unintended_diff_only": 0.002499997615814209,
363
+ "tpp_threshold_50_total_metric": 0.059749990701675415,
364
+ "tpp_threshold_50_intended_diff_only": 0.06499999761581421,
365
+ "tpp_threshold_50_unintended_diff_only": 0.005250006914138794,
366
+ "tpp_threshold_100_total_metric": 0.10475000739097595,
367
+ "tpp_threshold_100_intended_diff_only": 0.11500000953674316,
368
+ "tpp_threshold_100_unintended_diff_only": 0.010250002145767212,
369
+ "tpp_threshold_500_total_metric": 0.3474999964237213,
370
+ "tpp_threshold_500_intended_diff_only": 0.3799999952316284,
371
+ "tpp_threshold_500_unintended_diff_only": 0.032499998807907104
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.0012499839067459106,
375
+ "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
376
+ "tpp_threshold_2_unintended_diff_only": -0.00024999678134918213,
377
+ "tpp_threshold_5_total_metric": 0.010499924421310425,
378
+ "tpp_threshold_5_intended_diff_only": 0.008999943733215332,
379
+ "tpp_threshold_5_unintended_diff_only": -0.0014999806880950928,
380
+ "tpp_threshold_10_total_metric": 0.005499988794326782,
381
+ "tpp_threshold_10_intended_diff_only": 0.009999990463256836,
382
+ "tpp_threshold_10_unintended_diff_only": 0.004500001668930054,
383
+ "tpp_threshold_20_total_metric": 0.027749985456466675,
384
+ "tpp_threshold_20_intended_diff_only": 0.02799999713897705,
385
+ "tpp_threshold_20_unintended_diff_only": 0.000250011682510376,
386
+ "tpp_threshold_50_total_metric": 0.07424995303153992,
387
+ "tpp_threshold_50_intended_diff_only": 0.08199995756149292,
388
+ "tpp_threshold_50_unintended_diff_only": 0.007750004529953003,
389
+ "tpp_threshold_100_total_metric": 0.17574995756149292,
390
+ "tpp_threshold_100_intended_diff_only": 0.1839999556541443,
391
+ "tpp_threshold_100_unintended_diff_only": 0.008249998092651367,
392
+ "tpp_threshold_500_total_metric": 0.3725000023841858,
393
+ "tpp_threshold_500_intended_diff_only": 0.4089999794960022,
394
+ "tpp_threshold_500_unintended_diff_only": 0.036499977111816406
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.018000006675720215,
398
+ "tpp_threshold_2_intended_diff_only": 0.023000001907348633,
399
+ "tpp_threshold_2_unintended_diff_only": 0.004999995231628418,
400
+ "tpp_threshold_5_total_metric": 0.03299999237060547,
401
+ "tpp_threshold_5_intended_diff_only": 0.0339999794960022,
402
+ "tpp_threshold_5_unintended_diff_only": 0.0009999871253967285,
403
+ "tpp_threshold_10_total_metric": 0.07074999809265137,
404
+ "tpp_threshold_10_intended_diff_only": 0.07400000095367432,
405
+ "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492,
406
+ "tpp_threshold_20_total_metric": 0.10700000822544098,
407
+ "tpp_threshold_20_intended_diff_only": 0.11599999666213989,
408
+ "tpp_threshold_20_unintended_diff_only": 0.008999988436698914,
409
+ "tpp_threshold_50_total_metric": 0.1834999918937683,
410
+ "tpp_threshold_50_intended_diff_only": 0.1899999976158142,
411
+ "tpp_threshold_50_unintended_diff_only": 0.0065000057220458984,
412
+ "tpp_threshold_100_total_metric": 0.25974999368190765,
413
+ "tpp_threshold_100_intended_diff_only": 0.26899999380111694,
414
+ "tpp_threshold_100_unintended_diff_only": 0.00925000011920929,
415
+ "tpp_threshold_500_total_metric": 0.34700001776218414,
416
+ "tpp_threshold_500_intended_diff_only": 0.37300002574920654,
417
+ "tpp_threshold_500_unintended_diff_only": 0.0260000079870224
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-207519-tokens-850000000/scr/saebench_scr_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,8 +69,8 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "ffe01158-7a7e-449c-945a-589f0b31c74c",
69
- "datetime_epoch_millis": 1776884870701,
70
  "eval_result_metrics": {
71
  "scr_metrics": {
72
  "scr_dir1_threshold_2": 0.27512449920371496,
@@ -78,18 +82,18 @@
78
  "scr_dir1_threshold_10": 0.22405199724731734,
79
  "scr_metric_threshold_10": 0.20891156459134436,
80
  "scr_dir2_threshold_10": 0.20149287130608984,
81
- "scr_dir1_threshold_20": null,
82
- "scr_metric_threshold_20": null,
83
- "scr_dir2_threshold_20": null,
84
- "scr_dir1_threshold_50": null,
85
- "scr_metric_threshold_50": null,
86
- "scr_dir2_threshold_50": null,
87
- "scr_dir1_threshold_100": null,
88
- "scr_metric_threshold_100": null,
89
- "scr_dir2_threshold_100": null,
90
- "scr_dir1_threshold_500": null,
91
- "scr_metric_threshold_500": null,
92
- "scr_dir2_threshold_500": null
93
  }
94
  },
95
  "eval_result_details": [
@@ -104,18 +108,18 @@
104
  "scr_dir1_threshold_10": 0.5256410746257707,
105
  "scr_metric_threshold_10": 0.05958549142789553,
106
  "scr_dir2_threshold_10": 0.05958549142789553,
107
- "scr_dir1_threshold_20": null,
108
- "scr_metric_threshold_20": null,
109
- "scr_dir2_threshold_20": null,
110
- "scr_dir1_threshold_50": null,
111
- "scr_metric_threshold_50": null,
112
- "scr_dir2_threshold_50": null,
113
- "scr_dir1_threshold_100": null,
114
- "scr_metric_threshold_100": null,
115
- "scr_dir2_threshold_100": null,
116
- "scr_dir1_threshold_500": null,
117
- "scr_metric_threshold_500": null,
118
- "scr_dir2_threshold_500": null
119
  },
120
  {
121
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
@@ -128,18 +132,18 @@
128
  "scr_dir1_threshold_10": 0.3560604589646946,
129
  "scr_metric_threshold_10": 0.1804281691167759,
130
  "scr_dir2_threshold_10": 0.1804281691167759,
131
- "scr_dir1_threshold_20": null,
132
- "scr_metric_threshold_20": null,
133
- "scr_dir2_threshold_20": null,
134
- "scr_dir1_threshold_50": null,
135
- "scr_metric_threshold_50": null,
136
- "scr_dir2_threshold_50": null,
137
- "scr_dir1_threshold_100": null,
138
- "scr_metric_threshold_100": null,
139
- "scr_dir2_threshold_100": null,
140
- "scr_dir1_threshold_500": null,
141
- "scr_metric_threshold_500": null,
142
- "scr_dir2_threshold_500": null
143
  },
144
  {
145
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
@@ -152,18 +156,18 @@
152
  "scr_dir1_threshold_10": -0.04687532014208255,
153
  "scr_metric_threshold_10": 0.16331655168313458,
154
  "scr_dir2_threshold_10": 0.16331655168313458,
155
- "scr_dir1_threshold_20": null,
156
- "scr_metric_threshold_20": null,
157
- "scr_dir2_threshold_20": null,
158
- "scr_dir1_threshold_50": null,
159
- "scr_metric_threshold_50": null,
160
- "scr_dir2_threshold_50": null,
161
- "scr_dir1_threshold_100": null,
162
- "scr_metric_threshold_100": null,
163
- "scr_dir2_threshold_100": null,
164
- "scr_dir1_threshold_500": null,
165
- "scr_metric_threshold_500": null,
166
- "scr_dir2_threshold_500": null
167
  },
168
  {
169
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
@@ -176,18 +180,18 @@
176
  "scr_dir1_threshold_10": 0.013422646966463385,
177
  "scr_metric_threshold_10": 0.08917196243161127,
178
  "scr_dir2_threshold_10": 0.08917196243161127,
179
- "scr_dir1_threshold_20": null,
180
- "scr_metric_threshold_20": null,
181
- "scr_dir2_threshold_20": null,
182
- "scr_dir1_threshold_50": null,
183
- "scr_metric_threshold_50": null,
184
- "scr_dir2_threshold_50": null,
185
- "scr_dir1_threshold_100": null,
186
- "scr_metric_threshold_100": null,
187
- "scr_dir2_threshold_100": null,
188
- "scr_dir1_threshold_500": null,
189
- "scr_metric_threshold_500": null,
190
- "scr_dir2_threshold_500": null
191
  },
192
  {
193
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
@@ -200,18 +204,18 @@
200
  "scr_dir1_threshold_10": 0.08421069803000372,
201
  "scr_metric_threshold_10": 0.45238098053884956,
202
  "scr_dir2_threshold_10": 0.45238098053884956,
203
- "scr_dir1_threshold_20": null,
204
- "scr_metric_threshold_20": null,
205
- "scr_dir2_threshold_20": null,
206
- "scr_dir1_threshold_50": null,
207
- "scr_metric_threshold_50": null,
208
- "scr_dir2_threshold_50": null,
209
- "scr_dir1_threshold_100": null,
210
- "scr_metric_threshold_100": null,
211
- "scr_dir2_threshold_100": null,
212
- "scr_dir1_threshold_500": null,
213
- "scr_metric_threshold_500": null,
214
- "scr_dir2_threshold_500": null
215
  },
216
  {
217
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
@@ -224,18 +228,18 @@
224
  "scr_dir1_threshold_10": 0.371584539269045,
225
  "scr_metric_threshold_10": 0.14122136275770192,
226
  "scr_dir2_threshold_10": 0.14122136275770192,
227
- "scr_dir1_threshold_20": null,
228
- "scr_metric_threshold_20": null,
229
- "scr_dir2_threshold_20": null,
230
- "scr_dir1_threshold_50": null,
231
- "scr_metric_threshold_50": null,
232
- "scr_dir2_threshold_50": null,
233
- "scr_dir1_threshold_100": null,
234
- "scr_metric_threshold_100": null,
235
- "scr_dir2_threshold_100": null,
236
- "scr_dir1_threshold_500": null,
237
- "scr_metric_threshold_500": null,
238
- "scr_dir2_threshold_500": null
239
  },
240
  {
241
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
@@ -248,18 +252,18 @@
248
  "scr_dir1_threshold_10": 0.3023255233702845,
249
  "scr_metric_threshold_10": 0.39914164188042695,
250
  "scr_dir2_threshold_10": 0.39914164188042695,
251
- "scr_dir1_threshold_20": null,
252
- "scr_metric_threshold_20": null,
253
- "scr_dir2_threshold_20": null,
254
- "scr_dir1_threshold_50": null,
255
- "scr_metric_threshold_50": null,
256
- "scr_dir2_threshold_50": null,
257
- "scr_dir1_threshold_100": null,
258
- "scr_metric_threshold_100": null,
259
- "scr_dir2_threshold_100": null,
260
- "scr_dir1_threshold_500": null,
261
- "scr_metric_threshold_500": null,
262
- "scr_dir2_threshold_500": null
263
  },
264
  {
265
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
@@ -272,18 +276,18 @@
272
  "scr_dir1_threshold_10": 0.18604635689435922,
273
  "scr_metric_threshold_10": 0.18604635689435922,
274
  "scr_dir2_threshold_10": 0.12669681061232305,
275
- "scr_dir1_threshold_20": null,
276
- "scr_metric_threshold_20": null,
277
- "scr_dir2_threshold_20": null,
278
- "scr_dir1_threshold_50": null,
279
- "scr_metric_threshold_50": null,
280
- "scr_dir2_threshold_50": null,
281
- "scr_dir1_threshold_100": null,
282
- "scr_metric_threshold_100": null,
283
- "scr_dir2_threshold_100": null,
284
- "scr_dir1_threshold_500": null,
285
- "scr_metric_threshold_500": null,
286
- "scr_dir2_threshold_500": null
287
  }
288
  ],
289
  "sae_bench_commit_hash": "Unknown",
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "779ccae0-952e-42cb-a4f6-3510240b489d",
73
+ "datetime_epoch_millis": 1777316491341,
74
  "eval_result_metrics": {
75
  "scr_metrics": {
76
  "scr_dir1_threshold_2": 0.27512449920371496,
 
82
  "scr_dir1_threshold_10": 0.22405199724731734,
83
  "scr_metric_threshold_10": 0.20891156459134436,
84
  "scr_dir2_threshold_10": 0.20149287130608984,
85
+ "scr_dir1_threshold_20": 0.3165134937976086,
86
+ "scr_metric_threshold_20": 0.2213786835087885,
87
+ "scr_dir2_threshold_20": 0.20979549805083175,
88
+ "scr_dir1_threshold_50": 0.36370478567064457,
89
+ "scr_metric_threshold_50": 0.3134490732198989,
90
+ "scr_dir2_threshold_50": 0.3034364550289833,
91
+ "scr_dir1_threshold_100": 0.1412199235950044,
92
+ "scr_metric_threshold_100": 0.345270308848593,
93
+ "scr_dir2_threshold_100": 0.3432393798501045,
94
+ "scr_dir1_threshold_500": -1.0077250248002592,
95
+ "scr_metric_threshold_500": 0.19186126401474754,
96
+ "scr_dir2_threshold_500": 0.17056536767374397
97
  }
98
  },
99
  "eval_result_details": [
 
108
  "scr_dir1_threshold_10": 0.5256410746257707,
109
  "scr_metric_threshold_10": 0.05958549142789553,
110
  "scr_dir2_threshold_10": 0.05958549142789553,
111
+ "scr_dir1_threshold_20": 0.47435892537422925,
112
+ "scr_metric_threshold_20": 0.1062176285815671,
113
+ "scr_dir2_threshold_20": 0.1062176285815671,
114
+ "scr_dir1_threshold_50": 0.5000003820810118,
115
+ "scr_metric_threshold_50": 0.19170982855791055,
116
+ "scr_dir2_threshold_50": 0.19170982855791055,
117
+ "scr_dir1_threshold_100": 0.29487178507484585,
118
+ "scr_metric_threshold_100": 0.32383416568792556,
119
+ "scr_dir2_threshold_100": 0.32383416568792556,
120
+ "scr_dir1_threshold_500": -3.217948179116522,
121
+ "scr_metric_threshold_500": 0.044041497182059436,
122
+ "scr_dir2_threshold_500": 0.044041497182059436
123
  },
124
  {
125
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
 
132
  "scr_dir1_threshold_10": 0.3560604589646946,
133
  "scr_metric_threshold_10": 0.1804281691167759,
134
  "scr_dir2_threshold_10": 0.1804281691167759,
135
+ "scr_dir1_threshold_20": 0.27272731377729453,
136
+ "scr_metric_threshold_20": 0.08256883242346637,
137
+ "scr_dir2_threshold_20": 0.08256883242346637,
138
+ "scr_dir1_threshold_50": 0.5075758841466581,
139
+ "scr_metric_threshold_50": 0.46483180372392296,
140
+ "scr_dir2_threshold_50": 0.46483180372392296,
141
+ "scr_dir1_threshold_100": 0.12121234014557077,
142
+ "scr_metric_threshold_100": 0.6177370286995434,
143
+ "scr_dir2_threshold_100": 0.6177370286995434,
144
+ "scr_dir1_threshold_500": -0.5303028592612729,
145
+ "scr_metric_threshold_500": -0.1467889139013698,
146
+ "scr_dir2_threshold_500": -0.1467889139013698
147
  },
148
  {
149
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
 
156
  "scr_dir1_threshold_10": -0.04687532014208255,
157
  "scr_metric_threshold_10": 0.16331655168313458,
158
  "scr_dir2_threshold_10": 0.16331655168313458,
159
+ "scr_dir1_threshold_20": 0.28125005820765137,
160
+ "scr_metric_threshold_20": 0.20351763196473802,
161
+ "scr_dir2_threshold_20": 0.20351763196473802,
162
+ "scr_dir1_threshold_50": 0.4062502910382569,
163
+ "scr_metric_threshold_50": 0.21859296219014387,
164
+ "scr_dir2_threshold_50": 0.21859296219014387,
165
+ "scr_dir1_threshold_100": -0.6093745052349633,
166
+ "scr_metric_threshold_100": 0.128140681316927,
167
+ "scr_dir2_threshold_100": 0.128140681316927,
168
+ "scr_dir1_threshold_500": -4.187499417923486,
169
+ "scr_metric_threshold_500": -0.005025060155005,
170
+ "scr_dir2_threshold_500": -0.005025060155005
171
  },
172
  {
173
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
 
180
  "scr_dir1_threshold_10": 0.013422646966463385,
181
  "scr_metric_threshold_10": 0.08917196243161127,
182
  "scr_dir2_threshold_10": 0.08917196243161127,
183
+ "scr_dir1_threshold_20": 0.39597288588509905,
184
+ "scr_metric_threshold_20": -0.08917196243161127,
185
+ "scr_dir2_threshold_20": -0.08917196243161127,
186
+ "scr_dir1_threshold_50": 0.4630873208110233,
187
+ "scr_metric_threshold_50": -0.08917196243161127,
188
+ "scr_dir2_threshold_50": -0.08917196243161127,
189
+ "scr_dir1_threshold_100": 0.3422818979880431,
190
+ "scr_metric_threshold_100": -0.15605083934346814,
191
+ "scr_dir2_threshold_100": -0.15605083934346814,
192
+ "scr_dir1_threshold_500": -1.3557049449857088,
193
+ "scr_metric_threshold_500": -0.5350317745896676,
194
+ "scr_dir2_threshold_500": -0.5350317745896676
195
  },
196
  {
197
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
 
204
  "scr_dir1_threshold_10": 0.08421069803000372,
205
  "scr_metric_threshold_10": 0.45238098053884956,
206
  "scr_dir2_threshold_10": 0.45238098053884956,
207
+ "scr_dir1_threshold_20": 0.08947378822971379,
208
+ "scr_metric_threshold_20": 0.543650817115708,
209
+ "scr_dir2_threshold_20": 0.543650817115708,
210
+ "scr_dir1_threshold_50": -0.16315799215163673,
211
+ "scr_metric_threshold_50": 0.654761901946115,
212
+ "scr_dir2_threshold_50": 0.654761901946115,
213
+ "scr_dir1_threshold_100": -0.17894726275076694,
214
+ "scr_metric_threshold_100": 0.6666667455087788,
215
+ "scr_dir2_threshold_100": 0.6666667455087788,
216
+ "scr_dir1_threshold_500": -0.14736840784384586,
217
+ "scr_metric_threshold_500": 0.7857142350300708,
218
+ "scr_dir2_threshold_500": 0.7857142350300708
219
  },
220
  {
221
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
 
228
  "scr_dir1_threshold_10": 0.371584539269045,
229
  "scr_metric_threshold_10": 0.14122136275770192,
230
  "scr_dir2_threshold_10": 0.14122136275770192,
231
+ "scr_dir1_threshold_20": 0.4043716594522235,
232
+ "scr_metric_threshold_20": 0.20992367193697561,
233
+ "scr_dir2_threshold_20": 0.20992367193697561,
234
+ "scr_dir1_threshold_50": 0.4098360709132541,
235
+ "scr_metric_threshold_50": 0.25954203162185363,
236
+ "scr_dir2_threshold_50": 0.25954203162185363,
237
+ "scr_dir1_threshold_100": 0.4480872768489657,
238
+ "scr_metric_threshold_100": 0.35877852349298656,
239
+ "scr_dir2_threshold_100": 0.35877852349298656,
240
+ "scr_dir1_threshold_500": 0.5956283405477766,
241
+ "scr_metric_threshold_500": 0.4007633943795555,
242
+ "scr_dir2_threshold_500": 0.4007633943795555
243
  },
244
  {
245
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
 
252
  "scr_dir1_threshold_10": 0.3023255233702845,
253
  "scr_metric_threshold_10": 0.39914164188042695,
254
  "scr_dir2_threshold_10": 0.39914164188042695,
255
+ "scr_dir1_threshold_20": 0.3674417856772556,
256
+ "scr_metric_threshold_20": 0.46781131470206244,
257
+ "scr_dir2_threshold_20": 0.46781131470206244,
258
+ "scr_dir1_threshold_50": 0.5023254124779394,
259
+ "scr_metric_threshold_50": 0.5236051041022068,
260
+ "scr_dir2_threshold_50": 0.5236051041022068,
261
+ "scr_dir1_threshold_100": 0.4465116317753144,
262
+ "scr_metric_threshold_100": 0.5579399405130246,
263
+ "scr_dir2_threshold_100": 0.5579399405130246,
264
+ "scr_dir1_threshold_500": 0.3395348953259431,
265
+ "scr_metric_threshold_500": 0.549356359317294,
266
+ "scr_dir2_threshold_500": 0.549356359317294
267
  },
268
  {
269
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
 
276
  "scr_dir1_threshold_10": 0.18604635689435922,
277
  "scr_metric_threshold_10": 0.18604635689435922,
278
  "scr_dir2_threshold_10": 0.12669681061232305,
279
+ "scr_dir1_threshold_20": 0.24651153377740184,
280
+ "scr_metric_threshold_20": 0.24651153377740184,
281
+ "scr_dir2_threshold_20": 0.15384605011374777,
282
+ "scr_dir1_threshold_50": 0.2837209160486496,
283
+ "scr_metric_threshold_50": 0.2837209160486496,
284
+ "scr_dir2_threshold_50": 0.20361997052132483,
285
+ "scr_dir1_threshold_100": 0.2651162249130257,
286
+ "scr_metric_threshold_100": 0.2651162249130257,
287
+ "scr_dir2_threshold_100": 0.24886879292511796,
288
+ "scr_dir1_threshold_500": 0.44186037485504287,
289
+ "scr_metric_threshold_500": 0.44186037485504287,
290
+ "scr_dir2_threshold_500": 0.2714932041270145
291
  }
292
  ],
293
  "sae_bench_commit_hash": "Unknown",
gemma-2-2b-btk/k-100/seed-0/snapshots/step-207519-tokens-850000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "6235b8d8-7ed2-4ec6-9cfc-9e062b53e4f6",
69
- "datetime_epoch_millis": 1776885041130,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.035725003480911253,
73
- "tpp_threshold_2_intended_diff_only": 0.03910000920295715,
74
- "tpp_threshold_2_unintended_diff_only": 0.0033750057220458983,
75
- "tpp_threshold_5_total_metric": 0.08787501454353332,
76
- "tpp_threshold_5_intended_diff_only": 0.11270002126693726,
77
- "tpp_threshold_5_unintended_diff_only": 0.02482500672340393,
78
- "tpp_threshold_10_total_metric": 0.11080000847578049,
79
- "tpp_threshold_10_intended_diff_only": 0.1576000154018402,
80
- "tpp_threshold_10_unintended_diff_only": 0.04680000692605972,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.06110000014305115,
99
- "tpp_threshold_2_intended_diff_only": 0.06780000925064086,
100
- "tpp_threshold_2_unintended_diff_only": 0.006700009107589722,
101
- "tpp_threshold_5_total_metric": 0.16075001955032348,
102
- "tpp_threshold_5_intended_diff_only": 0.20820003747940063,
103
- "tpp_threshold_5_unintended_diff_only": 0.047450017929077146,
104
- "tpp_threshold_10_total_metric": 0.19885001182556153,
105
  "tpp_threshold_10_intended_diff_only": 0.28940002918243407,
106
- "tpp_threshold_10_unintended_diff_only": 0.09055001735687256,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.010350006818771362,
123
- "tpp_threshold_2_intended_diff_only": 0.010400009155273438,
124
- "tpp_threshold_2_unintended_diff_only": 5.0002336502075194e-05,
125
- "tpp_threshold_5_total_metric": 0.015000009536743164,
126
- "tpp_threshold_5_intended_diff_only": 0.017200005054473878,
127
- "tpp_threshold_5_unintended_diff_only": 0.002199995517730713,
128
- "tpp_threshold_10_total_metric": 0.02275000512599945,
129
- "tpp_threshold_10_intended_diff_only": 0.02580000162124634,
130
- "tpp_threshold_10_unintended_diff_only": 0.003049996495246887,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.015750020742416382,
190
  "tpp_threshold_10_total_metric": 0.1552499681711197,
191
  "tpp_threshold_10_intended_diff_only": 0.17100000381469727,
192
- "tpp_threshold_10_unintended_diff_only": 0.015750035643577576
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": 0.07550002634525299,
196
- "tpp_threshold_2_intended_diff_only": 0.09000003337860107,
197
- "tpp_threshold_2_unintended_diff_only": 0.014500007033348083,
198
- "tpp_threshold_5_total_metric": 0.1940000206232071,
199
  "tpp_threshold_5_intended_diff_only": 0.26500004529953003,
200
- "tpp_threshold_5_unintended_diff_only": 0.07100002467632294,
201
- "tpp_threshold_10_total_metric": 0.21450001001358032,
202
  "tpp_threshold_10_intended_diff_only": 0.3270000219345093,
203
- "tpp_threshold_10_unintended_diff_only": 0.11250001192092896
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.11775001883506775,
207
- "tpp_threshold_2_intended_diff_only": 0.12700003385543823,
208
- "tpp_threshold_2_unintended_diff_only": 0.009250015020370483,
209
- "tpp_threshold_5_total_metric": 0.21900004148483276,
210
- "tpp_threshold_5_intended_diff_only": 0.29600006341934204,
211
- "tpp_threshold_5_unintended_diff_only": 0.07700002193450928,
212
- "tpp_threshold_10_total_metric": 0.1625000387430191,
213
- "tpp_threshold_10_intended_diff_only": 0.34300005435943604,
214
- "tpp_threshold_10_unintended_diff_only": 0.18050001561641693
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.012000009417533875,
218
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
219
- "tpp_threshold_2_unintended_diff_only": -0.003000006079673767,
220
- "tpp_threshold_5_total_metric": 0.11325003206729889,
221
- "tpp_threshold_5_intended_diff_only": 0.12700003385543823,
222
- "tpp_threshold_5_unintended_diff_only": 0.013750001788139343,
223
- "tpp_threshold_10_total_metric": 0.2862500101327896,
224
  "tpp_threshold_10_intended_diff_only": 0.36000001430511475,
225
- "tpp_threshold_10_unintended_diff_only": 0.07375000417232513
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.0037499964237213135,
229
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
230
- "tpp_threshold_2_unintended_diff_only": 0.0012499988079071045,
231
- "tpp_threshold_5_total_metric": 0.14525002241134644,
232
- "tpp_threshold_5_intended_diff_only": 0.20500004291534424,
233
- "tpp_threshold_5_unintended_diff_only": 0.0597500205039978,
234
- "tpp_threshold_10_total_metric": 0.1757500320672989,
235
- "tpp_threshold_10_intended_diff_only": 0.24600005149841309,
236
- "tpp_threshold_10_unintended_diff_only": 0.0702500194311142
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.007500022649765015,
242
- "tpp_threshold_2_intended_diff_only": 0.003000020980834961,
243
- "tpp_threshold_2_unintended_diff_only": -0.004500001668930054,
244
- "tpp_threshold_5_total_metric": 0.011250004172325134,
245
- "tpp_threshold_5_intended_diff_only": 0.009000003337860107,
246
- "tpp_threshold_5_unintended_diff_only": -0.002250000834465027,
247
- "tpp_threshold_10_total_metric": -0.0007499903440475464,
248
  "tpp_threshold_10_intended_diff_only": 0.0040000081062316895,
249
- "tpp_threshold_10_unintended_diff_only": 0.004749998450279236
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": 0.0020000338554382324,
253
- "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
254
- "tpp_threshold_2_unintended_diff_only": 0.004999995231628418,
255
- "tpp_threshold_5_total_metric": 0.0032500624656677246,
256
- "tpp_threshold_5_intended_diff_only": 0.010000050067901611,
257
- "tpp_threshold_5_unintended_diff_only": 0.006749987602233887,
258
- "tpp_threshold_10_total_metric": 0.02000001072883606,
259
- "tpp_threshold_10_intended_diff_only": 0.023000001907348633,
260
- "tpp_threshold_10_unintended_diff_only": 0.0029999911785125732
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.014250010251998901,
264
- "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
265
- "tpp_threshold_2_unintended_diff_only": -0.0062499940395355225,
266
- "tpp_threshold_5_total_metric": 0.017750024795532227,
267
- "tpp_threshold_5_intended_diff_only": 0.017000019550323486,
268
- "tpp_threshold_5_unintended_diff_only": -0.0007500052452087402,
269
- "tpp_threshold_10_total_metric": 0.015000030398368835,
270
- "tpp_threshold_10_intended_diff_only": 0.017000019550323486,
271
- "tpp_threshold_10_unintended_diff_only": 0.001999989151954651
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.006750017404556274,
275
- "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
276
- "tpp_threshold_2_unintended_diff_only": 0.007750004529953003,
277
- "tpp_threshold_5_total_metric": 0.001999959349632263,
278
- "tpp_threshold_5_intended_diff_only": 0.0029999613761901855,
279
- "tpp_threshold_5_unintended_diff_only": 0.0010000020265579224,
280
- "tpp_threshold_10_total_metric": 0.007499977946281433,
281
- "tpp_threshold_10_intended_diff_only": 0.010999977588653564,
282
- "tpp_threshold_10_unintended_diff_only": 0.0034999996423721313
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.03474998474121094,
286
- "tpp_threshold_2_intended_diff_only": 0.03299999237060547,
287
- "tpp_threshold_2_unintended_diff_only": -0.0017499923706054688,
288
- "tpp_threshold_5_total_metric": 0.04074999690055847,
289
- "tpp_threshold_5_intended_diff_only": 0.046999990940093994,
290
- "tpp_threshold_5_unintended_diff_only": 0.0062499940395355225,
291
- "tpp_threshold_10_total_metric": 0.07199999690055847,
292
- "tpp_threshold_10_intended_diff_only": 0.07400000095367432,
293
- "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "8db2fbcf-b982-4d0b-99f8-66791dffce4d",
73
+ "datetime_epoch_millis": 1777316765632,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.033775000274181365,
77
+ "tpp_threshold_2_intended_diff_only": 0.03810000419616699,
78
+ "tpp_threshold_2_unintended_diff_only": 0.004325003921985626,
79
+ "tpp_threshold_5_total_metric": 0.0899750053882599,
80
+ "tpp_threshold_5_intended_diff_only": 0.11310001015663147,
81
+ "tpp_threshold_5_unintended_diff_only": 0.023125004768371583,
82
+ "tpp_threshold_10_total_metric": 0.11182500272989272,
83
+ "tpp_threshold_10_intended_diff_only": 0.15940001010894775,
84
+ "tpp_threshold_10_unintended_diff_only": 0.047575007379055026,
85
+ "tpp_threshold_20_total_metric": 0.1332250028848648,
86
+ "tpp_threshold_20_intended_diff_only": 0.20190001130104065,
87
+ "tpp_threshold_20_unintended_diff_only": 0.06867500841617584,
88
+ "tpp_threshold_50_total_metric": 0.16047500669956208,
89
+ "tpp_threshold_50_intended_diff_only": 0.2641000211238861,
90
+ "tpp_threshold_50_unintended_diff_only": 0.10362501442432404,
91
+ "tpp_threshold_100_total_metric": 0.18515002131462097,
92
+ "tpp_threshold_100_intended_diff_only": 0.32330002784729006,
93
+ "tpp_threshold_100_unintended_diff_only": 0.13815000653266907,
94
+ "tpp_threshold_500_total_metric": 0.2392500162124634,
95
+ "tpp_threshold_500_intended_diff_only": 0.4368000328540802,
96
+ "tpp_threshold_500_unintended_diff_only": 0.1975500166416168
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.0595999926328659,
103
+ "tpp_threshold_2_intended_diff_only": 0.06740000247955322,
104
+ "tpp_threshold_2_unintended_diff_only": 0.007800009846687317,
105
+ "tpp_threshold_5_total_metric": 0.16245001256465913,
106
+ "tpp_threshold_5_intended_diff_only": 0.20840002298355104,
107
+ "tpp_threshold_5_unintended_diff_only": 0.04595001041889191,
108
+ "tpp_threshold_10_total_metric": 0.19715000987052916,
109
  "tpp_threshold_10_intended_diff_only": 0.28940002918243407,
110
+ "tpp_threshold_10_unintended_diff_only": 0.09225001931190491,
111
+ "tpp_threshold_20_total_metric": 0.21865000426769257,
112
+ "tpp_threshold_20_intended_diff_only": 0.3510000228881836,
113
+ "tpp_threshold_20_unintended_diff_only": 0.13235001862049103,
114
+ "tpp_threshold_50_total_metric": 0.20735000669956208,
115
+ "tpp_threshold_50_intended_diff_only": 0.409600031375885,
116
+ "tpp_threshold_50_unintended_diff_only": 0.20225002467632294,
117
+ "tpp_threshold_100_total_metric": 0.1747500091791153,
118
+ "tpp_threshold_100_intended_diff_only": 0.4380000352859497,
119
+ "tpp_threshold_100_unintended_diff_only": 0.2632500261068344,
120
+ "tpp_threshold_500_total_metric": 0.09995001256465912,
121
+ "tpp_threshold_500_intended_diff_only": 0.46720004081726074,
122
+ "tpp_threshold_500_unintended_diff_only": 0.3672500282526016
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.007950007915496826,
127
+ "tpp_threshold_2_intended_diff_only": 0.008800005912780762,
128
+ "tpp_threshold_2_unintended_diff_only": 0.0008499979972839356,
129
+ "tpp_threshold_5_total_metric": 0.017499998211860657,
130
+ "tpp_threshold_5_intended_diff_only": 0.017799997329711915,
131
+ "tpp_threshold_5_unintended_diff_only": 0.0002999991178512573,
132
+ "tpp_threshold_10_total_metric": 0.026499995589256288,
133
+ "tpp_threshold_10_intended_diff_only": 0.029399991035461426,
134
+ "tpp_threshold_10_unintended_diff_only": 0.002899995446205139,
135
+ "tpp_threshold_20_total_metric": 0.047800001502037046,
136
+ "tpp_threshold_20_intended_diff_only": 0.052799999713897705,
137
+ "tpp_threshold_20_unintended_diff_only": 0.004999998211860657,
138
+ "tpp_threshold_50_total_metric": 0.11360000669956208,
139
+ "tpp_threshold_50_intended_diff_only": 0.11860001087188721,
140
+ "tpp_threshold_50_unintended_diff_only": 0.005000004172325134,
141
+ "tpp_threshold_100_total_metric": 0.19555003345012664,
142
+ "tpp_threshold_100_intended_diff_only": 0.20860002040863038,
143
+ "tpp_threshold_100_unintended_diff_only": 0.013049986958503724,
144
+ "tpp_threshold_500_total_metric": 0.37855001986026765,
145
+ "tpp_threshold_500_intended_diff_only": 0.40640002489089966,
146
+ "tpp_threshold_500_unintended_diff_only": 0.02785000503063202
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.015750020742416382,
194
  "tpp_threshold_10_total_metric": 0.1552499681711197,
195
  "tpp_threshold_10_intended_diff_only": 0.17100000381469727,
196
+ "tpp_threshold_10_unintended_diff_only": 0.015750035643577576,
197
+ "tpp_threshold_20_total_metric": 0.2319999784231186,
198
+ "tpp_threshold_20_intended_diff_only": 0.35600000619888306,
199
+ "tpp_threshold_20_unintended_diff_only": 0.12400002777576447,
200
+ "tpp_threshold_50_total_metric": 0.2149999737739563,
201
+ "tpp_threshold_50_intended_diff_only": 0.4169999957084656,
202
+ "tpp_threshold_50_unintended_diff_only": 0.20200002193450928,
203
+ "tpp_threshold_100_total_metric": 0.19324997067451477,
204
+ "tpp_threshold_100_intended_diff_only": 0.44099998474121094,
205
+ "tpp_threshold_100_unintended_diff_only": 0.24775001406669617,
206
+ "tpp_threshold_500_total_metric": 0.10175000131130219,
207
+ "tpp_threshold_500_intended_diff_only": 0.45600003004074097,
208
+ "tpp_threshold_500_unintended_diff_only": 0.3542500287294388
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.07924997806549072,
212
+ "tpp_threshold_2_intended_diff_only": 0.08899998664855957,
213
+ "tpp_threshold_2_unintended_diff_only": 0.009750008583068848,
214
+ "tpp_threshold_5_total_metric": 0.19575004279613495,
215
  "tpp_threshold_5_intended_diff_only": 0.26500004529953003,
216
+ "tpp_threshold_5_unintended_diff_only": 0.06925000250339508,
217
+ "tpp_threshold_10_total_metric": 0.20799998939037323,
218
  "tpp_threshold_10_intended_diff_only": 0.3270000219345093,
219
+ "tpp_threshold_10_unintended_diff_only": 0.11900003254413605,
220
+ "tpp_threshold_20_total_metric": 0.21424999833106995,
221
+ "tpp_threshold_20_intended_diff_only": 0.3410000205039978,
222
+ "tpp_threshold_20_unintended_diff_only": 0.12675002217292786,
223
+ "tpp_threshold_50_total_metric": 0.21249999105930328,
224
+ "tpp_threshold_50_intended_diff_only": 0.38700002431869507,
225
+ "tpp_threshold_50_unintended_diff_only": 0.17450003325939178,
226
+ "tpp_threshold_100_total_metric": 0.20249998569488525,
227
+ "tpp_threshold_100_intended_diff_only": 0.4280000329017639,
228
+ "tpp_threshold_100_unintended_diff_only": 0.22550004720687866,
229
+ "tpp_threshold_500_total_metric": 0.08500000834465027,
230
+ "tpp_threshold_500_intended_diff_only": 0.4590000510215759,
231
+ "tpp_threshold_500_unintended_diff_only": 0.37400004267692566
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.10625000298023224,
235
+ "tpp_threshold_2_intended_diff_only": 0.12400001287460327,
236
+ "tpp_threshold_2_unintended_diff_only": 0.017750009894371033,
237
+ "tpp_threshold_5_total_metric": 0.22300000488758087,
238
+ "tpp_threshold_5_intended_diff_only": 0.29500001668930054,
239
+ "tpp_threshold_5_unintended_diff_only": 0.07200001180171967,
240
+ "tpp_threshold_10_total_metric": 0.16225002706050873,
241
+ "tpp_threshold_10_intended_diff_only": 0.34400004148483276,
242
+ "tpp_threshold_10_unintended_diff_only": 0.18175001442432404,
243
+ "tpp_threshold_20_total_metric": 0.12725003063678741,
244
+ "tpp_threshold_20_intended_diff_only": 0.3670000433921814,
245
+ "tpp_threshold_20_unintended_diff_only": 0.23975001275539398,
246
+ "tpp_threshold_50_total_metric": 0.10850000381469727,
247
+ "tpp_threshold_50_intended_diff_only": 0.38200002908706665,
248
+ "tpp_threshold_50_unintended_diff_only": 0.2735000252723694,
249
+ "tpp_threshold_100_total_metric": 0.08225004374980927,
250
+ "tpp_threshold_100_intended_diff_only": 0.41200006008148193,
251
+ "tpp_threshold_100_unintended_diff_only": 0.32975001633167267,
252
+ "tpp_threshold_500_total_metric": 0.051750004291534424,
253
+ "tpp_threshold_500_intended_diff_only": 0.4520000219345093,
254
+ "tpp_threshold_500_unintended_diff_only": 0.40025001764297485
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.009749993681907654,
258
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0007499903440475464,
260
+ "tpp_threshold_5_total_metric": 0.11925001442432404,
261
+ "tpp_threshold_5_intended_diff_only": 0.12800002098083496,
262
+ "tpp_threshold_5_unintended_diff_only": 0.008750006556510925,
263
+ "tpp_threshold_10_total_metric": 0.2880000174045563,
264
  "tpp_threshold_10_intended_diff_only": 0.36000001430511475,
265
+ "tpp_threshold_10_unintended_diff_only": 0.07199999690055847,
266
+ "tpp_threshold_20_total_metric": 0.31299999356269836,
267
+ "tpp_threshold_20_intended_diff_only": 0.4150000214576721,
268
+ "tpp_threshold_20_unintended_diff_only": 0.10200002789497375,
269
+ "tpp_threshold_50_total_metric": 0.26225003600120544,
270
+ "tpp_threshold_50_intended_diff_only": 0.47700005769729614,
271
+ "tpp_threshold_50_unintended_diff_only": 0.2147500216960907,
272
+ "tpp_threshold_100_total_metric": 0.2082500159740448,
273
+ "tpp_threshold_100_intended_diff_only": 0.4830000400543213,
274
+ "tpp_threshold_100_unintended_diff_only": 0.2747500240802765,
275
+ "tpp_threshold_500_total_metric": 0.13300000131130219,
276
+ "tpp_threshold_500_intended_diff_only": 0.4930000305175781,
277
+ "tpp_threshold_500_unintended_diff_only": 0.36000002920627594
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.006250038743019104,
281
+ "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464,
283
+ "tpp_threshold_5_total_metric": 0.1420000195503235,
284
+ "tpp_threshold_5_intended_diff_only": 0.20600003004074097,
285
+ "tpp_threshold_5_unintended_diff_only": 0.06400001049041748,
286
+ "tpp_threshold_10_total_metric": 0.17225004732608795,
287
+ "tpp_threshold_10_intended_diff_only": 0.24500006437301636,
288
+ "tpp_threshold_10_unintended_diff_only": 0.0727500170469284,
289
+ "tpp_threshold_20_total_metric": 0.2067500203847885,
290
+ "tpp_threshold_20_intended_diff_only": 0.2760000228881836,
291
+ "tpp_threshold_20_unintended_diff_only": 0.06925000250339508,
292
+ "tpp_threshold_50_total_metric": 0.23850002884864807,
293
+ "tpp_threshold_50_intended_diff_only": 0.3850000500679016,
294
+ "tpp_threshold_50_unintended_diff_only": 0.14650002121925354,
295
+ "tpp_threshold_100_total_metric": 0.1875000298023224,
296
+ "tpp_threshold_100_intended_diff_only": 0.42600005865097046,
297
+ "tpp_threshold_100_unintended_diff_only": 0.23850002884864807,
298
+ "tpp_threshold_500_total_metric": 0.12825004756450653,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.3477500230073929
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.0012499988079071045,
306
+ "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
307
+ "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135,
308
+ "tpp_threshold_5_total_metric": 0.004249989986419678,
309
+ "tpp_threshold_5_intended_diff_only": 0.004999995231628418,
310
+ "tpp_threshold_5_unintended_diff_only": 0.0007500052452087402,
311
+ "tpp_threshold_10_total_metric": 0.0015000104904174805,
312
  "tpp_threshold_10_intended_diff_only": 0.0040000081062316895,
313
+ "tpp_threshold_10_unintended_diff_only": 0.002499997615814209,
314
+ "tpp_threshold_20_total_metric": 0.011500000953674316,
315
+ "tpp_threshold_20_intended_diff_only": 0.013999998569488525,
316
+ "tpp_threshold_20_unintended_diff_only": 0.002499997615814209,
317
+ "tpp_threshold_50_total_metric": 0.0442499965429306,
318
+ "tpp_threshold_50_intended_diff_only": 0.041999995708465576,
319
+ "tpp_threshold_50_unintended_diff_only": -0.002250000834465027,
320
+ "tpp_threshold_100_total_metric": 0.10675004124641418,
321
+ "tpp_threshold_100_intended_diff_only": 0.11800003051757812,
322
+ "tpp_threshold_100_unintended_diff_only": 0.01124998927116394,
323
+ "tpp_threshold_500_total_metric": 0.3985000252723694,
324
+ "tpp_threshold_500_intended_diff_only": 0.4140000343322754,
325
+ "tpp_threshold_500_unintended_diff_only": 0.015500009059906006
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.012000054121017456,
329
+ "tpp_threshold_2_intended_diff_only": 0.010000050067901611,
330
+ "tpp_threshold_2_unintended_diff_only": -0.0020000040531158447,
331
+ "tpp_threshold_5_total_metric": 0.007500007748603821,
332
+ "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
333
+ "tpp_threshold_5_unintended_diff_only": 0.006499990820884705,
334
+ "tpp_threshold_10_total_metric": 0.019500046968460083,
335
+ "tpp_threshold_10_intended_diff_only": 0.025000035762786865,
336
+ "tpp_threshold_10_unintended_diff_only": 0.005499988794326782,
337
+ "tpp_threshold_20_total_metric": 0.07250003516674042,
338
+ "tpp_threshold_20_intended_diff_only": 0.08100003004074097,
339
+ "tpp_threshold_20_unintended_diff_only": 0.00849999487400055,
340
+ "tpp_threshold_50_total_metric": 0.1405000537633896,
341
+ "tpp_threshold_50_intended_diff_only": 0.15300005674362183,
342
+ "tpp_threshold_50_unintended_diff_only": 0.012500002980232239,
343
+ "tpp_threshold_100_total_metric": 0.25400005280971527,
344
+ "tpp_threshold_100_intended_diff_only": 0.2710000276565552,
345
+ "tpp_threshold_100_unintended_diff_only": 0.016999974846839905,
346
+ "tpp_threshold_500_total_metric": 0.4137500524520874,
347
+ "tpp_threshold_500_intended_diff_only": 0.437000036239624,
348
+ "tpp_threshold_500_unintended_diff_only": 0.02324998378753662
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.005000010132789612,
352
+ "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
353
+ "tpp_threshold_2_unintended_diff_only": -0.0010000020265579224,
354
+ "tpp_threshold_5_total_metric": 0.011750012636184692,
355
+ "tpp_threshold_5_intended_diff_only": 0.008000016212463379,
356
+ "tpp_threshold_5_unintended_diff_only": -0.0037499964237213135,
357
+ "tpp_threshold_10_total_metric": 0.01824997365474701,
358
+ "tpp_threshold_10_intended_diff_only": 0.015999972820281982,
359
+ "tpp_threshold_10_unintended_diff_only": -0.002250000834465027,
360
+ "tpp_threshold_20_total_metric": 0.008499979972839355,
361
+ "tpp_threshold_20_intended_diff_only": 0.010999977588653564,
362
+ "tpp_threshold_20_unintended_diff_only": 0.002499997615814209,
363
+ "tpp_threshold_50_total_metric": 0.051249995827674866,
364
+ "tpp_threshold_50_intended_diff_only": 0.050999999046325684,
365
+ "tpp_threshold_50_unintended_diff_only": -0.00024999678134918213,
366
+ "tpp_threshold_100_total_metric": 0.11775004863739014,
367
+ "tpp_threshold_100_intended_diff_only": 0.12700003385543823,
368
+ "tpp_threshold_100_unintended_diff_only": 0.009249985218048096,
369
+ "tpp_threshold_500_total_metric": 0.359000027179718,
370
+ "tpp_threshold_500_intended_diff_only": 0.39100003242492676,
371
+ "tpp_threshold_500_unintended_diff_only": 0.03200000524520874
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.001499965786933899,
375
+ "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
376
+ "tpp_threshold_2_unintended_diff_only": 0.0005000084638595581,
377
+ "tpp_threshold_5_total_metric": 0.019999995827674866,
378
+ "tpp_threshold_5_intended_diff_only": 0.018999993801116943,
379
+ "tpp_threshold_5_unintended_diff_only": -0.0010000020265579224,
380
+ "tpp_threshold_10_total_metric": 0.006749972701072693,
381
+ "tpp_threshold_10_intended_diff_only": 0.011999964714050293,
382
+ "tpp_threshold_10_unintended_diff_only": 0.0052499920129776,
383
+ "tpp_threshold_20_total_metric": 0.034999996423721313,
384
+ "tpp_threshold_20_intended_diff_only": 0.03700000047683716,
385
+ "tpp_threshold_20_unintended_diff_only": 0.0020000040531158447,
386
+ "tpp_threshold_50_total_metric": 0.12074998021125793,
387
+ "tpp_threshold_50_intended_diff_only": 0.12599998712539673,
388
+ "tpp_threshold_50_unintended_diff_only": 0.005250006914138794,
389
+ "tpp_threshold_100_total_metric": 0.21724998950958252,
390
+ "tpp_threshold_100_intended_diff_only": 0.23199999332427979,
391
+ "tpp_threshold_100_unintended_diff_only": 0.014750003814697266,
392
+ "tpp_threshold_500_total_metric": 0.37574999034404755,
393
+ "tpp_threshold_500_intended_diff_only": 0.4169999957084656,
394
+ "tpp_threshold_500_unintended_diff_only": 0.04125000536441803
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.02000001072883606,
398
+ "tpp_threshold_2_intended_diff_only": 0.023000001907348633,
399
+ "tpp_threshold_2_unintended_diff_only": 0.0029999911785125732,
400
+ "tpp_threshold_5_total_metric": 0.04399998486042023,
401
+ "tpp_threshold_5_intended_diff_only": 0.042999982833862305,
402
+ "tpp_threshold_5_unintended_diff_only": -0.0010000020265579224,
403
+ "tpp_threshold_10_total_metric": 0.08649997413158417,
404
+ "tpp_threshold_10_intended_diff_only": 0.0899999737739563,
405
+ "tpp_threshold_10_unintended_diff_only": 0.0034999996423721313,
406
+ "tpp_threshold_20_total_metric": 0.11149999499320984,
407
+ "tpp_threshold_20_intended_diff_only": 0.12099999189376831,
408
+ "tpp_threshold_20_unintended_diff_only": 0.009499996900558472,
409
+ "tpp_threshold_50_total_metric": 0.21125000715255737,
410
+ "tpp_threshold_50_intended_diff_only": 0.22100001573562622,
411
+ "tpp_threshold_50_unintended_diff_only": 0.009750008583068848,
412
+ "tpp_threshold_100_total_metric": 0.28200003504753113,
413
+ "tpp_threshold_100_intended_diff_only": 0.29500001668930054,
414
+ "tpp_threshold_100_unintended_diff_only": 0.01299998164176941,
415
+ "tpp_threshold_500_total_metric": 0.34575000405311584,
416
+ "tpp_threshold_500_intended_diff_only": 0.37300002574920654,
417
+ "tpp_threshold_500_unintended_diff_only": 0.027250021696090698
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-219726-tokens-900000000/scr/saebench_scr_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,8 +69,8 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "da2a6887-913b-4c10-8426-ea8bfa58f10f",
69
- "datetime_epoch_millis": 1776884712767,
70
  "eval_result_metrics": {
71
  "scr_metrics": {
72
  "scr_dir1_threshold_2": 0.2896188146090331,
@@ -78,18 +82,18 @@
78
  "scr_dir1_threshold_10": 0.2889266157448796,
79
  "scr_metric_threshold_10": 0.23494818923208868,
80
  "scr_dir2_threshold_10": 0.22345971109744955,
81
- "scr_dir1_threshold_20": null,
82
- "scr_metric_threshold_20": null,
83
- "scr_dir2_threshold_20": null,
84
- "scr_dir1_threshold_50": null,
85
- "scr_metric_threshold_50": null,
86
- "scr_dir2_threshold_50": null,
87
- "scr_dir1_threshold_100": null,
88
- "scr_metric_threshold_100": null,
89
- "scr_dir2_threshold_100": null,
90
- "scr_dir1_threshold_500": null,
91
- "scr_metric_threshold_500": null,
92
- "scr_dir2_threshold_500": null
93
  }
94
  },
95
  "eval_result_details": [
@@ -104,18 +108,18 @@
104
  "scr_dir1_threshold_10": 0.5256410746257707,
105
  "scr_metric_threshold_10": 0.05181341709689583,
106
  "scr_dir2_threshold_10": 0.05181341709689583,
107
- "scr_dir1_threshold_20": null,
108
- "scr_metric_threshold_20": null,
109
- "scr_dir2_threshold_20": null,
110
- "scr_dir1_threshold_50": null,
111
- "scr_metric_threshold_50": null,
112
- "scr_dir2_threshold_50": null,
113
- "scr_dir1_threshold_100": null,
114
- "scr_metric_threshold_100": null,
115
- "scr_dir2_threshold_100": null,
116
- "scr_dir1_threshold_500": null,
117
- "scr_metric_threshold_500": null,
118
- "scr_dir2_threshold_500": null
119
  },
120
  {
121
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
@@ -128,18 +132,18 @@
128
  "scr_dir1_threshold_10": 0.3560604589646946,
129
  "scr_metric_threshold_10": 0.3455658339637086,
130
  "scr_dir2_threshold_10": 0.3455658339637086,
131
- "scr_dir1_threshold_20": null,
132
- "scr_metric_threshold_20": null,
133
- "scr_dir2_threshold_20": null,
134
- "scr_dir1_threshold_50": null,
135
- "scr_metric_threshold_50": null,
136
- "scr_dir2_threshold_50": null,
137
- "scr_dir1_threshold_100": null,
138
- "scr_metric_threshold_100": null,
139
- "scr_dir2_threshold_100": null,
140
- "scr_dir1_threshold_500": null,
141
- "scr_metric_threshold_500": null,
142
- "scr_dir2_threshold_500": null
143
  },
144
  {
145
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
@@ -152,18 +156,18 @@
152
  "scr_dir1_threshold_10": 0.45312467985791743,
153
  "scr_metric_threshold_10": 0.13819095138732784,
154
  "scr_dir2_threshold_10": 0.13819095138732784,
155
- "scr_dir1_threshold_20": null,
156
- "scr_metric_threshold_20": null,
157
- "scr_dir2_threshold_20": null,
158
- "scr_dir1_threshold_50": null,
159
- "scr_metric_threshold_50": null,
160
- "scr_dir2_threshold_50": null,
161
- "scr_dir1_threshold_100": null,
162
- "scr_metric_threshold_100": null,
163
- "scr_dir2_threshold_100": null,
164
- "scr_dir1_threshold_500": null,
165
- "scr_metric_threshold_500": null,
166
- "scr_dir2_threshold_500": null
167
  },
168
  {
169
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
@@ -176,18 +180,18 @@
176
  "scr_dir1_threshold_10": -0.134228469820646,
177
  "scr_metric_threshold_10": 0.09872616905774931,
178
  "scr_dir2_threshold_10": 0.09872616905774931,
179
- "scr_dir1_threshold_20": null,
180
- "scr_metric_threshold_20": null,
181
- "scr_dir2_threshold_20": null,
182
- "scr_dir1_threshold_50": null,
183
- "scr_metric_threshold_50": null,
184
- "scr_dir2_threshold_50": null,
185
- "scr_dir1_threshold_100": null,
186
- "scr_metric_threshold_100": null,
187
- "scr_dir2_threshold_100": null,
188
- "scr_dir1_threshold_500": null,
189
- "scr_metric_threshold_500": null,
190
- "scr_dir2_threshold_500": null
191
  },
192
  {
193
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
@@ -200,18 +204,18 @@
200
  "scr_dir1_threshold_10": 0.06842111372221286,
201
  "scr_metric_threshold_10": 0.515872927644938,
202
  "scr_dir2_threshold_10": 0.515872927644938,
203
- "scr_dir1_threshold_20": null,
204
- "scr_metric_threshold_20": null,
205
- "scr_dir2_threshold_20": null,
206
- "scr_dir1_threshold_50": null,
207
- "scr_metric_threshold_50": null,
208
- "scr_dir2_threshold_50": null,
209
- "scr_dir1_threshold_100": null,
210
- "scr_metric_threshold_100": null,
211
- "scr_dir2_threshold_100": null,
212
- "scr_dir1_threshold_500": null,
213
- "scr_metric_threshold_500": null,
214
- "scr_dir2_threshold_500": null
215
  },
216
  {
217
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
@@ -224,18 +228,18 @@
224
  "scr_dir1_threshold_10": 0.4098360709132541,
225
  "scr_metric_threshold_10": 0.17175577294818423,
226
  "scr_dir2_threshold_10": 0.17175577294818423,
227
- "scr_dir1_threshold_20": null,
228
- "scr_metric_threshold_20": null,
229
- "scr_dir2_threshold_20": null,
230
- "scr_dir1_threshold_50": null,
231
- "scr_metric_threshold_50": null,
232
- "scr_dir2_threshold_50": null,
233
- "scr_dir1_threshold_100": null,
234
- "scr_metric_threshold_100": null,
235
- "scr_dir2_threshold_100": null,
236
- "scr_dir1_threshold_500": null,
237
- "scr_metric_threshold_500": null,
238
- "scr_dir2_threshold_500": null
239
  },
240
  {
241
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
@@ -248,18 +252,18 @@
248
  "scr_dir1_threshold_10": 0.4139533620063974,
249
  "scr_metric_threshold_10": 0.33905580606846963,
250
  "scr_dir2_threshold_10": 0.33905580606846963,
251
- "scr_dir1_threshold_20": null,
252
- "scr_metric_threshold_20": null,
253
- "scr_dir2_threshold_20": null,
254
- "scr_dir1_threshold_50": null,
255
- "scr_metric_threshold_50": null,
256
- "scr_dir2_threshold_50": null,
257
- "scr_dir1_threshold_100": null,
258
- "scr_metric_threshold_100": null,
259
- "scr_dir2_threshold_100": null,
260
- "scr_dir1_threshold_500": null,
261
- "scr_metric_threshold_500": null,
262
- "scr_dir2_threshold_500": null
263
  },
264
  {
265
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
@@ -272,18 +276,18 @@
272
  "scr_dir1_threshold_10": 0.21860463568943592,
273
  "scr_metric_threshold_10": 0.21860463568943592,
274
  "scr_dir2_threshold_10": 0.12669681061232305,
275
- "scr_dir1_threshold_20": null,
276
- "scr_metric_threshold_20": null,
277
- "scr_dir2_threshold_20": null,
278
- "scr_dir1_threshold_50": null,
279
- "scr_metric_threshold_50": null,
280
- "scr_dir2_threshold_50": null,
281
- "scr_dir1_threshold_100": null,
282
- "scr_metric_threshold_100": null,
283
- "scr_dir2_threshold_100": null,
284
- "scr_dir1_threshold_500": null,
285
- "scr_metric_threshold_500": null,
286
- "scr_dir2_threshold_500": null
287
  }
288
  ],
289
  "sae_bench_commit_hash": "Unknown",
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "d6582b6e-d498-4999-9ec3-a0af5792c811",
73
+ "datetime_epoch_millis": 1777316550644,
74
  "eval_result_metrics": {
75
  "scr_metrics": {
76
  "scr_dir1_threshold_2": 0.2896188146090331,
 
82
  "scr_dir1_threshold_10": 0.2889266157448796,
83
  "scr_metric_threshold_10": 0.23494818923208868,
84
  "scr_dir2_threshold_10": 0.22345971109744955,
85
+ "scr_dir1_threshold_20": 0.21992587333994348,
86
+ "scr_metric_threshold_20": 0.22353422720014693,
87
+ "scr_dir2_threshold_20": 0.21245350675047425,
88
+ "scr_dir1_threshold_50": 0.08432192518155021,
89
+ "scr_metric_threshold_50": 0.33824373291787563,
90
+ "scr_dir2_threshold_50": 0.32354837314889334,
91
+ "scr_dir1_threshold_100": -0.2103083440146591,
92
+ "scr_metric_threshold_100": 0.33500288328703,
93
+ "scr_dir2_threshold_100": 0.31697700710952653,
94
+ "scr_dir1_threshold_500": -0.9815309813458944,
95
+ "scr_metric_threshold_500": 0.1855971065432701,
96
+ "scr_dir2_threshold_500": 0.17933854684292247
97
  }
98
  },
99
  "eval_result_details": [
 
108
  "scr_dir1_threshold_10": 0.5256410746257707,
109
  "scr_metric_threshold_10": 0.05181341709689583,
110
  "scr_dir2_threshold_10": 0.05181341709689583,
111
+ "scr_dir1_threshold_20": 0.37179539103317,
112
+ "scr_metric_threshold_20": 0.10880826855317922,
113
+ "scr_dir2_threshold_20": 0.10880826855317922,
114
+ "scr_dir1_threshold_50": 0.5384614208981502,
115
+ "scr_metric_threshold_50": 0.26683931423164214,
116
+ "scr_dir2_threshold_50": 0.26683931423164214,
117
+ "scr_dir1_threshold_100": 0.3333335880540079,
118
+ "scr_metric_threshold_100": 0.30569953147047735,
119
+ "scr_dir2_threshold_100": 0.30569953147047735,
120
+ "scr_dir1_threshold_500": -2.7435892537422926,
121
+ "scr_metric_threshold_500": 0.07772012564534375,
122
+ "scr_dir2_threshold_500": 0.07772012564534375
123
  },
124
  {
125
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
 
132
  "scr_dir1_threshold_10": 0.3560604589646946,
133
  "scr_metric_threshold_10": 0.3455658339637086,
134
  "scr_dir2_threshold_10": 0.3455658339637086,
135
+ "scr_dir1_threshold_20": 0.48484845748180366,
136
+ "scr_metric_threshold_20": 0.08256883242346637,
137
+ "scr_dir2_threshold_20": 0.08256883242346637,
138
+ "scr_dir1_threshold_50": 0.5681820542194435,
139
+ "scr_metric_threshold_50": 0.45871567492686127,
140
+ "scr_dir2_threshold_50": 0.45871567492686127,
141
+ "scr_dir1_threshold_100": -0.5075754325964184,
142
+ "scr_metric_threshold_100": 0.608562653226762,
143
+ "scr_dir2_threshold_100": 0.608562653226762,
144
+ "scr_dir1_threshold_500": -2.113635778673553,
145
+ "scr_metric_threshold_500": 0.04281344841099861,
146
+ "scr_dir2_threshold_500": 0.04281344841099861
147
  },
148
  {
149
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
 
156
  "scr_dir1_threshold_10": 0.45312467985791743,
157
  "scr_metric_threshold_10": 0.13819095138732784,
158
  "scr_dir2_threshold_10": 0.13819095138732784,
159
+ "scr_dir1_threshold_20": 0.3437501746229541,
160
+ "scr_metric_threshold_20": 0.1809045617464338,
161
+ "scr_dir2_threshold_20": 0.1809045617464338,
162
+ "scr_dir1_threshold_50": -0.8437501746229541,
163
+ "scr_metric_threshold_50": 0.24623124232384397,
164
+ "scr_dir2_threshold_50": 0.24623124232384397,
165
+ "scr_dir1_threshold_100": -1.1874994179234863,
166
+ "scr_metric_threshold_100": 0.13567842130982535,
167
+ "scr_dir2_threshold_100": 0.13567842130982535,
168
+ "scr_dir1_threshold_500": -2.6406250291038256,
169
+ "scr_metric_threshold_500": -0.017587860302908362,
170
+ "scr_dir2_threshold_500": -0.017587860302908362
171
  },
172
  {
173
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
 
180
  "scr_dir1_threshold_10": -0.134228469820646,
181
  "scr_metric_threshold_10": 0.09872616905774931,
182
  "scr_dir2_threshold_10": 0.09872616905774931,
183
+ "scr_dir1_threshold_20": -0.6711411490096227,
184
+ "scr_metric_threshold_20": -0.025477757787232685,
185
+ "scr_dir2_threshold_20": -0.025477757787232685,
186
+ "scr_dir1_threshold_50": -0.7181212134546494,
187
+ "scr_metric_threshold_50": 0.13694280573859832,
188
+ "scr_dir2_threshold_50": 0.13694280573859832,
189
+ "scr_dir1_threshold_100": -1.4630877208422257,
190
+ "scr_metric_threshold_100": -0.11464972021884395,
191
+ "scr_dir2_threshold_100": -0.11464972021884395,
192
+ "scr_dir1_threshold_500": -1.5906044671484376,
193
+ "scr_metric_threshold_500": -0.5509553257507622,
194
+ "scr_dir2_threshold_500": -0.5509553257507622
195
  },
196
  {
197
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
 
204
  "scr_dir1_threshold_10": 0.06842111372221286,
205
  "scr_metric_threshold_10": 0.515872927644938,
206
  "scr_dir2_threshold_10": 0.515872927644938,
207
+ "scr_dir1_threshold_20": 0.09473687842942387,
208
+ "scr_metric_threshold_20": 0.5753969089319204,
209
+ "scr_dir2_threshold_20": 0.5753969089319204,
210
+ "scr_dir1_threshold_50": -0.12631573333634494,
211
+ "scr_metric_threshold_50": 0.6865079937623274,
212
+ "scr_dir2_threshold_50": 0.6865079937623274,
213
+ "scr_dir1_threshold_100": -0.184210352950477,
214
+ "scr_metric_threshold_100": 0.714285646706761,
215
+ "scr_dir2_threshold_100": 0.714285646706761,
216
+ "scr_dir1_threshold_500": -0.16315799215163673,
217
+ "scr_metric_threshold_500": 0.801587280938177,
218
+ "scr_dir2_threshold_500": 0.801587280938177
219
  },
220
  {
221
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
 
228
  "scr_dir1_threshold_10": 0.4098360709132541,
229
  "scr_metric_threshold_10": 0.17175577294818423,
230
  "scr_dir2_threshold_10": 0.17175577294818423,
231
+ "scr_dir1_threshold_20": 0.4098360709132541,
232
+ "scr_metric_threshold_20": 0.2061069275378211,
233
+ "scr_dir2_threshold_20": 0.2061069275378211,
234
+ "scr_dir1_threshold_50": 0.377049276438573,
235
+ "scr_metric_threshold_50": 0.2671755204201627,
236
+ "scr_dir2_threshold_50": 0.2671755204201627,
237
+ "scr_dir1_threshold_100": 0.4754099855711136,
238
+ "scr_metric_threshold_100": 0.36259549539076413,
239
+ "scr_dir2_threshold_100": 0.36259549539076413,
240
+ "scr_dir1_threshold_500": 0.4644808369405549,
241
+ "scr_metric_threshold_500": 0.36641223978991866,
242
+ "scr_dir2_threshold_500": 0.36641223978991866
243
  },
244
  {
245
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
 
252
  "scr_dir1_threshold_10": 0.4139533620063974,
253
  "scr_metric_threshold_10": 0.33905580606846963,
254
  "scr_dir2_threshold_10": 0.33905580606846963,
255
+ "scr_dir1_threshold_20": 0.46046493833553914,
256
+ "scr_metric_threshold_20": 0.3948498512825616,
257
+ "scr_dir2_threshold_20": 0.3948498512825616,
258
+ "scr_dir1_threshold_50": 0.548837266037944,
259
+ "scr_metric_threshold_50": 0.31330480666733007,
260
+ "scr_dir2_threshold_50": 0.31330480666733007,
261
+ "scr_dir1_threshold_100": 0.5395347844335979,
262
+ "scr_metric_threshold_100": 0.3562232242738785,
263
+ "scr_dir2_threshold_100": 0.3562232242738785,
264
+ "scr_dir1_threshold_500": 0.5906977401803443,
265
+ "scr_metric_threshold_500": 0.4206008506837011,
266
+ "scr_dir2_threshold_500": 0.4206008506837011
267
  },
268
  {
269
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
 
276
  "scr_dir1_threshold_10": 0.21860463568943592,
277
  "scr_metric_threshold_10": 0.21860463568943592,
278
  "scr_dir2_threshold_10": 0.12669681061232305,
279
+ "scr_dir1_threshold_20": 0.2651162249130257,
280
+ "scr_metric_threshold_20": 0.2651162249130257,
281
+ "scr_dir2_threshold_20": 0.17647046131564434,
282
+ "scr_dir1_threshold_50": 0.3302325052722394,
283
+ "scr_metric_threshold_50": 0.3302325052722394,
284
+ "scr_dir2_threshold_50": 0.21266962712038115,
285
+ "scr_dir1_threshold_100": 0.3116278141366155,
286
+ "scr_metric_threshold_100": 0.3116278141366155,
287
+ "scr_dir2_threshold_100": 0.16742080471658802,
288
+ "scr_dir1_threshold_500": 0.3441860929316922,
289
+ "scr_metric_threshold_500": 0.3441860929316922,
290
+ "scr_dir2_threshold_500": 0.29411761532891106
291
  }
292
  ],
293
  "sae_bench_commit_hash": "Unknown",
gemma-2-2b-btk/k-100/seed-0/snapshots/step-219726-tokens-900000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "892cbbec-a6ec-499c-b18e-a7866ca2fb4d",
69
- "datetime_epoch_millis": 1776884868502,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.037274999916553496,
73
- "tpp_threshold_2_intended_diff_only": 0.04110000133514404,
74
- "tpp_threshold_2_unintended_diff_only": 0.0038250014185905457,
75
- "tpp_threshold_5_total_metric": 0.09112499803304672,
76
- "tpp_threshold_5_intended_diff_only": 0.11650000214576721,
77
- "tpp_threshold_5_unintended_diff_only": 0.02537500411272049,
78
- "tpp_threshold_10_total_metric": 0.11410000622272491,
79
- "tpp_threshold_10_intended_diff_only": 0.16100001335144043,
80
- "tpp_threshold_10_unintended_diff_only": 0.04690000712871552,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.06464999914169312,
99
- "tpp_threshold_2_intended_diff_only": 0.0718000054359436,
100
- "tpp_threshold_2_unintended_diff_only": 0.007150006294250488,
101
- "tpp_threshold_5_total_metric": 0.1643500119447708,
102
  "tpp_threshold_5_intended_diff_only": 0.2128000259399414,
103
- "tpp_threshold_5_unintended_diff_only": 0.04845001399517059,
104
- "tpp_threshold_10_total_metric": 0.2018500030040741,
105
  "tpp_threshold_10_intended_diff_only": 0.29300001859664915,
106
- "tpp_threshold_10_unintended_diff_only": 0.09115001559257507,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.00990000069141388,
123
- "tpp_threshold_2_intended_diff_only": 0.010399997234344482,
124
- "tpp_threshold_2_unintended_diff_only": 0.000499996542930603,
125
- "tpp_threshold_5_total_metric": 0.017899984121322633,
126
- "tpp_threshold_5_intended_diff_only": 0.02019997835159302,
127
- "tpp_threshold_5_unintended_diff_only": 0.0022999942302703857,
128
- "tpp_threshold_10_total_metric": 0.02635000944137573,
129
- "tpp_threshold_10_intended_diff_only": 0.02900000810623169,
130
- "tpp_threshold_10_unintended_diff_only": 0.002649998664855957,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.0155000239610672,
190
  "tpp_threshold_10_total_metric": 0.1409999579191208,
191
  "tpp_threshold_10_intended_diff_only": 0.15799999237060547,
192
- "tpp_threshold_10_unintended_diff_only": 0.01700003445148468
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": 0.08574998378753662,
196
- "tpp_threshold_2_intended_diff_only": 0.10199999809265137,
197
- "tpp_threshold_2_unintended_diff_only": 0.016250014305114746,
198
- "tpp_threshold_5_total_metric": 0.19975002110004425,
199
  "tpp_threshold_5_intended_diff_only": 0.27500003576278687,
200
- "tpp_threshold_5_unintended_diff_only": 0.07525001466274261,
201
- "tpp_threshold_10_total_metric": 0.22324998676776886,
202
  "tpp_threshold_10_intended_diff_only": 0.3330000042915344,
203
- "tpp_threshold_10_unintended_diff_only": 0.10975001752376556
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.11775003373622894,
207
- "tpp_threshold_2_intended_diff_only": 0.12700003385543823,
208
- "tpp_threshold_2_unintended_diff_only": 0.00925000011920929,
209
- "tpp_threshold_5_total_metric": 0.22825002670288086,
210
- "tpp_threshold_5_intended_diff_only": 0.2980000376701355,
211
- "tpp_threshold_5_unintended_diff_only": 0.06975001096725464,
212
- "tpp_threshold_10_total_metric": 0.18250004947185516,
213
- "tpp_threshold_10_intended_diff_only": 0.34800004959106445,
214
- "tpp_threshold_10_unintended_diff_only": 0.1655000001192093
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.011750012636184692,
218
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
219
- "tpp_threshold_2_unintended_diff_only": -0.002750009298324585,
220
- "tpp_threshold_5_total_metric": 0.24400001764297485,
221
- "tpp_threshold_5_intended_diff_only": 0.31300002336502075,
222
- "tpp_threshold_5_unintended_diff_only": 0.0690000057220459,
223
- "tpp_threshold_10_total_metric": 0.27275000512599945,
224
  "tpp_threshold_10_intended_diff_only": 0.36500000953674316,
225
- "tpp_threshold_10_unintended_diff_only": 0.09225000441074371
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.003999993205070496,
229
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
230
- "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224,
231
- "tpp_threshold_5_total_metric": 0.025250032544136047,
232
  "tpp_threshold_5_intended_diff_only": 0.03800004720687866,
233
- "tpp_threshold_5_unintended_diff_only": 0.012750014662742615,
234
- "tpp_threshold_10_total_metric": 0.18975001573562622,
235
- "tpp_threshold_10_intended_diff_only": 0.26100003719329834,
236
- "tpp_threshold_10_unintended_diff_only": 0.07125002145767212
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.008750006556510925,
242
- "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
243
- "tpp_threshold_2_unintended_diff_only": -0.004749998450279236,
244
- "tpp_threshold_5_total_metric": 0.009499981999397278,
245
- "tpp_threshold_5_intended_diff_only": 0.006999969482421875,
246
- "tpp_threshold_5_unintended_diff_only": -0.002500012516975403,
247
- "tpp_threshold_10_total_metric": -0.0007499903440475464,
248
- "tpp_threshold_10_intended_diff_only": 0.003000020980834961,
249
- "tpp_threshold_10_unintended_diff_only": 0.0037500113248825073
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": 0.0052500516176223755,
253
- "tpp_threshold_2_intended_diff_only": 0.010000050067901611,
254
- "tpp_threshold_2_unintended_diff_only": 0.004749998450279236,
255
- "tpp_threshold_5_total_metric": 0.011250019073486328,
256
- "tpp_threshold_5_intended_diff_only": 0.018000006675720215,
257
- "tpp_threshold_5_unintended_diff_only": 0.006749987602233887,
258
- "tpp_threshold_10_total_metric": 0.027250051498413086,
259
- "tpp_threshold_10_intended_diff_only": 0.029000043869018555,
260
- "tpp_threshold_10_unintended_diff_only": 0.0017499923706054688
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.011999979615211487,
264
- "tpp_threshold_2_intended_diff_only": 0.0059999823570251465,
265
- "tpp_threshold_2_unintended_diff_only": -0.00599999725818634,
266
- "tpp_threshold_5_total_metric": 0.016749992966651917,
267
- "tpp_threshold_5_intended_diff_only": 0.014999985694885254,
268
- "tpp_threshold_5_unintended_diff_only": -0.0017500072717666626,
269
- "tpp_threshold_10_total_metric": 0.019250035285949707,
270
- "tpp_threshold_10_intended_diff_only": 0.021000027656555176,
271
- "tpp_threshold_10_unintended_diff_only": 0.0017499923706054688
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.007249996066093445,
275
- "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
276
- "tpp_threshold_2_unintended_diff_only": 0.008249983191490173,
277
- "tpp_threshold_5_total_metric": 0.007249951362609863,
278
- "tpp_threshold_5_intended_diff_only": 0.008999943733215332,
279
- "tpp_threshold_5_unintended_diff_only": 0.0017499923706054688,
280
- "tpp_threshold_10_total_metric": 0.014499947428703308,
281
- "tpp_threshold_10_intended_diff_only": 0.01799994707107544,
282
- "tpp_threshold_10_unintended_diff_only": 0.0034999996423721313
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.030749961733818054,
286
- "tpp_threshold_2_intended_diff_only": 0.030999958515167236,
287
- "tpp_threshold_2_unintended_diff_only": 0.00024999678134918213,
288
- "tpp_threshold_5_total_metric": 0.04474997520446777,
289
- "tpp_threshold_5_intended_diff_only": 0.05199998617172241,
290
- "tpp_threshold_5_unintended_diff_only": 0.007250010967254639,
291
- "tpp_threshold_10_total_metric": 0.07150000333786011,
292
- "tpp_threshold_10_intended_diff_only": 0.07400000095367432,
293
- "tpp_threshold_10_unintended_diff_only": 0.002499997615814209
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "e239af40-39b3-4458-84cb-08e5f69c02af",
73
+ "datetime_epoch_millis": 1777316823043,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.035974989831447604,
77
+ "tpp_threshold_2_intended_diff_only": 0.04059999585151672,
78
+ "tpp_threshold_2_unintended_diff_only": 0.004625006020069122,
79
+ "tpp_threshold_5_total_metric": 0.09110001027584076,
80
+ "tpp_threshold_5_intended_diff_only": 0.11510001420974732,
81
+ "tpp_threshold_5_unintended_diff_only": 0.024000003933906555,
82
+ "tpp_threshold_10_total_metric": 0.11435000449419021,
83
+ "tpp_threshold_10_intended_diff_only": 0.16220000982284544,
84
+ "tpp_threshold_10_unintended_diff_only": 0.04785000532865524,
85
+ "tpp_threshold_20_total_metric": 0.13832499384880065,
86
+ "tpp_threshold_20_intended_diff_only": 0.20879999995231627,
87
+ "tpp_threshold_20_unintended_diff_only": 0.07047500610351562,
88
+ "tpp_threshold_50_total_metric": 0.16092500686645508,
89
+ "tpp_threshold_50_intended_diff_only": 0.2679000198841095,
90
+ "tpp_threshold_50_unintended_diff_only": 0.10697501301765441,
91
+ "tpp_threshold_100_total_metric": 0.19520001709461213,
92
+ "tpp_threshold_100_intended_diff_only": 0.331900030374527,
93
+ "tpp_threshold_100_unintended_diff_only": 0.13670001327991485,
94
+ "tpp_threshold_500_total_metric": 0.2363000199198723,
95
+ "tpp_threshold_500_intended_diff_only": 0.4355000376701355,
96
+ "tpp_threshold_500_unintended_diff_only": 0.19920001775026322
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.0632999986410141,
103
+ "tpp_threshold_2_intended_diff_only": 0.0714000105857849,
104
+ "tpp_threshold_2_unintended_diff_only": 0.008100011944770813,
105
+ "tpp_threshold_5_total_metric": 0.16535001397132873,
106
  "tpp_threshold_5_intended_diff_only": 0.2128000259399414,
107
+ "tpp_threshold_5_unintended_diff_only": 0.04745001196861267,
108
+ "tpp_threshold_10_total_metric": 0.1999499976634979,
109
  "tpp_threshold_10_intended_diff_only": 0.29300001859664915,
110
+ "tpp_threshold_10_unintended_diff_only": 0.09305002093315125,
111
+ "tpp_threshold_20_total_metric": 0.223799991607666,
112
+ "tpp_threshold_20_intended_diff_only": 0.35940001010894773,
113
+ "tpp_threshold_20_unintended_diff_only": 0.13560001850128173,
114
+ "tpp_threshold_50_total_metric": 0.2084500163793564,
115
+ "tpp_threshold_50_intended_diff_only": 0.41620004177093506,
116
+ "tpp_threshold_50_unintended_diff_only": 0.20775002539157866,
117
+ "tpp_threshold_100_total_metric": 0.18575001358985901,
118
+ "tpp_threshold_100_intended_diff_only": 0.44400004148483274,
119
+ "tpp_threshold_100_unintended_diff_only": 0.25825002789497375,
120
+ "tpp_threshold_500_total_metric": 0.10000000596046447,
121
+ "tpp_threshold_500_intended_diff_only": 0.46720004081726074,
122
+ "tpp_threshold_500_unintended_diff_only": 0.36720003485679625
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.008649981021881104,
127
+ "tpp_threshold_2_intended_diff_only": 0.009799981117248535,
128
+ "tpp_threshold_2_unintended_diff_only": 0.0011500000953674316,
129
+ "tpp_threshold_5_total_metric": 0.016850006580352784,
130
+ "tpp_threshold_5_intended_diff_only": 0.01740000247955322,
131
+ "tpp_threshold_5_unintended_diff_only": 0.0005499958992004394,
132
+ "tpp_threshold_10_total_metric": 0.02875001132488251,
133
+ "tpp_threshold_10_intended_diff_only": 0.03140000104904175,
134
+ "tpp_threshold_10_unintended_diff_only": 0.0026499897241592405,
135
+ "tpp_threshold_20_total_metric": 0.052849996089935306,
136
+ "tpp_threshold_20_intended_diff_only": 0.05819998979568482,
137
+ "tpp_threshold_20_unintended_diff_only": 0.005349993705749512,
138
+ "tpp_threshold_50_total_metric": 0.11339999735355377,
139
+ "tpp_threshold_50_intended_diff_only": 0.11959999799728394,
140
+ "tpp_threshold_50_unintended_diff_only": 0.006200000643730164,
141
+ "tpp_threshold_100_total_metric": 0.20465002059936524,
142
+ "tpp_threshold_100_intended_diff_only": 0.21980001926422119,
143
+ "tpp_threshold_100_unintended_diff_only": 0.015149998664855956,
144
+ "tpp_threshold_500_total_metric": 0.3726000338792801,
145
+ "tpp_threshold_500_intended_diff_only": 0.4038000345230103,
146
+ "tpp_threshold_500_unintended_diff_only": 0.031200000643730165
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.0155000239610672,
194
  "tpp_threshold_10_total_metric": 0.1409999579191208,
195
  "tpp_threshold_10_intended_diff_only": 0.15799999237060547,
196
+ "tpp_threshold_10_unintended_diff_only": 0.01700003445148468,
197
+ "tpp_threshold_20_total_metric": 0.23149995505809784,
198
+ "tpp_threshold_20_intended_diff_only": 0.343999981880188,
199
+ "tpp_threshold_20_unintended_diff_only": 0.11250002682209015,
200
+ "tpp_threshold_50_total_metric": 0.2254999577999115,
201
+ "tpp_threshold_50_intended_diff_only": 0.4179999828338623,
202
+ "tpp_threshold_50_unintended_diff_only": 0.1925000250339508,
203
+ "tpp_threshold_100_total_metric": 0.21774999797344208,
204
+ "tpp_threshold_100_intended_diff_only": 0.44200003147125244,
205
+ "tpp_threshold_100_unintended_diff_only": 0.22425003349781036,
206
+ "tpp_threshold_500_total_metric": 0.10149998962879181,
207
+ "tpp_threshold_500_intended_diff_only": 0.45600003004074097,
208
+ "tpp_threshold_500_unintended_diff_only": 0.35450004041194916
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.08900000154972076,
212
+ "tpp_threshold_2_intended_diff_only": 0.10000002384185791,
213
+ "tpp_threshold_2_unintended_diff_only": 0.011000022292137146,
214
+ "tpp_threshold_5_total_metric": 0.20075003802776337,
215
  "tpp_threshold_5_intended_diff_only": 0.27500003576278687,
216
+ "tpp_threshold_5_unintended_diff_only": 0.0742499977350235,
217
+ "tpp_threshold_10_total_metric": 0.21774998307228088,
218
  "tpp_threshold_10_intended_diff_only": 0.3330000042915344,
219
+ "tpp_threshold_10_unintended_diff_only": 0.11525002121925354,
220
+ "tpp_threshold_20_total_metric": 0.23224997520446777,
221
+ "tpp_threshold_20_intended_diff_only": 0.3569999933242798,
222
+ "tpp_threshold_20_unintended_diff_only": 0.12475001811981201,
223
+ "tpp_threshold_50_total_metric": 0.21050000190734863,
224
+ "tpp_threshold_50_intended_diff_only": 0.39500004053115845,
225
+ "tpp_threshold_50_unintended_diff_only": 0.18450003862380981,
226
+ "tpp_threshold_100_total_metric": 0.1807500123977661,
227
+ "tpp_threshold_100_intended_diff_only": 0.4360000491142273,
228
+ "tpp_threshold_100_unintended_diff_only": 0.2552500367164612,
229
+ "tpp_threshold_500_total_metric": 0.0885000079870224,
230
+ "tpp_threshold_500_intended_diff_only": 0.4590000510215759,
231
+ "tpp_threshold_500_unintended_diff_only": 0.37050004303455353
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.1067500114440918,
235
+ "tpp_threshold_2_intended_diff_only": 0.12400001287460327,
236
+ "tpp_threshold_2_unintended_diff_only": 0.017250001430511475,
237
+ "tpp_threshold_5_total_metric": 0.23225004971027374,
238
+ "tpp_threshold_5_intended_diff_only": 0.29700005054473877,
239
+ "tpp_threshold_5_unintended_diff_only": 0.06475000083446503,
240
+ "tpp_threshold_10_total_metric": 0.18050001561641693,
241
+ "tpp_threshold_10_intended_diff_only": 0.3490000367164612,
242
+ "tpp_threshold_10_unintended_diff_only": 0.16850002110004425,
243
+ "tpp_threshold_20_total_metric": 0.1367499977350235,
244
+ "tpp_threshold_20_intended_diff_only": 0.36900001764297485,
245
+ "tpp_threshold_20_unintended_diff_only": 0.23225001990795135,
246
+ "tpp_threshold_50_total_metric": 0.11875003576278687,
247
+ "tpp_threshold_50_intended_diff_only": 0.393000066280365,
248
+ "tpp_threshold_50_unintended_diff_only": 0.2742500305175781,
249
+ "tpp_threshold_100_total_metric": 0.08375002443790436,
250
+ "tpp_threshold_100_intended_diff_only": 0.41700005531311035,
251
+ "tpp_threshold_100_unintended_diff_only": 0.333250030875206,
252
+ "tpp_threshold_500_total_metric": 0.05399997532367706,
253
+ "tpp_threshold_500_intended_diff_only": 0.4520000219345093,
254
+ "tpp_threshold_500_unintended_diff_only": 0.3980000466108322
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.009499981999397278,
258
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704,
260
+ "tpp_threshold_5_total_metric": 0.24699999392032623,
261
+ "tpp_threshold_5_intended_diff_only": 0.3140000104904175,
262
+ "tpp_threshold_5_unintended_diff_only": 0.06700001657009125,
263
+ "tpp_threshold_10_total_metric": 0.2749999910593033,
264
  "tpp_threshold_10_intended_diff_only": 0.36500000953674316,
265
+ "tpp_threshold_10_unintended_diff_only": 0.09000001847743988,
266
+ "tpp_threshold_20_total_metric": 0.3005000352859497,
267
+ "tpp_threshold_20_intended_diff_only": 0.4270000457763672,
268
+ "tpp_threshold_20_unintended_diff_only": 0.12650001049041748,
269
+ "tpp_threshold_50_total_metric": 0.2460000365972519,
270
+ "tpp_threshold_50_intended_diff_only": 0.47700005769729614,
271
+ "tpp_threshold_50_unintended_diff_only": 0.23100002110004425,
272
+ "tpp_threshold_100_total_metric": 0.22150002419948578,
273
+ "tpp_threshold_100_intended_diff_only": 0.4830000400543213,
274
+ "tpp_threshold_100_unintended_diff_only": 0.2615000158548355,
275
+ "tpp_threshold_500_total_metric": 0.13225001096725464,
276
+ "tpp_threshold_500_intended_diff_only": 0.4930000305175781,
277
+ "tpp_threshold_500_unintended_diff_only": 0.3607500195503235
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.0072500258684158325,
281
+ "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464,
283
+ "tpp_threshold_5_total_metric": 0.02225002646446228,
284
  "tpp_threshold_5_intended_diff_only": 0.03800004720687866,
285
+ "tpp_threshold_5_unintended_diff_only": 0.015750020742416382,
286
+ "tpp_threshold_10_total_metric": 0.18550004065036774,
287
+ "tpp_threshold_10_intended_diff_only": 0.2600000500679016,
288
+ "tpp_threshold_10_unintended_diff_only": 0.07450000941753387,
289
+ "tpp_threshold_20_total_metric": 0.21799999475479126,
290
+ "tpp_threshold_20_intended_diff_only": 0.30000001192092896,
291
+ "tpp_threshold_20_unintended_diff_only": 0.0820000171661377,
292
+ "tpp_threshold_50_total_metric": 0.24150004982948303,
293
+ "tpp_threshold_50_intended_diff_only": 0.3980000615119934,
294
+ "tpp_threshold_50_unintended_diff_only": 0.15650001168251038,
295
+ "tpp_threshold_100_total_metric": 0.22500000894069672,
296
+ "tpp_threshold_100_intended_diff_only": 0.44200003147125244,
297
+ "tpp_threshold_100_unintended_diff_only": 0.21700002253055573,
298
+ "tpp_threshold_500_total_metric": 0.12375004589557648,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.35225002467632294
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.0012499988079071045,
306
+ "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
307
+ "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135,
308
+ "tpp_threshold_5_total_metric": 0.0015000253915786743,
309
+ "tpp_threshold_5_intended_diff_only": 0.003000020980834961,
310
+ "tpp_threshold_5_unintended_diff_only": 0.0014999955892562866,
311
+ "tpp_threshold_10_total_metric": 0.0007499903440475464,
312
+ "tpp_threshold_10_intended_diff_only": 0.001999974250793457,
313
+ "tpp_threshold_10_unintended_diff_only": 0.0012499839067459106,
314
+ "tpp_threshold_20_total_metric": 0.01000000536441803,
315
+ "tpp_threshold_20_intended_diff_only": 0.013999998569488525,
316
+ "tpp_threshold_20_unintended_diff_only": 0.003999993205070496,
317
+ "tpp_threshold_50_total_metric": 0.046249985694885254,
318
+ "tpp_threshold_50_intended_diff_only": 0.046999990940093994,
319
+ "tpp_threshold_50_unintended_diff_only": 0.0007500052452087402,
320
+ "tpp_threshold_100_total_metric": 0.09825000166893005,
321
+ "tpp_threshold_100_intended_diff_only": 0.10600000619888306,
322
+ "tpp_threshold_100_unintended_diff_only": 0.007750004529953003,
323
+ "tpp_threshold_500_total_metric": 0.3865000456571579,
324
+ "tpp_threshold_500_intended_diff_only": 0.40000003576278687,
325
+ "tpp_threshold_500_unintended_diff_only": 0.013499990105628967
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.015000015497207642,
329
+ "tpp_threshold_2_intended_diff_only": 0.013000011444091797,
330
+ "tpp_threshold_2_unintended_diff_only": -0.0020000040531158447,
331
+ "tpp_threshold_5_total_metric": 0.007750004529953003,
332
+ "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
333
+ "tpp_threshold_5_unintended_diff_only": 0.0062499940395355225,
334
+ "tpp_threshold_10_total_metric": 0.02800007164478302,
335
+ "tpp_threshold_10_intended_diff_only": 0.033000051975250244,
336
+ "tpp_threshold_10_unintended_diff_only": 0.004999980330467224,
337
+ "tpp_threshold_20_total_metric": 0.0975000411272049,
338
+ "tpp_threshold_20_intended_diff_only": 0.1040000319480896,
339
+ "tpp_threshold_20_unintended_diff_only": 0.006499990820884705,
340
+ "tpp_threshold_50_total_metric": 0.166500061750412,
341
+ "tpp_threshold_50_intended_diff_only": 0.18100005388259888,
342
+ "tpp_threshold_50_unintended_diff_only": 0.01449999213218689,
343
+ "tpp_threshold_100_total_metric": 0.28525008261203766,
344
+ "tpp_threshold_100_intended_diff_only": 0.30500006675720215,
345
+ "tpp_threshold_100_unintended_diff_only": 0.01974998414516449,
346
+ "tpp_threshold_500_total_metric": 0.40500007569789886,
347
+ "tpp_threshold_500_intended_diff_only": 0.43900007009506226,
348
+ "tpp_threshold_500_unintended_diff_only": 0.03399999439716339
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.0029999762773513794,
352
+ "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
353
+ "tpp_threshold_2_unintended_diff_only": -0.0010000020265579224,
354
+ "tpp_threshold_5_total_metric": 0.013250008225440979,
355
+ "tpp_threshold_5_intended_diff_only": 0.008000016212463379,
356
+ "tpp_threshold_5_unintended_diff_only": -0.0052499920129776,
357
+ "tpp_threshold_10_total_metric": 0.020500004291534424,
358
+ "tpp_threshold_10_intended_diff_only": 0.018000006675720215,
359
+ "tpp_threshold_10_unintended_diff_only": -0.002499997615814209,
360
+ "tpp_threshold_20_total_metric": 0.012249991297721863,
361
+ "tpp_threshold_20_intended_diff_only": 0.014999985694885254,
362
+ "tpp_threshold_20_unintended_diff_only": 0.002749994397163391,
363
+ "tpp_threshold_50_total_metric": 0.05524998903274536,
364
+ "tpp_threshold_50_intended_diff_only": 0.05699998140335083,
365
+ "tpp_threshold_50_unintended_diff_only": 0.0017499923706054688,
366
+ "tpp_threshold_100_total_metric": 0.1325000375509262,
367
+ "tpp_threshold_100_intended_diff_only": 0.15000003576278687,
368
+ "tpp_threshold_100_unintended_diff_only": 0.017499998211860657,
369
+ "tpp_threshold_500_total_metric": 0.3602500259876251,
370
+ "tpp_threshold_500_intended_diff_only": 0.3920000195503235,
371
+ "tpp_threshold_500_unintended_diff_only": 0.031749993562698364
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.0029999613761901855,
375
+ "tpp_threshold_2_intended_diff_only": 0.0029999613761901855,
376
+ "tpp_threshold_2_unintended_diff_only": 0.0,
377
+ "tpp_threshold_5_total_metric": 0.014750003814697266,
378
+ "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
379
+ "tpp_threshold_5_unintended_diff_only": -0.0007500052452087402,
380
+ "tpp_threshold_10_total_metric": 0.011249974370002747,
381
+ "tpp_threshold_10_intended_diff_only": 0.01699995994567871,
382
+ "tpp_threshold_10_unintended_diff_only": 0.005749985575675964,
383
+ "tpp_threshold_20_total_metric": 0.028499960899353027,
384
+ "tpp_threshold_20_intended_diff_only": 0.030999958515167236,
385
+ "tpp_threshold_20_unintended_diff_only": 0.002499997615814209,
386
+ "tpp_threshold_50_total_metric": 0.10099998116493225,
387
+ "tpp_threshold_50_intended_diff_only": 0.10799998044967651,
388
+ "tpp_threshold_50_unintended_diff_only": 0.006999999284744263,
389
+ "tpp_threshold_100_total_metric": 0.23374995589256287,
390
+ "tpp_threshold_100_intended_diff_only": 0.2529999613761902,
391
+ "tpp_threshold_100_unintended_diff_only": 0.01925000548362732,
392
+ "tpp_threshold_500_total_metric": 0.37175001204013824,
393
+ "tpp_threshold_500_intended_diff_only": 0.4150000214576721,
394
+ "tpp_threshold_500_unintended_diff_only": 0.043250009417533875
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.020999953150749207,
398
+ "tpp_threshold_2_intended_diff_only": 0.02599996328353882,
399
+ "tpp_threshold_2_unintended_diff_only": 0.005000010132789612,
400
+ "tpp_threshold_5_total_metric": 0.046999990940093994,
401
+ "tpp_threshold_5_intended_diff_only": 0.04799997806549072,
402
+ "tpp_threshold_5_unintended_diff_only": 0.0009999871253967285,
403
+ "tpp_threshold_10_total_metric": 0.0832500159740448,
404
+ "tpp_threshold_10_intended_diff_only": 0.08700001239776611,
405
+ "tpp_threshold_10_unintended_diff_only": 0.0037499964237213135,
406
+ "tpp_threshold_20_total_metric": 0.1159999817609787,
407
+ "tpp_threshold_20_intended_diff_only": 0.12699997425079346,
408
+ "tpp_threshold_20_unintended_diff_only": 0.010999992489814758,
409
+ "tpp_threshold_50_total_metric": 0.197999969124794,
410
+ "tpp_threshold_50_intended_diff_only": 0.20499998331069946,
411
+ "tpp_threshold_50_unintended_diff_only": 0.0070000141859054565,
412
+ "tpp_threshold_100_total_metric": 0.2735000252723694,
413
+ "tpp_threshold_100_intended_diff_only": 0.2850000262260437,
414
+ "tpp_threshold_100_unintended_diff_only": 0.011500000953674316,
415
+ "tpp_threshold_500_total_metric": 0.3395000100135803,
416
+ "tpp_threshold_500_intended_diff_only": 0.37300002574920654,
417
+ "tpp_threshold_500_unintended_diff_only": 0.03350001573562622
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-231933-tokens-950000000/scr/saebench_scr_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,8 +69,8 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "d0470eb0-5d97-4d7d-93f3-ec6aa1b0289d",
69
- "datetime_epoch_millis": 1776885164730,
70
  "eval_result_metrics": {
71
  "scr_metrics": {
72
  "scr_dir1_threshold_2": 0.28746353073212944,
@@ -78,18 +82,18 @@
78
  "scr_dir1_threshold_10": 0.06195265936947533,
79
  "scr_metric_threshold_10": 0.2138710296363283,
80
  "scr_dir2_threshold_10": 0.20874631929499857,
81
- "scr_dir1_threshold_20": null,
82
- "scr_metric_threshold_20": null,
83
- "scr_dir2_threshold_20": null,
84
- "scr_dir1_threshold_50": null,
85
- "scr_metric_threshold_50": null,
86
- "scr_dir2_threshold_50": null,
87
- "scr_dir1_threshold_100": null,
88
- "scr_metric_threshold_100": null,
89
- "scr_dir2_threshold_100": null,
90
- "scr_dir1_threshold_500": null,
91
- "scr_metric_threshold_500": null,
92
- "scr_dir2_threshold_500": null
93
  }
94
  },
95
  "eval_result_details": [
@@ -104,18 +108,18 @@
104
  "scr_dir1_threshold_10": 0.21794894327854547,
105
  "scr_metric_threshold_10": 0.015543994245836091,
106
  "scr_dir2_threshold_10": 0.015543994245836091,
107
- "scr_dir1_threshold_20": null,
108
- "scr_metric_threshold_20": null,
109
- "scr_dir2_threshold_20": null,
110
- "scr_dir1_threshold_50": null,
111
- "scr_metric_threshold_50": null,
112
- "scr_dir2_threshold_50": null,
113
- "scr_dir1_threshold_100": null,
114
- "scr_metric_threshold_100": null,
115
- "scr_dir2_threshold_100": null,
116
- "scr_dir1_threshold_500": null,
117
- "scr_metric_threshold_500": null,
118
- "scr_dir2_threshold_500": null
119
  },
120
  {
121
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
@@ -128,18 +132,18 @@
128
  "scr_dir1_threshold_10": -0.07575748681586188,
129
  "scr_metric_threshold_10": 0.32721708301814567,
130
  "scr_dir2_threshold_10": 0.32721708301814567,
131
- "scr_dir1_threshold_20": null,
132
- "scr_metric_threshold_20": null,
133
- "scr_dir2_threshold_20": null,
134
- "scr_dir1_threshold_50": null,
135
- "scr_metric_threshold_50": null,
136
- "scr_dir2_threshold_50": null,
137
- "scr_dir1_threshold_100": null,
138
- "scr_metric_threshold_100": null,
139
- "scr_dir2_threshold_100": null,
140
- "scr_dir1_threshold_500": null,
141
- "scr_metric_threshold_500": null,
142
- "scr_dir2_threshold_500": null
143
  },
144
  {
145
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
@@ -152,18 +156,18 @@
152
  "scr_dir1_threshold_10": -0.10937450523496334,
153
  "scr_metric_threshold_10": 0.10552761109862276,
154
  "scr_dir2_threshold_10": 0.10552761109862276,
155
- "scr_dir1_threshold_20": null,
156
- "scr_metric_threshold_20": null,
157
- "scr_dir2_threshold_20": null,
158
- "scr_dir1_threshold_50": null,
159
- "scr_metric_threshold_50": null,
160
- "scr_dir2_threshold_50": null,
161
- "scr_dir1_threshold_100": null,
162
- "scr_metric_threshold_100": null,
163
- "scr_dir2_threshold_100": null,
164
- "scr_dir1_threshold_500": null,
165
- "scr_metric_threshold_500": null,
166
- "scr_dir2_threshold_500": null
167
  },
168
  {
169
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
@@ -176,18 +180,18 @@
176
  "scr_dir1_threshold_10": -0.47651036780868905,
177
  "scr_metric_threshold_10": 0.08598729016413297,
178
  "scr_dir2_threshold_10": 0.08598729016413297,
179
- "scr_dir1_threshold_20": null,
180
- "scr_metric_threshold_20": null,
181
- "scr_dir2_threshold_20": null,
182
- "scr_dir1_threshold_50": null,
183
- "scr_metric_threshold_50": null,
184
- "scr_dir2_threshold_50": null,
185
- "scr_dir1_threshold_100": null,
186
- "scr_metric_threshold_100": null,
187
- "scr_dir2_threshold_100": null,
188
- "scr_dir1_threshold_500": null,
189
- "scr_metric_threshold_500": null,
190
- "scr_dir2_threshold_500": null
191
  },
192
  {
193
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
@@ -200,18 +204,18 @@
200
  "scr_dir1_threshold_10": 0.08947378822971379,
201
  "scr_metric_threshold_10": 0.4166666863771947,
202
  "scr_dir2_threshold_10": 0.4166666863771947,
203
- "scr_dir1_threshold_20": null,
204
- "scr_metric_threshold_20": null,
205
- "scr_dir2_threshold_20": null,
206
- "scr_dir1_threshold_50": null,
207
- "scr_metric_threshold_50": null,
208
- "scr_dir2_threshold_50": null,
209
- "scr_dir1_threshold_100": null,
210
- "scr_metric_threshold_100": null,
211
- "scr_dir2_threshold_100": null,
212
- "scr_dir1_threshold_500": null,
213
- "scr_metric_threshold_500": null,
214
- "scr_dir2_threshold_500": null
215
  },
216
  {
217
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
@@ -224,18 +228,18 @@
224
  "scr_dir1_threshold_10": 0.3661201278080144,
225
  "scr_metric_threshold_10": 0.14122136275770192,
226
  "scr_dir2_threshold_10": 0.14122136275770192,
227
- "scr_dir1_threshold_20": null,
228
- "scr_metric_threshold_20": null,
229
- "scr_dir2_threshold_20": null,
230
- "scr_dir1_threshold_50": null,
231
- "scr_metric_threshold_50": null,
232
- "scr_dir2_threshold_50": null,
233
- "scr_dir1_threshold_100": null,
234
- "scr_metric_threshold_100": null,
235
- "scr_dir2_threshold_100": null,
236
- "scr_dir1_threshold_500": null,
237
- "scr_metric_threshold_500": null,
238
- "scr_dir2_threshold_500": null
239
  },
240
  {
241
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
@@ -248,18 +252,18 @@
248
  "scr_dir1_threshold_10": 0.3069766255570261,
249
  "scr_metric_threshold_10": 0.44206005948697535,
250
  "scr_dir2_threshold_10": 0.44206005948697535,
251
- "scr_dir1_threshold_20": null,
252
- "scr_metric_threshold_20": null,
253
- "scr_dir2_threshold_20": null,
254
- "scr_dir1_threshold_50": null,
255
- "scr_metric_threshold_50": null,
256
- "scr_dir2_threshold_50": null,
257
- "scr_dir1_threshold_100": null,
258
- "scr_metric_threshold_100": null,
259
- "scr_dir2_threshold_100": null,
260
- "scr_dir1_threshold_500": null,
261
- "scr_metric_threshold_500": null,
262
- "scr_dir2_threshold_500": null
263
  },
264
  {
265
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
@@ -272,18 +276,18 @@
272
  "scr_dir1_threshold_10": 0.17674414994201715,
273
  "scr_metric_threshold_10": 0.17674414994201715,
274
  "scr_dir2_threshold_10": 0.13574646721137937,
275
- "scr_dir1_threshold_20": null,
276
- "scr_metric_threshold_20": null,
277
- "scr_dir2_threshold_20": null,
278
- "scr_dir1_threshold_50": null,
279
- "scr_metric_threshold_50": null,
280
- "scr_dir2_threshold_50": null,
281
- "scr_dir1_threshold_100": null,
282
- "scr_metric_threshold_100": null,
283
- "scr_dir2_threshold_100": null,
284
- "scr_dir1_threshold_500": null,
285
- "scr_metric_threshold_500": null,
286
- "scr_dir2_threshold_500": null
287
  }
288
  ],
289
  "sae_bench_commit_hash": "Unknown",
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "59dc63a6-488a-4b00-88b6-92ffd0068930",
73
+ "datetime_epoch_millis": 1777316537790,
74
  "eval_result_metrics": {
75
  "scr_metrics": {
76
  "scr_dir1_threshold_2": 0.28746353073212944,
 
82
  "scr_dir1_threshold_10": 0.06195265936947533,
83
  "scr_metric_threshold_10": 0.2138710296363283,
84
  "scr_dir2_threshold_10": 0.20874631929499857,
85
+ "scr_dir1_threshold_20": 0.047000745989897424,
86
+ "scr_metric_threshold_20": 0.24324757819781082,
87
+ "scr_dir2_threshold_20": 0.23505808139056414,
88
+ "scr_dir1_threshold_50": 0.007289984864363129,
89
+ "scr_metric_threshold_50": 0.35559486332268514,
90
+ "scr_dir2_threshold_50": 0.34089950355370285,
91
+ "scr_dir1_threshold_100": -0.03458230235004653,
92
+ "scr_metric_threshold_100": 0.3201833541088444,
93
+ "scr_dir2_threshold_100": 0.30790828805513276,
94
+ "scr_dir1_threshold_500": -1.0669637122741167,
95
+ "scr_metric_threshold_500": 0.18804673121676252,
96
+ "scr_dir2_threshold_500": 0.17188868827086867
97
  }
98
  },
99
  "eval_result_details": [
 
108
  "scr_dir1_threshold_10": 0.21794894327854547,
109
  "scr_metric_threshold_10": 0.015543994245836091,
110
  "scr_dir2_threshold_10": 0.015543994245836091,
111
+ "scr_dir1_threshold_20": 0.2820514388024664,
112
+ "scr_metric_threshold_20": 0.08808283994795557,
113
+ "scr_dir2_threshold_20": 0.08808283994795557,
114
+ "scr_dir1_threshold_50": 0.03846180297916207,
115
+ "scr_metric_threshold_50": 0.24870468001419394,
116
+ "scr_dir2_threshold_50": 0.24870468001419394,
117
+ "scr_dir1_threshold_100": 0.37179539103317,
118
+ "scr_metric_threshold_100": 0.26683931423164214,
119
+ "scr_dir2_threshold_100": 0.26683931423164214,
120
+ "scr_dir1_threshold_500": -2.6410249552392093,
121
+ "scr_metric_threshold_500": -0.1398964114610147,
122
+ "scr_dir2_threshold_500": -0.1398964114610147
123
  },
124
  {
125
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
 
132
  "scr_dir1_threshold_10": -0.07575748681586188,
133
  "scr_metric_threshold_10": 0.32721708301814567,
134
  "scr_dir2_threshold_10": 0.32721708301814567,
135
+ "scr_dir1_threshold_20": -0.06818182844432363,
136
+ "scr_metric_threshold_20": 0.11926615203740328,
137
+ "scr_dir2_threshold_20": 0.11926615203740328,
138
+ "scr_dir1_threshold_50": 0.5757577125909817,
139
+ "scr_metric_threshold_50": 0.35779827383502094,
140
+ "scr_dir2_threshold_50": 0.35779827383502094,
141
+ "scr_dir1_threshold_100": 0.03787874340793094,
142
+ "scr_metric_threshold_100": 0.535168196276077,
143
+ "scr_dir2_threshold_100": 0.535168196276077,
144
+ "scr_dir1_threshold_500": -1.9848482317066838,
145
+ "scr_metric_threshold_500": 0.10703371216609098,
146
+ "scr_dir2_threshold_500": 0.10703371216609098
147
  },
148
  {
149
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
 
156
  "scr_dir1_threshold_10": -0.10937450523496334,
157
  "scr_metric_threshold_10": 0.10552761109862276,
158
  "scr_dir2_threshold_10": 0.10552761109862276,
159
+ "scr_dir1_threshold_20": 0.21874994179234863,
160
+ "scr_metric_threshold_20": 0.19597989197183965,
161
+ "scr_dir2_threshold_20": 0.19597989197183965,
162
+ "scr_dir1_threshold_50": -0.9062502910382568,
163
+ "scr_metric_threshold_50": 0.16331655168313458,
164
+ "scr_dir2_threshold_50": 0.16331655168313458,
165
+ "scr_dir1_threshold_100": -0.5156247962732202,
166
+ "scr_metric_threshold_100": 0.12060309108441948,
167
+ "scr_dir2_threshold_100": 0.12060309108441948,
168
+ "scr_dir1_threshold_500": -3.4062493597158348,
169
+ "scr_metric_threshold_500": -0.012562800147903362,
170
+ "scr_dir2_threshold_500": -0.012562800147903362
171
  },
172
  {
173
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
 
180
  "scr_dir1_threshold_10": -0.47651036780868905,
181
  "scr_metric_threshold_10": 0.08598729016413297,
182
  "scr_dir2_threshold_10": 0.08598729016413297,
183
+ "scr_dir1_threshold_20": -1.2080538281985995,
184
+ "scr_metric_threshold_20": 0.08598729016413297,
185
+ "scr_dir2_threshold_20": 0.08598729016413297,
186
+ "scr_dir1_threshold_50": -0.8456379597608613,
187
+ "scr_metric_threshold_50": 0.2324841127051662,
188
+ "scr_dir2_threshold_50": 0.2324841127051662,
189
+ "scr_dir1_threshold_100": -1.3288592510215798,
190
+ "scr_metric_threshold_100": -0.09235663469908957,
191
+ "scr_dir2_threshold_100": -0.09235663469908957,
192
+ "scr_dir1_threshold_500": -1.7114098899714179,
193
+ "scr_metric_threshold_500": -0.5382164468571459,
194
+ "scr_dir2_threshold_500": -0.5382164468571459
195
  },
196
  {
197
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
 
204
  "scr_dir1_threshold_10": 0.08947378822971379,
205
  "scr_metric_threshold_10": 0.4166666863771947,
206
  "scr_dir2_threshold_10": 0.4166666863771947,
207
+ "scr_dir1_threshold_20": 0.09473687842942387,
208
+ "scr_metric_threshold_20": 0.4999998817368318,
209
+ "scr_dir2_threshold_20": 0.4999998817368318,
210
+ "scr_dir1_threshold_50": -0.12105264313663487,
211
+ "scr_metric_threshold_50": 0.6150794054390177,
212
+ "scr_dir2_threshold_50": 0.6150794054390177,
213
+ "scr_dir1_threshold_100": -0.16842108235134678,
214
+ "scr_metric_threshold_100": 0.654761901946115,
215
+ "scr_dir2_threshold_100": 0.654761901946115,
216
+ "scr_dir1_threshold_500": -0.14736840784384586,
217
+ "scr_metric_threshold_500": 0.7261904902694248,
218
+ "scr_dir2_threshold_500": 0.7261904902694248
219
  },
220
  {
221
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
 
228
  "scr_dir1_threshold_10": 0.3661201278080144,
229
  "scr_metric_threshold_10": 0.14122136275770192,
230
  "scr_dir2_threshold_10": 0.14122136275770192,
231
+ "scr_dir1_threshold_20": 0.3497268934249226,
232
+ "scr_metric_threshold_20": 0.1908397224425799,
233
+ "scr_dir2_threshold_20": 0.1908397224425799,
234
+ "scr_dir1_threshold_50": 0.4426228653879351,
235
+ "scr_metric_threshold_50": 0.3091603913067316,
236
+ "scr_dir2_threshold_50": 0.3091603913067316,
237
+ "scr_dir1_threshold_100": 0.4754099855711136,
238
+ "scr_metric_threshold_100": 0.35496177909383203,
239
+ "scr_dir2_threshold_100": 0.35496177909383203,
240
+ "scr_dir1_threshold_500": 0.3551913048859532,
241
+ "scr_metric_threshold_500": 0.4274808326722602,
242
+ "scr_dir2_threshold_500": 0.4274808326722602
243
  },
244
  {
245
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
 
252
  "scr_dir1_threshold_10": 0.3069766255570261,
253
  "scr_metric_threshold_10": 0.44206005948697535,
254
  "scr_dir2_threshold_10": 0.44206005948697535,
255
+ "scr_dir1_threshold_20": 0.46046493833553914,
256
+ "scr_metric_threshold_20": 0.5193133135043415,
257
+ "scr_dir2_threshold_20": 0.5193133135043415,
258
+ "scr_dir1_threshold_50": 0.5441858866203395,
259
+ "scr_metric_threshold_50": 0.587982986325977,
260
+ "scr_dir2_threshold_50": 0.587982986325977,
261
+ "scr_dir1_threshold_100": 0.5674416747849105,
262
+ "scr_metric_threshold_100": 0.43776826888911,
263
+ "scr_dir2_threshold_100": 0.43776826888911,
264
+ "scr_dir1_threshold_500": 0.5720930542025149,
265
+ "scr_metric_threshold_500": 0.506437685896798,
266
+ "scr_dir2_threshold_500": 0.506437685896798
267
  },
268
  {
269
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
 
276
  "scr_dir1_threshold_10": 0.17674414994201715,
277
  "scr_metric_threshold_10": 0.17674414994201715,
278
  "scr_dir2_threshold_10": 0.13574646721137937,
279
+ "scr_dir1_threshold_20": 0.24651153377740184,
280
+ "scr_metric_threshold_20": 0.24651153377740184,
281
+ "scr_dir2_threshold_20": 0.1809955593194283,
282
+ "scr_dir1_threshold_50": 0.3302325052722394,
283
+ "scr_metric_threshold_50": 0.3302325052722394,
284
+ "scr_dir2_threshold_50": 0.21266962712038115,
285
+ "scr_dir1_threshold_100": 0.2837209160486496,
286
+ "scr_metric_threshold_100": 0.2837209160486496,
287
+ "scr_dir2_threshold_100": 0.18552038761895642,
288
+ "scr_dir1_threshold_500": 0.42790678719559005,
289
+ "scr_metric_threshold_500": 0.42790678719559005,
290
+ "scr_dir2_threshold_500": 0.2986424436284392
291
  }
292
  ],
293
  "sae_bench_commit_hash": "Unknown",
gemma-2-2b-btk/k-100/seed-0/snapshots/step-231933-tokens-950000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "e1c16378-1481-4085-bdc1-6d9c2c28dc1d",
69
- "datetime_epoch_millis": 1776885330410,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.024399994313716887,
73
- "tpp_threshold_2_intended_diff_only": 0.025899994373321533,
74
- "tpp_threshold_2_unintended_diff_only": 0.0015000000596046448,
75
- "tpp_threshold_5_total_metric": 0.0909250020980835,
76
- "tpp_threshold_5_intended_diff_only": 0.1090000033378601,
77
- "tpp_threshold_5_unintended_diff_only": 0.01807500123977661,
78
- "tpp_threshold_10_total_metric": 0.1162249967455864,
79
- "tpp_threshold_10_intended_diff_only": 0.15770000219345093,
80
- "tpp_threshold_10_unintended_diff_only": 0.04147500544786453,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.039299994707107544,
99
- "tpp_threshold_2_intended_diff_only": 0.042200005054473876,
100
- "tpp_threshold_2_unintended_diff_only": 0.002900010347366333,
101
- "tpp_threshold_5_total_metric": 0.16690001487731934,
102
- "tpp_threshold_5_intended_diff_only": 0.20100002288818358,
103
- "tpp_threshold_5_unintended_diff_only": 0.034100008010864255,
104
- "tpp_threshold_10_total_metric": 0.206850004196167,
105
  "tpp_threshold_10_intended_diff_only": 0.2876000165939331,
106
- "tpp_threshold_10_unintended_diff_only": 0.08075001239776611,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.009499993920326234,
123
- "tpp_threshold_2_intended_diff_only": 0.00959998369216919,
124
- "tpp_threshold_2_unintended_diff_only": 9.998977184295654e-05,
125
- "tpp_threshold_5_total_metric": 0.014949989318847657,
126
- "tpp_threshold_5_intended_diff_only": 0.016999983787536622,
127
- "tpp_threshold_5_unintended_diff_only": 0.0020499944686889647,
128
- "tpp_threshold_10_total_metric": 0.025599989295005798,
129
- "tpp_threshold_10_intended_diff_only": 0.02779998779296875,
130
- "tpp_threshold_10_unintended_diff_only": 0.0021999984979629516,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.018750011920928955,
190
  "tpp_threshold_10_total_metric": 0.17674997448921204,
191
  "tpp_threshold_10_intended_diff_only": 0.19599997997283936,
192
- "tpp_threshold_10_unintended_diff_only": 0.01925000548362732
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": -0.0002500265836715698,
196
- "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
197
- "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983,
198
- "tpp_threshold_5_total_metric": 0.1860000342130661,
199
  "tpp_threshold_5_intended_diff_only": 0.22400003671646118,
200
- "tpp_threshold_5_unintended_diff_only": 0.03800000250339508,
201
- "tpp_threshold_10_total_metric": 0.21649998426437378,
202
  "tpp_threshold_10_intended_diff_only": 0.29600000381469727,
203
- "tpp_threshold_10_unintended_diff_only": 0.07950001955032349
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.06874999403953552,
207
- "tpp_threshold_2_intended_diff_only": 0.07300001382827759,
208
- "tpp_threshold_2_unintended_diff_only": 0.004250019788742065,
209
- "tpp_threshold_5_total_metric": 0.21925002336502075,
210
- "tpp_threshold_5_intended_diff_only": 0.25700002908706665,
211
- "tpp_threshold_5_unintended_diff_only": 0.0377500057220459,
212
- "tpp_threshold_10_total_metric": 0.18250000476837158,
213
- "tpp_threshold_10_intended_diff_only": 0.33100003004074097,
214
- "tpp_threshold_10_unintended_diff_only": 0.14850002527236938
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.011750012636184692,
218
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
219
- "tpp_threshold_2_unintended_diff_only": -0.002750009298324585,
220
- "tpp_threshold_5_total_metric": 0.1184999942779541,
221
  "tpp_threshold_5_intended_diff_only": 0.1340000033378601,
222
- "tpp_threshold_5_unintended_diff_only": 0.015500009059906006,
223
- "tpp_threshold_10_total_metric": 0.2645000219345093,
224
  "tpp_threshold_10_intended_diff_only": 0.3500000238418579,
225
- "tpp_threshold_10_unintended_diff_only": 0.08550000190734863
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.0050000399351119995,
229
- "tpp_threshold_2_intended_diff_only": 0.006000041961669922,
230
- "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224,
231
- "tpp_threshold_5_total_metric": 0.15750004351139069,
232
  "tpp_threshold_5_intended_diff_only": 0.21800005435943604,
233
- "tpp_threshold_5_unintended_diff_only": 0.06050001084804535,
234
- "tpp_threshold_10_total_metric": 0.1940000355243683,
235
- "tpp_threshold_10_intended_diff_only": 0.26500004529953003,
236
- "tpp_threshold_10_unintended_diff_only": 0.07100000977516174
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.00875002145767212,
242
- "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
243
- "tpp_threshold_2_unintended_diff_only": -0.00475001335144043,
244
- "tpp_threshold_5_total_metric": 0.010499969124794006,
245
- "tpp_threshold_5_intended_diff_only": 0.006999969482421875,
246
- "tpp_threshold_5_unintended_diff_only": -0.0034999996423721313,
247
- "tpp_threshold_10_total_metric": 0.0007499903440475464,
248
- "tpp_threshold_10_intended_diff_only": 0.004999995231628418,
249
- "tpp_threshold_10_unintended_diff_only": 0.004250004887580872
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": 0.002750024199485779,
253
- "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
254
- "tpp_threshold_2_unintended_diff_only": 0.0052499920129776,
255
- "tpp_threshold_5_total_metric": 0.006750017404556274,
256
- "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
257
- "tpp_threshold_5_unintended_diff_only": 0.007249981164932251,
258
- "tpp_threshold_10_total_metric": 0.03975000977516174,
259
- "tpp_threshold_10_intended_diff_only": 0.04100000858306885,
260
- "tpp_threshold_10_unintended_diff_only": 0.0012499988079071045
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.012500002980232239,
264
- "tpp_threshold_2_intended_diff_only": 0.0059999823570251465,
265
- "tpp_threshold_2_unintended_diff_only": -0.006500020623207092,
266
- "tpp_threshold_5_total_metric": 0.019000008702278137,
267
- "tpp_threshold_5_intended_diff_only": 0.018000006675720215,
268
- "tpp_threshold_5_unintended_diff_only": -0.0010000020265579224,
269
- "tpp_threshold_10_total_metric": 0.01799999177455902,
270
- "tpp_threshold_10_intended_diff_only": 0.019999980926513672,
271
- "tpp_threshold_10_unintended_diff_only": 0.001999989151954651
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.00850003957748413,
275
  "tpp_threshold_2_intended_diff_only": -0.001000046730041504,
276
- "tpp_threshold_2_unintended_diff_only": 0.007499992847442627,
277
- "tpp_threshold_5_total_metric": 0.001999959349632263,
278
- "tpp_threshold_5_intended_diff_only": 0.0029999613761901855,
279
- "tpp_threshold_5_unintended_diff_only": 0.0010000020265579224,
280
- "tpp_threshold_10_total_metric": 0.0054999589920043945,
281
- "tpp_threshold_10_intended_diff_only": 0.007999956607818604,
282
- "tpp_threshold_10_unintended_diff_only": 0.002499997615814209
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.03199996054172516,
286
- "tpp_threshold_2_intended_diff_only": 0.030999958515167236,
287
- "tpp_threshold_2_unintended_diff_only": -0.0010000020265579224,
288
- "tpp_threshold_5_total_metric": 0.0364999920129776,
289
- "tpp_threshold_5_intended_diff_only": 0.042999982833862305,
290
- "tpp_threshold_5_unintended_diff_only": 0.006499990820884705,
291
- "tpp_threshold_10_total_metric": 0.06399999558925629,
292
- "tpp_threshold_10_intended_diff_only": 0.06499999761581421,
293
- "tpp_threshold_10_unintended_diff_only": 0.0010000020265579224
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "41086c4e-63d7-4953-8646-1ffdb6f54620",
73
+ "datetime_epoch_millis": 1777316814695,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.022574988007545472,
77
+ "tpp_threshold_2_intended_diff_only": 0.025399994850158692,
78
+ "tpp_threshold_2_unintended_diff_only": 0.00282500684261322,
79
+ "tpp_threshold_5_total_metric": 0.09110000878572465,
80
+ "tpp_threshold_5_intended_diff_only": 0.10790001153945923,
81
+ "tpp_threshold_5_unintended_diff_only": 0.016800002753734587,
82
+ "tpp_threshold_10_total_metric": 0.11690000593662261,
83
+ "tpp_threshold_10_intended_diff_only": 0.15910000801086424,
84
+ "tpp_threshold_10_unintended_diff_only": 0.042200002074241634,
85
+ "tpp_threshold_20_total_metric": 0.14222500771284105,
86
+ "tpp_threshold_20_intended_diff_only": 0.20410001873970032,
87
+ "tpp_threshold_20_unintended_diff_only": 0.061875011026859286,
88
+ "tpp_threshold_50_total_metric": 0.15880000591278076,
89
+ "tpp_threshold_50_intended_diff_only": 0.2608000159263611,
90
+ "tpp_threshold_50_unintended_diff_only": 0.10200001001358033,
91
+ "tpp_threshold_100_total_metric": 0.17900000810623168,
92
+ "tpp_threshold_100_intended_diff_only": 0.3216000199317932,
93
+ "tpp_threshold_100_unintended_diff_only": 0.14260001182556153,
94
+ "tpp_threshold_500_total_metric": 0.23692501187324522,
95
+ "tpp_threshold_500_intended_diff_only": 0.43640002608299255,
96
+ "tpp_threshold_500_unintended_diff_only": 0.1994750142097473
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.03794998526573181,
103
+ "tpp_threshold_2_intended_diff_only": 0.04259999990463257,
104
+ "tpp_threshold_2_unintended_diff_only": 0.004650014638900757,
105
+ "tpp_threshold_5_total_metric": 0.1678500145673752,
106
+ "tpp_threshold_5_intended_diff_only": 0.20080002546310424,
107
+ "tpp_threshold_5_unintended_diff_only": 0.03295001089572906,
108
+ "tpp_threshold_10_total_metric": 0.20570000410079955,
109
  "tpp_threshold_10_intended_diff_only": 0.2876000165939331,
110
+ "tpp_threshold_10_unintended_diff_only": 0.08190001249313354,
111
+ "tpp_threshold_20_total_metric": 0.23370001912117006,
112
+ "tpp_threshold_20_intended_diff_only": 0.3536000370979309,
113
+ "tpp_threshold_20_unintended_diff_only": 0.11990001797676086,
114
+ "tpp_threshold_50_total_metric": 0.21265000998973846,
115
+ "tpp_threshold_50_intended_diff_only": 0.410200035572052,
116
+ "tpp_threshold_50_unintended_diff_only": 0.19755002558231355,
117
+ "tpp_threshold_100_total_metric": 0.1687000125646591,
118
+ "tpp_threshold_100_intended_diff_only": 0.44060003757476807,
119
+ "tpp_threshold_100_unintended_diff_only": 0.27190002501010896,
120
+ "tpp_threshold_500_total_metric": 0.09975000619888305,
121
+ "tpp_threshold_500_intended_diff_only": 0.46720004081726074,
122
+ "tpp_threshold_500_unintended_diff_only": 0.36745003461837766
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.007199990749359131,
127
+ "tpp_threshold_2_intended_diff_only": 0.008199989795684814,
128
+ "tpp_threshold_2_unintended_diff_only": 0.0009999990463256836,
129
+ "tpp_threshold_5_total_metric": 0.014350003004074097,
130
+ "tpp_threshold_5_intended_diff_only": 0.01499999761581421,
131
+ "tpp_threshold_5_unintended_diff_only": 0.0006499946117401123,
132
+ "tpp_threshold_10_total_metric": 0.02810000777244568,
133
+ "tpp_threshold_10_intended_diff_only": 0.03059999942779541,
134
+ "tpp_threshold_10_unintended_diff_only": 0.0024999916553497315,
135
+ "tpp_threshold_20_total_metric": 0.050749996304512025,
136
+ "tpp_threshold_20_intended_diff_only": 0.05460000038146973,
137
+ "tpp_threshold_20_unintended_diff_only": 0.0038500040769577025,
138
+ "tpp_threshold_50_total_metric": 0.10495000183582306,
139
+ "tpp_threshold_50_intended_diff_only": 0.11139999628067017,
140
+ "tpp_threshold_50_unintended_diff_only": 0.006449994444847107,
141
+ "tpp_threshold_100_total_metric": 0.18930000364780425,
142
+ "tpp_threshold_100_intended_diff_only": 0.20260000228881836,
143
+ "tpp_threshold_100_unintended_diff_only": 0.0132999986410141,
144
+ "tpp_threshold_500_total_metric": 0.3741000175476074,
145
+ "tpp_threshold_500_intended_diff_only": 0.40560001134872437,
146
+ "tpp_threshold_500_unintended_diff_only": 0.03149999380111694
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.018750011920928955,
194
  "tpp_threshold_10_total_metric": 0.17674997448921204,
195
  "tpp_threshold_10_intended_diff_only": 0.19599997997283936,
196
+ "tpp_threshold_10_unintended_diff_only": 0.01925000548362732,
197
+ "tpp_threshold_20_total_metric": 0.25324998795986176,
198
+ "tpp_threshold_20_intended_diff_only": 0.3460000157356262,
199
+ "tpp_threshold_20_unintended_diff_only": 0.09275002777576447,
200
+ "tpp_threshold_50_total_metric": 0.23749999701976776,
201
+ "tpp_threshold_50_intended_diff_only": 0.4190000295639038,
202
+ "tpp_threshold_50_unintended_diff_only": 0.18150003254413605,
203
+ "tpp_threshold_100_total_metric": 0.2082499861717224,
204
+ "tpp_threshold_100_intended_diff_only": 0.44300001859664917,
205
+ "tpp_threshold_100_unintended_diff_only": 0.23475003242492676,
206
+ "tpp_threshold_500_total_metric": 0.10849998891353607,
207
+ "tpp_threshold_500_intended_diff_only": 0.45600003004074097,
208
+ "tpp_threshold_500_unintended_diff_only": 0.3475000411272049
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.004499971866607666,
212
+ "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
213
+ "tpp_threshold_2_unintended_diff_only": 0.000500023365020752,
214
+ "tpp_threshold_5_total_metric": 0.19050002098083496,
215
  "tpp_threshold_5_intended_diff_only": 0.22400003671646118,
216
+ "tpp_threshold_5_unintended_diff_only": 0.03350001573562622,
217
+ "tpp_threshold_10_total_metric": 0.21424998342990875,
218
  "tpp_threshold_10_intended_diff_only": 0.29600000381469727,
219
+ "tpp_threshold_10_unintended_diff_only": 0.08175002038478851,
220
+ "tpp_threshold_20_total_metric": 0.24300000071525574,
221
+ "tpp_threshold_20_intended_diff_only": 0.32600003480911255,
222
+ "tpp_threshold_20_unintended_diff_only": 0.08300003409385681,
223
+ "tpp_threshold_50_total_metric": 0.21724997460842133,
224
+ "tpp_threshold_50_intended_diff_only": 0.375,
225
+ "tpp_threshold_50_unintended_diff_only": 0.15775002539157867,
226
+ "tpp_threshold_100_total_metric": 0.16199998557567596,
227
+ "tpp_threshold_100_intended_diff_only": 0.42100000381469727,
228
+ "tpp_threshold_100_unintended_diff_only": 0.2590000182390213,
229
+ "tpp_threshold_500_total_metric": 0.08475001156330109,
230
+ "tpp_threshold_500_intended_diff_only": 0.4590000510215759,
231
+ "tpp_threshold_500_unintended_diff_only": 0.37425003945827484
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.056000009179115295,
235
+ "tpp_threshold_2_intended_diff_only": 0.06800001859664917,
236
+ "tpp_threshold_2_unintended_diff_only": 0.012000009417533875,
237
+ "tpp_threshold_5_total_metric": 0.218500018119812,
238
+ "tpp_threshold_5_intended_diff_only": 0.2560000419616699,
239
+ "tpp_threshold_5_unintended_diff_only": 0.03750002384185791,
240
+ "tpp_threshold_10_total_metric": 0.17949999868869781,
241
+ "tpp_threshold_10_intended_diff_only": 0.3320000171661377,
242
+ "tpp_threshold_10_unintended_diff_only": 0.15250001847743988,
243
+ "tpp_threshold_20_total_metric": 0.1472500115633011,
244
+ "tpp_threshold_20_intended_diff_only": 0.36000001430511475,
245
+ "tpp_threshold_20_unintended_diff_only": 0.21275000274181366,
246
+ "tpp_threshold_50_total_metric": 0.11400002241134644,
247
+ "tpp_threshold_50_intended_diff_only": 0.3800000548362732,
248
+ "tpp_threshold_50_unintended_diff_only": 0.26600003242492676,
249
+ "tpp_threshold_100_total_metric": 0.06650002300739288,
250
+ "tpp_threshold_100_intended_diff_only": 0.40800005197525024,
251
+ "tpp_threshold_100_unintended_diff_only": 0.34150002896785736,
252
+ "tpp_threshold_500_total_metric": 0.05399999022483826,
253
+ "tpp_threshold_500_intended_diff_only": 0.4520000219345093,
254
+ "tpp_threshold_500_unintended_diff_only": 0.398000031709671
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.009499981999397278,
258
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704,
260
+ "tpp_threshold_5_total_metric": 0.12399999797344208,
261
  "tpp_threshold_5_intended_diff_only": 0.1340000033378601,
262
+ "tpp_threshold_5_unintended_diff_only": 0.01000000536441803,
263
+ "tpp_threshold_10_total_metric": 0.26775000989437103,
264
  "tpp_threshold_10_intended_diff_only": 0.3500000238418579,
265
+ "tpp_threshold_10_unintended_diff_only": 0.08225001394748688,
266
+ "tpp_threshold_20_total_metric": 0.29575005173683167,
267
+ "tpp_threshold_20_intended_diff_only": 0.43000006675720215,
268
+ "tpp_threshold_20_unintended_diff_only": 0.13425001502037048,
269
+ "tpp_threshold_50_total_metric": 0.2575000077486038,
270
+ "tpp_threshold_50_intended_diff_only": 0.4750000238418579,
271
+ "tpp_threshold_50_unintended_diff_only": 0.2175000160932541,
272
+ "tpp_threshold_100_total_metric": 0.20125003159046173,
273
+ "tpp_threshold_100_intended_diff_only": 0.4830000400543213,
274
+ "tpp_threshold_100_unintended_diff_only": 0.28175000846385956,
275
+ "tpp_threshold_500_total_metric": 0.1290000081062317,
276
+ "tpp_threshold_500_intended_diff_only": 0.4930000305175781,
277
+ "tpp_threshold_500_unintended_diff_only": 0.36400002241134644
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.008500009775161743,
281
+ "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0004999935626983643,
283
+ "tpp_threshold_5_total_metric": 0.15300005674362183,
284
  "tpp_threshold_5_intended_diff_only": 0.21800005435943604,
285
+ "tpp_threshold_5_unintended_diff_only": 0.06499999761581421,
286
+ "tpp_threshold_10_total_metric": 0.19025005400180817,
287
+ "tpp_threshold_10_intended_diff_only": 0.2640000581741333,
288
+ "tpp_threshold_10_unintended_diff_only": 0.07375000417232513,
289
+ "tpp_threshold_20_total_metric": 0.22925004363059998,
290
+ "tpp_threshold_20_intended_diff_only": 0.3060000538825989,
291
+ "tpp_threshold_20_unintended_diff_only": 0.0767500102519989,
292
+ "tpp_threshold_50_total_metric": 0.23700004816055298,
293
+ "tpp_threshold_50_intended_diff_only": 0.4020000696182251,
294
+ "tpp_threshold_50_unintended_diff_only": 0.16500002145767212,
295
+ "tpp_threshold_100_total_metric": 0.2055000364780426,
296
+ "tpp_threshold_100_intended_diff_only": 0.44800007343292236,
297
+ "tpp_threshold_100_unintended_diff_only": 0.24250003695487976,
298
+ "tpp_threshold_500_total_metric": 0.12250003218650818,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.35350003838539124
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.002249985933303833,
306
+ "tpp_threshold_2_intended_diff_only": 0.0059999823570251465,
307
+ "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135,
308
+ "tpp_threshold_5_total_metric": 0.004250004887580872,
309
+ "tpp_threshold_5_intended_diff_only": 0.004999995231628418,
310
+ "tpp_threshold_5_unintended_diff_only": 0.0007499903440475464,
311
+ "tpp_threshold_10_total_metric": 0.00449998676776886,
312
+ "tpp_threshold_10_intended_diff_only": 0.0059999823570251465,
313
+ "tpp_threshold_10_unintended_diff_only": 0.0014999955892562866,
314
+ "tpp_threshold_20_total_metric": 0.017999976873397827,
315
+ "tpp_threshold_20_intended_diff_only": 0.019999980926513672,
316
+ "tpp_threshold_20_unintended_diff_only": 0.0020000040531158447,
317
+ "tpp_threshold_50_total_metric": 0.06425002217292786,
318
+ "tpp_threshold_50_intended_diff_only": 0.0690000057220459,
319
+ "tpp_threshold_50_unintended_diff_only": 0.004749983549118042,
320
+ "tpp_threshold_100_total_metric": 0.13375000655651093,
321
+ "tpp_threshold_100_intended_diff_only": 0.14300000667572021,
322
+ "tpp_threshold_100_unintended_diff_only": 0.00925000011920929,
323
+ "tpp_threshold_500_total_metric": 0.40700003504753113,
324
+ "tpp_threshold_500_intended_diff_only": 0.4230000376701355,
325
+ "tpp_threshold_500_unintended_diff_only": 0.01600000262260437
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.012000054121017456,
329
+ "tpp_threshold_2_intended_diff_only": 0.010000050067901611,
330
+ "tpp_threshold_2_unintended_diff_only": -0.0020000040531158447,
331
+ "tpp_threshold_5_total_metric": 0.0025000572204589844,
332
+ "tpp_threshold_5_intended_diff_only": 0.010000050067901611,
333
+ "tpp_threshold_5_unintended_diff_only": 0.007499992847442627,
334
+ "tpp_threshold_10_total_metric": 0.04000005125999451,
335
+ "tpp_threshold_10_intended_diff_only": 0.04300004243850708,
336
+ "tpp_threshold_10_unintended_diff_only": 0.0029999911785125732,
337
+ "tpp_threshold_20_total_metric": 0.08475004136562347,
338
+ "tpp_threshold_20_intended_diff_only": 0.08900004625320435,
339
+ "tpp_threshold_20_unintended_diff_only": 0.004250004887580872,
340
+ "tpp_threshold_50_total_metric": 0.13425004482269287,
341
+ "tpp_threshold_50_intended_diff_only": 0.14500004053115845,
342
+ "tpp_threshold_50_unintended_diff_only": 0.010749995708465576,
343
+ "tpp_threshold_100_total_metric": 0.24800005555152893,
344
+ "tpp_threshold_100_intended_diff_only": 0.26500004529953003,
345
+ "tpp_threshold_100_unintended_diff_only": 0.0169999897480011,
346
+ "tpp_threshold_500_total_metric": 0.40800005197525024,
347
+ "tpp_threshold_500_intended_diff_only": 0.43800002336502075,
348
+ "tpp_threshold_500_unintended_diff_only": 0.029999971389770508
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.0029999762773513794,
352
+ "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
353
+ "tpp_threshold_2_unintended_diff_only": -0.0010000020265579224,
354
+ "tpp_threshold_5_total_metric": 0.016750037670135498,
355
+ "tpp_threshold_5_intended_diff_only": 0.012000024318695068,
356
+ "tpp_threshold_5_unintended_diff_only": -0.00475001335144043,
357
+ "tpp_threshold_10_total_metric": 0.019000038504600525,
358
+ "tpp_threshold_10_intended_diff_only": 0.017000019550323486,
359
+ "tpp_threshold_10_unintended_diff_only": -0.0020000189542770386,
360
+ "tpp_threshold_20_total_metric": 0.008249983191490173,
361
+ "tpp_threshold_20_intended_diff_only": 0.010999977588653564,
362
+ "tpp_threshold_20_unintended_diff_only": 0.002749994397163391,
363
+ "tpp_threshold_50_total_metric": 0.04150000214576721,
364
+ "tpp_threshold_50_intended_diff_only": 0.041999995708465576,
365
+ "tpp_threshold_50_unintended_diff_only": 0.0004999935626983643,
366
+ "tpp_threshold_100_total_metric": 0.10274998843669891,
367
+ "tpp_threshold_100_intended_diff_only": 0.11699998378753662,
368
+ "tpp_threshold_100_unintended_diff_only": 0.014249995350837708,
369
+ "tpp_threshold_500_total_metric": 0.35099999606609344,
370
+ "tpp_threshold_500_intended_diff_only": 0.38499999046325684,
371
+ "tpp_threshold_500_unintended_diff_only": 0.03399999439716339
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": -0.0012500584125518799,
375
  "tpp_threshold_2_intended_diff_only": -0.001000046730041504,
376
+ "tpp_threshold_2_unintended_diff_only": 0.000250011682510376,
377
+ "tpp_threshold_5_total_metric": 0.009249955415725708,
378
+ "tpp_threshold_5_intended_diff_only": 0.007999956607818604,
379
+ "tpp_threshold_5_unintended_diff_only": -0.0012499988079071045,
380
+ "tpp_threshold_10_total_metric": 0.004250004887580872,
381
+ "tpp_threshold_10_intended_diff_only": 0.009999990463256836,
382
+ "tpp_threshold_10_unintended_diff_only": 0.005749985575675964,
383
+ "tpp_threshold_20_total_metric": 0.02724999189376831,
384
+ "tpp_threshold_20_intended_diff_only": 0.02799999713897705,
385
+ "tpp_threshold_20_unintended_diff_only": 0.0007500052452087402,
386
+ "tpp_threshold_50_total_metric": 0.0849999487400055,
387
+ "tpp_threshold_50_intended_diff_only": 0.09199994802474976,
388
+ "tpp_threshold_50_unintended_diff_only": 0.006999999284744263,
389
+ "tpp_threshold_100_total_metric": 0.18324998021125793,
390
+ "tpp_threshold_100_intended_diff_only": 0.19499999284744263,
391
+ "tpp_threshold_100_unintended_diff_only": 0.011750012636184692,
392
+ "tpp_threshold_500_total_metric": 0.3662499785423279,
393
+ "tpp_threshold_500_intended_diff_only": 0.4089999794960022,
394
+ "tpp_threshold_500_unintended_diff_only": 0.042750000953674316
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.019999995827674866,
398
+ "tpp_threshold_2_intended_diff_only": 0.02399998903274536,
399
+ "tpp_threshold_2_unintended_diff_only": 0.003999993205070496,
400
+ "tpp_threshold_5_total_metric": 0.03899995982646942,
401
+ "tpp_threshold_5_intended_diff_only": 0.039999961853027344,
402
+ "tpp_threshold_5_unintended_diff_only": 0.0010000020265579224,
403
+ "tpp_threshold_10_total_metric": 0.07274995744228363,
404
+ "tpp_threshold_10_intended_diff_only": 0.0769999623298645,
405
+ "tpp_threshold_10_unintended_diff_only": 0.004250004887580872,
406
+ "tpp_threshold_20_total_metric": 0.11549998819828033,
407
+ "tpp_threshold_20_intended_diff_only": 0.125,
408
+ "tpp_threshold_20_unintended_diff_only": 0.009500011801719666,
409
+ "tpp_threshold_50_total_metric": 0.19974999129772186,
410
+ "tpp_threshold_50_intended_diff_only": 0.20899999141693115,
411
+ "tpp_threshold_50_unintended_diff_only": 0.00925000011920929,
412
+ "tpp_threshold_100_total_metric": 0.2787499874830246,
413
+ "tpp_threshold_100_intended_diff_only": 0.2929999828338623,
414
+ "tpp_threshold_100_unintended_diff_only": 0.014249995350837708,
415
+ "tpp_threshold_500_total_metric": 0.3382500261068344,
416
+ "tpp_threshold_500_intended_diff_only": 0.37300002574920654,
417
+ "tpp_threshold_500_unintended_diff_only": 0.03474999964237213
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-2441-tokens-10000000/scr/saebench_scr_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,8 +69,8 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "c60d67bf-9707-4e04-b483-16786886d228",
69
- "datetime_epoch_millis": 1776885285314,
70
  "eval_result_metrics": {
71
  "scr_metrics": {
72
  "scr_dir1_threshold_2": 0.20086747439820368,
@@ -78,18 +82,18 @@
78
  "scr_dir1_threshold_10": 0.2522650669526282,
79
  "scr_metric_threshold_10": 0.2776267778909501,
80
  "scr_dir2_threshold_10": 0.2736648780725306,
81
- "scr_dir1_threshold_20": null,
82
- "scr_metric_threshold_20": null,
83
- "scr_dir2_threshold_20": null,
84
- "scr_dir1_threshold_50": null,
85
- "scr_metric_threshold_50": null,
86
- "scr_dir2_threshold_50": null,
87
- "scr_dir1_threshold_100": null,
88
- "scr_metric_threshold_100": null,
89
- "scr_dir2_threshold_100": null,
90
- "scr_dir1_threshold_500": null,
91
- "scr_metric_threshold_500": null,
92
- "scr_dir2_threshold_500": null
93
  }
94
  },
95
  "eval_result_details": [
@@ -104,18 +108,18 @@
104
  "scr_dir1_threshold_10": 0.5256410746257707,
105
  "scr_metric_threshold_10": 0.1968911085011348,
106
  "scr_dir2_threshold_10": 0.1968911085011348,
107
- "scr_dir1_threshold_20": null,
108
- "scr_metric_threshold_20": null,
109
- "scr_dir2_threshold_20": null,
110
- "scr_dir1_threshold_50": null,
111
- "scr_metric_threshold_50": null,
112
- "scr_dir2_threshold_50": null,
113
- "scr_dir1_threshold_100": null,
114
- "scr_metric_threshold_100": null,
115
- "scr_dir2_threshold_100": null,
116
- "scr_dir1_threshold_500": null,
117
- "scr_metric_threshold_500": null,
118
- "scr_dir2_threshold_500": null
119
  },
120
  {
121
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
@@ -128,18 +132,18 @@
128
  "scr_dir1_threshold_10": 0.1439393152601855,
129
  "scr_metric_threshold_10": 0.321100954221084,
130
  "scr_dir2_threshold_10": 0.321100954221084,
131
- "scr_dir1_threshold_20": null,
132
- "scr_metric_threshold_20": null,
133
- "scr_dir2_threshold_20": null,
134
- "scr_dir1_threshold_50": null,
135
- "scr_metric_threshold_50": null,
136
- "scr_dir2_threshold_50": null,
137
- "scr_dir1_threshold_100": null,
138
- "scr_metric_threshold_100": null,
139
- "scr_dir2_threshold_100": null,
140
- "scr_dir1_threshold_500": null,
141
- "scr_metric_threshold_500": null,
142
- "scr_dir2_threshold_500": null
143
  },
144
  {
145
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
@@ -152,18 +156,18 @@
152
  "scr_dir1_threshold_10": 0.2656252619344312,
153
  "scr_metric_threshold_10": 0.13567842130982535,
154
  "scr_dir2_threshold_10": 0.13567842130982535,
155
- "scr_dir1_threshold_20": null,
156
- "scr_metric_threshold_20": null,
157
- "scr_dir2_threshold_20": null,
158
- "scr_dir1_threshold_50": null,
159
- "scr_metric_threshold_50": null,
160
- "scr_dir2_threshold_50": null,
161
- "scr_dir1_threshold_100": null,
162
- "scr_metric_threshold_100": null,
163
- "scr_dir2_threshold_100": null,
164
- "scr_dir1_threshold_500": null,
165
- "scr_metric_threshold_500": null,
166
- "scr_dir2_threshold_500": null
167
  },
168
  {
169
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
@@ -176,18 +180,18 @@
176
  "scr_dir1_threshold_10": 0.14765071675590696,
177
  "scr_metric_threshold_10": 0.19426747602431718,
178
  "scr_dir2_threshold_10": 0.19426747602431718,
179
- "scr_dir1_threshold_20": null,
180
- "scr_metric_threshold_20": null,
181
- "scr_dir2_threshold_20": null,
182
- "scr_dir1_threshold_50": null,
183
- "scr_metric_threshold_50": null,
184
- "scr_dir2_threshold_50": null,
185
- "scr_dir1_threshold_100": null,
186
- "scr_metric_threshold_100": null,
187
- "scr_dir2_threshold_100": null,
188
- "scr_dir1_threshold_500": null,
189
- "scr_metric_threshold_500": null,
190
- "scr_dir2_threshold_500": null
191
  },
192
  {
193
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
@@ -200,18 +204,18 @@
200
  "scr_dir1_threshold_10": 0.13157913724471565,
201
  "scr_metric_threshold_10": 0.567460267714699,
202
  "scr_dir2_threshold_10": 0.567460267714699,
203
- "scr_dir1_threshold_20": null,
204
- "scr_metric_threshold_20": null,
205
- "scr_dir2_threshold_20": null,
206
- "scr_dir1_threshold_50": null,
207
- "scr_metric_threshold_50": null,
208
- "scr_dir2_threshold_50": null,
209
- "scr_dir1_threshold_100": null,
210
- "scr_metric_threshold_100": null,
211
- "scr_dir2_threshold_100": null,
212
- "scr_dir1_threshold_500": null,
213
- "scr_metric_threshold_500": null,
214
- "scr_dir2_threshold_500": null
215
  },
216
  {
217
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
@@ -224,18 +228,18 @@
224
  "scr_dir1_threshold_10": 0.3060109503196829,
225
  "scr_metric_threshold_10": 0.2519083153249215,
226
  "scr_dir2_threshold_10": 0.2519083153249215,
227
- "scr_dir1_threshold_20": null,
228
- "scr_metric_threshold_20": null,
229
- "scr_dir2_threshold_20": null,
230
- "scr_dir1_threshold_50": null,
231
- "scr_metric_threshold_50": null,
232
- "scr_dir2_threshold_50": null,
233
- "scr_dir1_threshold_100": null,
234
- "scr_metric_threshold_100": null,
235
- "scr_dir2_threshold_100": null,
236
- "scr_dir1_threshold_500": null,
237
- "scr_metric_threshold_500": null,
238
- "scr_dir2_threshold_500": null
239
  },
240
  {
241
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
@@ -248,18 +252,18 @@
248
  "scr_dir1_threshold_10": 0.330232413721597,
249
  "scr_metric_threshold_10": 0.3862660142728834,
250
  "scr_dir2_threshold_10": 0.3862660142728834,
251
- "scr_dir1_threshold_20": null,
252
- "scr_metric_threshold_20": null,
253
- "scr_dir2_threshold_20": null,
254
- "scr_dir1_threshold_50": null,
255
- "scr_metric_threshold_50": null,
256
- "scr_dir2_threshold_50": null,
257
- "scr_dir1_threshold_100": null,
258
- "scr_metric_threshold_100": null,
259
- "scr_dir2_threshold_100": null,
260
- "scr_dir1_threshold_500": null,
261
- "scr_metric_threshold_500": null,
262
- "scr_dir2_threshold_500": null
263
  },
264
  {
265
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
@@ -272,18 +276,18 @@
272
  "scr_dir1_threshold_10": 0.16744166575873534,
273
  "scr_metric_threshold_10": 0.16744166575873534,
274
  "scr_dir2_threshold_10": 0.13574646721137937,
275
- "scr_dir1_threshold_20": null,
276
- "scr_metric_threshold_20": null,
277
- "scr_dir2_threshold_20": null,
278
- "scr_dir1_threshold_50": null,
279
- "scr_metric_threshold_50": null,
280
- "scr_dir2_threshold_50": null,
281
- "scr_dir1_threshold_100": null,
282
- "scr_metric_threshold_100": null,
283
- "scr_dir2_threshold_100": null,
284
- "scr_dir1_threshold_500": null,
285
- "scr_metric_threshold_500": null,
286
- "scr_dir2_threshold_500": null
287
  }
288
  ],
289
  "sae_bench_commit_hash": "Unknown",
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "40add3df-7a9a-44e4-94f1-06a20761cddb",
73
+ "datetime_epoch_millis": 1777316682993,
74
  "eval_result_metrics": {
75
  "scr_metrics": {
76
  "scr_dir1_threshold_2": 0.20086747439820368,
 
82
  "scr_dir1_threshold_10": 0.2522650669526282,
83
  "scr_metric_threshold_10": 0.2776267778909501,
84
  "scr_dir2_threshold_10": 0.2736648780725306,
85
+ "scr_dir1_threshold_20": 0.2373239502783135,
86
+ "scr_metric_threshold_20": 0.3074054764488834,
87
+ "scr_dir2_threshold_20": 0.2952566874534856,
88
+ "scr_dir1_threshold_50": 0.23638686186846655,
89
+ "scr_metric_threshold_50": 0.3400536817425774,
90
+ "scr_dir2_threshold_50": 0.3278259698209423,
91
+ "scr_dir1_threshold_100": -0.19920822841766905,
92
+ "scr_metric_threshold_100": 0.3790442844503187,
93
+ "scr_dir2_threshold_100": 0.36452255493089136,
94
+ "scr_dir1_threshold_500": 0.06468838438344704,
95
+ "scr_metric_threshold_500": 0.25111199801456385,
96
+ "scr_dir2_threshold_500": 0.22106621753024489
97
  }
98
  },
99
  "eval_result_details": [
 
108
  "scr_dir1_threshold_10": 0.5256410746257707,
109
  "scr_metric_threshold_10": 0.1968911085011348,
110
  "scr_dir2_threshold_10": 0.1968911085011348,
111
+ "scr_dir1_threshold_20": 0.48718003580863245,
112
+ "scr_metric_threshold_20": 0.16839375998107475,
113
+ "scr_dir2_threshold_20": 0.16839375998107475,
114
+ "scr_dir1_threshold_50": 0.48718003580863245,
115
+ "scr_metric_threshold_50": 0.23575132573996999,
116
+ "scr_dir2_threshold_50": 0.23575132573996999,
117
+ "scr_dir1_threshold_100": -1.730768907469913,
118
+ "scr_metric_threshold_100": 0.25906739431680575,
119
+ "scr_dir2_threshold_100": 0.25906739431680575,
120
+ "scr_dir1_threshold_500": -1.1153838806134386,
121
+ "scr_metric_threshold_500": 0.23834196571158212,
122
+ "scr_dir2_threshold_500": 0.23834196571158212
123
  },
124
  {
125
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
 
132
  "scr_dir1_threshold_10": 0.1439393152601855,
133
  "scr_metric_threshold_10": 0.321100954221084,
134
  "scr_dir2_threshold_10": 0.321100954221084,
135
+ "scr_dir1_threshold_20": 0.18181805866811646,
136
+ "scr_metric_threshold_20": 0.28440363460714707,
137
+ "scr_dir2_threshold_20": 0.28440363460714707,
138
+ "scr_dir1_threshold_50": 0.2045454853329709,
139
+ "scr_metric_threshold_50": 0.42813466638717496,
140
+ "scr_dir2_threshold_50": 0.42813466638717496,
141
+ "scr_dir1_threshold_100": 0.22727291199782534,
142
+ "scr_metric_threshold_100": 0.47094811479817356,
143
+ "scr_dir2_threshold_100": 0.47094811479817356,
144
+ "scr_dir1_threshold_500": 0.1439393152601855,
145
+ "scr_metric_threshold_500": 0.07645270362640467,
146
+ "scr_dir2_threshold_500": 0.07645270362640467
147
  },
148
  {
149
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
 
156
  "scr_dir1_threshold_10": 0.2656252619344312,
157
  "scr_metric_threshold_10": 0.13567842130982535,
158
  "scr_dir2_threshold_10": 0.13567842130982535,
159
+ "scr_dir1_threshold_20": 0.09374970896174313,
160
+ "scr_metric_threshold_20": 0.17336682175353543,
161
+ "scr_dir2_threshold_20": 0.17336682175353543,
162
+ "scr_dir1_threshold_50": -0.28125005820765137,
163
+ "scr_metric_threshold_50": 0.16331655168313458,
164
+ "scr_dir2_threshold_50": 0.16331655168313458,
165
+ "scr_dir1_threshold_100": -1.4374998835846973,
166
+ "scr_metric_threshold_100": 0.0050252099153958625,
167
+ "scr_dir2_threshold_100": 0.0050252099153958625,
168
+ "scr_dir1_threshold_500": 0.2968748544808716,
169
+ "scr_metric_threshold_500": 0.027638280133700087,
170
+ "scr_dir2_threshold_500": 0.027638280133700087
171
  },
172
  {
173
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
 
180
  "scr_dir1_threshold_10": 0.14765071675590696,
181
  "scr_metric_threshold_10": 0.19426747602431718,
182
  "scr_dir2_threshold_10": 0.19426747602431718,
183
+ "scr_dir1_threshold_20": 0.10738237582531437,
184
+ "scr_metric_threshold_20": 0.2675158872948338,
185
+ "scr_dir2_threshold_20": 0.2675158872948338,
186
+ "scr_dir1_threshold_50": 0.5704696966363376,
187
+ "scr_metric_threshold_50": 0.31847140286929915,
188
+ "scr_dir2_threshold_50": 0.31847140286929915,
189
+ "scr_dir1_threshold_100": 0.5704696966363376,
190
+ "scr_metric_threshold_100": 0.3630573840851048,
191
+ "scr_dir2_threshold_100": 0.3630573840851048,
192
+ "scr_dir1_threshold_500": 0.30201315702624804,
193
+ "scr_metric_threshold_500": 0.07006373900303832,
194
+ "scr_dir2_threshold_500": 0.07006373900303832
195
  },
196
  {
197
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
 
204
  "scr_dir1_threshold_10": 0.13157913724471565,
205
  "scr_metric_threshold_10": 0.567460267714699,
206
  "scr_dir2_threshold_10": 0.567460267714699,
207
+ "scr_dir1_threshold_20": 0.12105264313663487,
208
+ "scr_metric_threshold_20": 0.6269840124753451,
209
+ "scr_dir2_threshold_20": 0.6269840124753451,
210
+ "scr_dir1_threshold_50": -0.01052618039942014,
211
+ "scr_metric_threshold_50": 0.6825395548905486,
212
+ "scr_dir2_threshold_50": 0.6825395548905486,
213
+ "scr_dir1_threshold_100": -0.09473687842942387,
214
+ "scr_metric_threshold_100": 0.7341268949603097,
215
+ "scr_dir2_threshold_100": 0.7341268949603097,
216
+ "scr_dir1_threshold_500": 0.03684225881529179,
217
+ "scr_metric_threshold_500": 0.7539681432138584,
218
+ "scr_dir2_threshold_500": 0.7539681432138584
219
  },
220
  {
221
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
 
228
  "scr_dir1_threshold_10": 0.3060109503196829,
229
  "scr_metric_threshold_10": 0.2519083153249215,
230
  "scr_dir2_threshold_10": 0.2519083153249215,
231
+ "scr_dir1_threshold_20": 0.2841529787670631,
232
+ "scr_metric_threshold_20": 0.2671755204201627,
233
+ "scr_dir2_threshold_20": 0.2671755204201627,
234
+ "scr_dir1_threshold_50": 0.1857922696345225,
235
+ "scr_metric_threshold_50": 0.2748092367170948,
236
+ "scr_dir2_threshold_50": 0.2748092367170948,
237
+ "scr_dir1_threshold_100": 0.2622950072144432,
238
+ "scr_metric_threshold_100": 0.4618319872618971,
239
+ "scr_dir2_threshold_100": 0.4618319872618971,
240
+ "scr_dir1_threshold_500": 0.2950818016891243,
241
+ "scr_metric_threshold_500": 0.31297713570588614,
242
+ "scr_dir2_threshold_500": 0.31297713570588614
243
  },
244
  {
245
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
 
252
  "scr_dir1_threshold_10": 0.330232413721597,
253
  "scr_metric_threshold_10": 0.3862660142728834,
254
  "scr_dir2_threshold_10": 0.3862660142728834,
255
+ "scr_dir1_threshold_20": 0.3767442672816017,
256
+ "scr_metric_threshold_20": 0.4248926412815665,
257
+ "scr_dir2_threshold_20": 0.4248926412815665,
258
+ "scr_dir1_threshold_50": 0.46511631775314366,
259
+ "scr_metric_threshold_50": 0.3476393872642003,
260
+ "scr_dir2_threshold_50": 0.3476393872642003,
261
+ "scr_dir1_threshold_100": 0.330232413721597,
262
+ "scr_metric_threshold_100": 0.45922747769238426,
263
+ "scr_dir2_threshold_100": 0.45922747769238426,
264
+ "scr_dir1_threshold_500": 0.20465126852525933,
265
+ "scr_metric_threshold_500": 0.17596571683800658,
266
+ "scr_dir2_threshold_500": 0.17596571683800658
267
  },
268
  {
269
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
 
276
  "scr_dir1_threshold_10": 0.16744166575873534,
277
  "scr_metric_threshold_10": 0.16744166575873534,
278
  "scr_dir2_threshold_10": 0.13574646721137937,
279
+ "scr_dir1_threshold_20": 0.24651153377740184,
280
+ "scr_metric_threshold_20": 0.24651153377740184,
281
+ "scr_dir2_threshold_20": 0.1493212218142196,
282
+ "scr_dir1_threshold_50": 0.26976732838919676,
283
+ "scr_metric_threshold_50": 0.26976732838919676,
284
+ "scr_dir2_threshold_50": 0.17194563301611618,
285
+ "scr_dir1_threshold_100": 0.27906981257247854,
286
+ "scr_metric_threshold_100": 0.27906981257247854,
287
+ "scr_dir2_threshold_100": 0.16289597641705988,
288
+ "scr_dir1_threshold_500": 0.3534882998840343,
289
+ "scr_metric_threshold_500": 0.3534882998840343,
290
+ "scr_dir2_threshold_500": 0.1131220560094828
291
  }
292
  ],
293
  "sae_bench_commit_hash": "Unknown",
gemma-2-2b-btk/k-100/seed-0/snapshots/step-2441-tokens-10000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "b12321ac-7c3b-43f3-a124-23afb3858b0f",
69
- "datetime_epoch_millis": 1776885438722,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.008525006473064423,
73
- "tpp_threshold_2_intended_diff_only": 0.008600008487701417,
74
- "tpp_threshold_2_unintended_diff_only": 7.50020146369934e-05,
75
- "tpp_threshold_5_total_metric": 0.01054999977350235,
76
- "tpp_threshold_5_intended_diff_only": 0.012900006771087647,
77
- "tpp_threshold_5_unintended_diff_only": 0.0023500069975852965,
78
- "tpp_threshold_10_total_metric": 0.019874994456768033,
79
- "tpp_threshold_10_intended_diff_only": 0.02279999852180481,
80
- "tpp_threshold_10_unintended_diff_only": 0.002925004065036774,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.007350000739097595,
99
- "tpp_threshold_2_intended_diff_only": 0.006600010395050049,
100
- "tpp_threshold_2_unintended_diff_only": -0.0007499903440475464,
101
- "tpp_threshold_5_total_metric": 0.006949999928474426,
102
- "tpp_threshold_5_intended_diff_only": 0.00860002040863037,
103
- "tpp_threshold_5_unintended_diff_only": 0.0016500204801559448,
104
- "tpp_threshold_10_total_metric": 0.02139999270439148,
105
- "tpp_threshold_10_intended_diff_only": 0.023600006103515626,
106
- "tpp_threshold_10_unintended_diff_only": 0.0022000133991241454,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.00970001220703125,
123
- "tpp_threshold_2_intended_diff_only": 0.010600006580352784,
124
- "tpp_threshold_2_unintended_diff_only": 0.0008999943733215332,
125
- "tpp_threshold_5_total_metric": 0.014149999618530274,
126
- "tpp_threshold_5_intended_diff_only": 0.017199993133544922,
127
- "tpp_threshold_5_unintended_diff_only": 0.0030499935150146485,
128
- "tpp_threshold_10_total_metric": 0.01834999620914459,
129
- "tpp_threshold_10_intended_diff_only": 0.021999990940093993,
130
- "tpp_threshold_10_unintended_diff_only": 0.003649994730949402,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.0015000253915786743,
190
  "tpp_threshold_10_total_metric": 0.008749961853027344,
191
  "tpp_threshold_10_intended_diff_only": 0.010999977588653564,
192
- "tpp_threshold_10_unintended_diff_only": 0.0022500157356262207
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": -1.4901161193847656e-08,
196
- "tpp_threshold_2_intended_diff_only": 0.0,
197
- "tpp_threshold_2_unintended_diff_only": 1.4901161193847656e-08,
198
- "tpp_threshold_5_total_metric": 0.002250000834465027,
199
- "tpp_threshold_5_intended_diff_only": 0.003000020980834961,
200
- "tpp_threshold_5_unintended_diff_only": 0.0007500201463699341,
201
- "tpp_threshold_10_total_metric": -0.0022500157356262207,
202
- "tpp_threshold_10_intended_diff_only": 0.0,
203
- "tpp_threshold_10_unintended_diff_only": 0.0022500157356262207
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.015250012278556824,
207
- "tpp_threshold_2_intended_diff_only": 0.012000024318695068,
208
- "tpp_threshold_2_unintended_diff_only": -0.0032499879598617554,
209
- "tpp_threshold_5_total_metric": 0.012499988079071045,
210
- "tpp_threshold_5_intended_diff_only": 0.013000011444091797,
211
- "tpp_threshold_5_unintended_diff_only": 0.000500023365020752,
212
- "tpp_threshold_10_total_metric": 0.016500011086463928,
213
- "tpp_threshold_10_intended_diff_only": 0.017000019550323486,
214
- "tpp_threshold_10_unintended_diff_only": 0.0005000084638595581
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.0102500319480896,
218
  "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
219
- "tpp_threshold_2_unintended_diff_only": -0.0032500028610229492,
220
- "tpp_threshold_5_total_metric": 0.000250011682510376,
221
- "tpp_threshold_5_intended_diff_only": 0.003000020980834961,
222
- "tpp_threshold_5_unintended_diff_only": 0.002750009298324585,
223
- "tpp_threshold_10_total_metric": 0.006499990820884705,
224
- "tpp_threshold_10_intended_diff_only": 0.009000003337860107,
225
- "tpp_threshold_10_unintended_diff_only": 0.002500012516975403
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.006000027060508728,
229
- "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
230
- "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224,
231
- "tpp_threshold_5_total_metric": 0.008250012993812561,
232
- "tpp_threshold_5_intended_diff_only": 0.01100003719329834,
233
- "tpp_threshold_5_unintended_diff_only": 0.002750024199485779,
234
- "tpp_threshold_10_total_metric": 0.07750001549720764,
235
- "tpp_threshold_10_intended_diff_only": 0.08100003004074097,
236
- "tpp_threshold_10_unintended_diff_only": 0.003500014543533325
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.007500022649765015,
242
- "tpp_threshold_2_intended_diff_only": 0.003000020980834961,
243
- "tpp_threshold_2_unintended_diff_only": -0.004500001668930054,
244
- "tpp_threshold_5_total_metric": 0.00599999725818634,
245
  "tpp_threshold_5_intended_diff_only": 0.004999995231628418,
246
- "tpp_threshold_5_unintended_diff_only": -0.0010000020265579224,
247
- "tpp_threshold_10_total_metric": -0.003250017762184143,
248
- "tpp_threshold_10_intended_diff_only": 0.001999974250793457,
249
- "tpp_threshold_10_unintended_diff_only": 0.0052499920129776
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": -0.0004999488592147827,
253
- "tpp_threshold_2_intended_diff_only": 0.006000041961669922,
254
- "tpp_threshold_2_unintended_diff_only": 0.006499990820884705,
255
- "tpp_threshold_5_total_metric": 0.002750024199485779,
256
- "tpp_threshold_5_intended_diff_only": 0.009000003337860107,
257
- "tpp_threshold_5_unintended_diff_only": 0.006249979138374329,
258
- "tpp_threshold_10_total_metric": 0.010500013828277588,
259
- "tpp_threshold_10_intended_diff_only": 0.013000011444091797,
260
- "tpp_threshold_10_unintended_diff_only": 0.002499997615814209
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.018750011920928955,
264
- "tpp_threshold_2_intended_diff_only": 0.013999998569488525,
265
- "tpp_threshold_2_unintended_diff_only": -0.00475001335144043,
266
- "tpp_threshold_5_total_metric": 0.016000017523765564,
267
- "tpp_threshold_5_intended_diff_only": 0.018000006675720215,
268
- "tpp_threshold_5_unintended_diff_only": 0.001999989151954651,
269
- "tpp_threshold_10_total_metric": 0.013999998569488525,
270
  "tpp_threshold_10_intended_diff_only": 0.018999993801116943,
271
- "tpp_threshold_10_unintended_diff_only": 0.004999995231628418
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.0025000274181365967,
275
- "tpp_threshold_2_intended_diff_only": 0.0059999823570251465,
276
- "tpp_threshold_2_unintended_diff_only": 0.008500009775161743,
277
- "tpp_threshold_5_total_metric": 0.008000001311302185,
278
- "tpp_threshold_5_intended_diff_only": 0.009999990463256836,
279
- "tpp_threshold_5_unintended_diff_only": 0.001999989151954651,
280
- "tpp_threshold_10_total_metric": 0.01649998128414154,
281
- "tpp_threshold_10_intended_diff_only": 0.019999980926513672,
282
- "tpp_threshold_10_unintended_diff_only": 0.0034999996423721313
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.02525000274181366,
286
- "tpp_threshold_2_intended_diff_only": 0.02399998903274536,
287
- "tpp_threshold_2_unintended_diff_only": -0.0012500137090682983,
288
- "tpp_threshold_5_total_metric": 0.0379999577999115,
289
- "tpp_threshold_5_intended_diff_only": 0.04399996995925903,
290
- "tpp_threshold_5_unintended_diff_only": 0.006000012159347534,
291
- "tpp_threshold_10_total_metric": 0.05400000512599945,
292
- "tpp_threshold_10_intended_diff_only": 0.0559999942779541,
293
- "tpp_threshold_10_unintended_diff_only": 0.001999989151954651
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "a20cdede-f11b-47e8-a1c0-b653a0d431e7",
73
+ "datetime_epoch_millis": 1777317083537,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.007074996829032898,
77
+ "tpp_threshold_2_intended_diff_only": 0.00819999575614929,
78
+ "tpp_threshold_2_unintended_diff_only": 0.001124998927116394,
79
+ "tpp_threshold_5_total_metric": 0.009175001084804535,
80
+ "tpp_threshold_5_intended_diff_only": 0.01040000319480896,
81
+ "tpp_threshold_5_unintended_diff_only": 0.001225002110004425,
82
+ "tpp_threshold_10_total_metric": 0.022575002908706666,
83
+ "tpp_threshold_10_intended_diff_only": 0.02590000629425049,
84
+ "tpp_threshold_10_unintended_diff_only": 0.003325003385543823,
85
+ "tpp_threshold_20_total_metric": 0.039849989116191864,
86
+ "tpp_threshold_20_intended_diff_only": 0.04269999861717224,
87
+ "tpp_threshold_20_unintended_diff_only": 0.002850009500980377,
88
+ "tpp_threshold_50_total_metric": 0.07912500053644181,
89
+ "tpp_threshold_50_intended_diff_only": 0.08320000767707825,
90
+ "tpp_threshold_50_unintended_diff_only": 0.004075007140636444,
91
+ "tpp_threshold_100_total_metric": 0.14174999743700029,
92
+ "tpp_threshold_100_intended_diff_only": 0.14919999837875364,
93
+ "tpp_threshold_100_unintended_diff_only": 0.007450000941753387,
94
+ "tpp_threshold_500_total_metric": 0.31735001504421234,
95
+ "tpp_threshold_500_intended_diff_only": 0.3318000197410583,
96
+ "tpp_threshold_500_unintended_diff_only": 0.014450004696846007
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.0064999938011169435,
103
+ "tpp_threshold_2_intended_diff_only": 0.007400000095367431,
104
+ "tpp_threshold_2_unintended_diff_only": 0.0009000062942504883,
105
+ "tpp_threshold_5_total_metric": 0.006599998474121094,
106
+ "tpp_threshold_5_intended_diff_only": 0.007400012016296387,
107
+ "tpp_threshold_5_unintended_diff_only": 0.000800013542175293,
108
+ "tpp_threshold_10_total_metric": 0.02254999577999115,
109
+ "tpp_threshold_10_intended_diff_only": 0.024600005149841307,
110
+ "tpp_threshold_10_unintended_diff_only": 0.0020500093698501585,
111
+ "tpp_threshold_20_total_metric": 0.043299993872642516,
112
+ "tpp_threshold_20_intended_diff_only": 0.04560000896453857,
113
+ "tpp_threshold_20_unintended_diff_only": 0.002300015091896057,
114
+ "tpp_threshold_50_total_metric": 0.07815000116825103,
115
+ "tpp_threshold_50_intended_diff_only": 0.08140001296997071,
116
+ "tpp_threshold_50_unintended_diff_only": 0.0032500118017196656,
117
+ "tpp_threshold_100_total_metric": 0.16464999914169312,
118
+ "tpp_threshold_100_intended_diff_only": 0.1680000066757202,
119
+ "tpp_threshold_100_unintended_diff_only": 0.0033500075340270998,
120
+ "tpp_threshold_500_total_metric": 0.36230001747608187,
121
+ "tpp_threshold_500_intended_diff_only": 0.37480002641677856,
122
+ "tpp_threshold_500_unintended_diff_only": 0.012500008940696717
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.007649999856948852,
127
+ "tpp_threshold_2_intended_diff_only": 0.008999991416931152,
128
+ "tpp_threshold_2_unintended_diff_only": 0.0013499915599822997,
129
+ "tpp_threshold_5_total_metric": 0.011750003695487976,
130
+ "tpp_threshold_5_intended_diff_only": 0.013399994373321534,
131
+ "tpp_threshold_5_unintended_diff_only": 0.0016499906778335571,
132
+ "tpp_threshold_10_total_metric": 0.02260001003742218,
133
+ "tpp_threshold_10_intended_diff_only": 0.02720000743865967,
134
+ "tpp_threshold_10_unintended_diff_only": 0.0045999974012374874,
135
+ "tpp_threshold_20_total_metric": 0.03639998435974121,
136
+ "tpp_threshold_20_intended_diff_only": 0.03979998826980591,
137
+ "tpp_threshold_20_unintended_diff_only": 0.003400003910064697,
138
+ "tpp_threshold_50_total_metric": 0.08009999990463257,
139
+ "tpp_threshold_50_intended_diff_only": 0.08500000238418579,
140
+ "tpp_threshold_50_unintended_diff_only": 0.004900002479553222,
141
+ "tpp_threshold_100_total_metric": 0.11884999573230744,
142
+ "tpp_threshold_100_intended_diff_only": 0.1303999900817871,
143
+ "tpp_threshold_100_unintended_diff_only": 0.011549994349479675,
144
+ "tpp_threshold_500_total_metric": 0.2724000126123428,
145
+ "tpp_threshold_500_intended_diff_only": 0.28880001306533815,
146
+ "tpp_threshold_500_unintended_diff_only": 0.0164000004529953
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.0015000253915786743,
194
  "tpp_threshold_10_total_metric": 0.008749961853027344,
195
  "tpp_threshold_10_intended_diff_only": 0.010999977588653564,
196
+ "tpp_threshold_10_unintended_diff_only": 0.0022500157356262207,
197
+ "tpp_threshold_20_total_metric": 0.02774997055530548,
198
+ "tpp_threshold_20_intended_diff_only": 0.03299999237060547,
199
+ "tpp_threshold_20_unintended_diff_only": 0.005250021815299988,
200
+ "tpp_threshold_50_total_metric": 0.05949994921684265,
201
+ "tpp_threshold_50_intended_diff_only": 0.06699997186660767,
202
+ "tpp_threshold_50_unintended_diff_only": 0.007500022649765015,
203
+ "tpp_threshold_100_total_metric": 0.16950000822544098,
204
+ "tpp_threshold_100_intended_diff_only": 0.17500001192092896,
205
+ "tpp_threshold_100_unintended_diff_only": 0.005500003695487976,
206
+ "tpp_threshold_500_total_metric": 0.4129999876022339,
207
+ "tpp_threshold_500_intended_diff_only": 0.42100000381469727,
208
+ "tpp_threshold_500_unintended_diff_only": 0.008000016212463379
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.0052499920129776,
212
+ "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
213
+ "tpp_threshold_2_unintended_diff_only": -0.00024999678134918213,
214
+ "tpp_threshold_5_total_metric": -0.003750026226043701,
215
+ "tpp_threshold_5_intended_diff_only": -0.003000020980834961,
216
+ "tpp_threshold_5_unintended_diff_only": 0.0007500052452087402,
217
+ "tpp_threshold_10_total_metric": 0.005249977111816406,
218
+ "tpp_threshold_10_intended_diff_only": 0.004999995231628418,
219
+ "tpp_threshold_10_unintended_diff_only": -0.0002499818801879883,
220
+ "tpp_threshold_20_total_metric": 0.0352499783039093,
221
+ "tpp_threshold_20_intended_diff_only": 0.03700000047683716,
222
+ "tpp_threshold_20_unintended_diff_only": 0.0017500221729278564,
223
+ "tpp_threshold_50_total_metric": 0.09299996495246887,
224
+ "tpp_threshold_50_intended_diff_only": 0.09799998998641968,
225
+ "tpp_threshold_50_unintended_diff_only": 0.005000025033950806,
226
+ "tpp_threshold_100_total_metric": 0.22074998915195465,
227
+ "tpp_threshold_100_intended_diff_only": 0.22699999809265137,
228
+ "tpp_threshold_100_unintended_diff_only": 0.006250008940696716,
229
+ "tpp_threshold_500_total_metric": 0.4177500158548355,
230
+ "tpp_threshold_500_intended_diff_only": 0.4230000376701355,
231
+ "tpp_threshold_500_unintended_diff_only": 0.005250021815299988
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.006000012159347534,
235
+ "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
236
+ "tpp_threshold_2_unintended_diff_only": 0.0029999911785125732,
237
+ "tpp_threshold_5_total_metric": 0.009250029921531677,
238
+ "tpp_threshold_5_intended_diff_only": 0.01100003719329834,
239
+ "tpp_threshold_5_unintended_diff_only": 0.0017500072717666626,
240
+ "tpp_threshold_10_total_metric": 0.015499994158744812,
241
+ "tpp_threshold_10_intended_diff_only": 0.018000006675720215,
242
+ "tpp_threshold_10_unintended_diff_only": 0.002500012516975403,
243
+ "tpp_threshold_20_total_metric": 0.032500043511390686,
244
+ "tpp_threshold_20_intended_diff_only": 0.033000051975250244,
245
+ "tpp_threshold_20_unintended_diff_only": 0.0005000084638595581,
246
+ "tpp_threshold_50_total_metric": 0.05475005507469177,
247
+ "tpp_threshold_50_intended_diff_only": 0.05600005388259888,
248
+ "tpp_threshold_50_unintended_diff_only": 0.0012499988079071045,
249
+ "tpp_threshold_100_total_metric": 0.09600000083446503,
250
+ "tpp_threshold_100_intended_diff_only": 0.09600001573562622,
251
+ "tpp_threshold_100_unintended_diff_only": 1.4901161193847656e-08,
252
+ "tpp_threshold_500_total_metric": 0.31650005280971527,
253
+ "tpp_threshold_500_intended_diff_only": 0.35200005769729614,
254
+ "tpp_threshold_500_unintended_diff_only": 0.03550000488758087
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.007750019431114197,
258
  "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0007499903440475464,
260
+ "tpp_threshold_5_total_metric": 0.006749987602233887,
261
+ "tpp_threshold_5_intended_diff_only": 0.0040000081062316895,
262
+ "tpp_threshold_5_unintended_diff_only": -0.0027499794960021973,
263
+ "tpp_threshold_10_total_metric": 0.008999988436698914,
264
+ "tpp_threshold_10_intended_diff_only": 0.009999990463256836,
265
+ "tpp_threshold_10_unintended_diff_only": 0.0010000020265579224,
266
+ "tpp_threshold_20_total_metric": 0.01124998927116394,
267
+ "tpp_threshold_20_intended_diff_only": 0.009999990463256836,
268
+ "tpp_threshold_20_unintended_diff_only": -0.0012499988079071045,
269
+ "tpp_threshold_50_total_metric": 0.01600000262260437,
270
+ "tpp_threshold_50_intended_diff_only": 0.013999998569488525,
271
+ "tpp_threshold_50_unintended_diff_only": -0.0020000040531158447,
272
+ "tpp_threshold_100_total_metric": 0.01774999499320984,
273
+ "tpp_threshold_100_intended_diff_only": 0.018999993801116943,
274
+ "tpp_threshold_100_unintended_diff_only": 0.0012499988079071045,
275
+ "tpp_threshold_500_total_metric": 0.20600000023841858,
276
+ "tpp_threshold_500_intended_diff_only": 0.20800000429153442,
277
+ "tpp_threshold_500_unintended_diff_only": 0.0020000040531158447
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.008249998092651367,
281
+ "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0007500052452087402,
283
+ "tpp_threshold_5_total_metric": 0.009250015020370483,
284
+ "tpp_threshold_5_intended_diff_only": 0.012000024318695068,
285
+ "tpp_threshold_5_unintended_diff_only": 0.002750009298324585,
286
+ "tpp_threshold_10_total_metric": 0.07425005733966827,
287
+ "tpp_threshold_10_intended_diff_only": 0.07900005578994751,
288
+ "tpp_threshold_10_unintended_diff_only": 0.004749998450279236,
289
+ "tpp_threshold_20_total_metric": 0.10974998772144318,
290
+ "tpp_threshold_20_intended_diff_only": 0.11500000953674316,
291
+ "tpp_threshold_20_unintended_diff_only": 0.005250021815299988,
292
+ "tpp_threshold_50_total_metric": 0.16750003397464752,
293
+ "tpp_threshold_50_intended_diff_only": 0.17200005054473877,
294
+ "tpp_threshold_50_unintended_diff_only": 0.0045000165700912476,
295
+ "tpp_threshold_100_total_metric": 0.3192500025033951,
296
+ "tpp_threshold_100_intended_diff_only": 0.3230000138282776,
297
+ "tpp_threshold_100_unintended_diff_only": 0.0037500113248825073,
298
+ "tpp_threshold_500_total_metric": 0.458250030875206,
299
+ "tpp_threshold_500_intended_diff_only": 0.4700000286102295,
300
+ "tpp_threshold_500_unintended_diff_only": 0.011749997735023499
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.0020000040531158447,
306
+ "tpp_threshold_2_intended_diff_only": 0.0059999823570251465,
307
+ "tpp_threshold_2_unintended_diff_only": 0.003999978303909302,
308
+ "tpp_threshold_5_total_metric": 0.0007500052452087402,
309
  "tpp_threshold_5_intended_diff_only": 0.004999995231628418,
310
+ "tpp_threshold_5_unintended_diff_only": 0.004249989986419678,
311
+ "tpp_threshold_10_total_metric": -0.0007499754428863525,
312
+ "tpp_threshold_10_intended_diff_only": 0.003000020980834961,
313
+ "tpp_threshold_10_unintended_diff_only": 0.0037499964237213135,
314
+ "tpp_threshold_20_total_metric": 0.007250010967254639,
315
+ "tpp_threshold_20_intended_diff_only": 0.009000003337860107,
316
+ "tpp_threshold_20_unintended_diff_only": 0.0017499923706054688,
317
+ "tpp_threshold_50_total_metric": 0.027499988675117493,
318
+ "tpp_threshold_50_intended_diff_only": 0.02399998903274536,
319
+ "tpp_threshold_50_unintended_diff_only": -0.0034999996423721313,
320
+ "tpp_threshold_100_total_metric": 0.051249995827674866,
321
+ "tpp_threshold_100_intended_diff_only": 0.0559999942779541,
322
+ "tpp_threshold_100_unintended_diff_only": 0.004749998450279236,
323
+ "tpp_threshold_500_total_metric": 0.2067500203847885,
324
+ "tpp_threshold_500_intended_diff_only": 0.20600003004074097,
325
+ "tpp_threshold_500_unintended_diff_only": -0.0007499903440475464
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.00850003957748413,
329
+ "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
330
+ "tpp_threshold_2_unintended_diff_only": -0.0015000104904174805,
331
+ "tpp_threshold_5_total_metric": -0.0009999722242355347,
332
+ "tpp_threshold_5_intended_diff_only": 0.0040000081062316895,
333
+ "tpp_threshold_5_unintended_diff_only": 0.004999980330467224,
334
+ "tpp_threshold_10_total_metric": 0.013750061392784119,
335
+ "tpp_threshold_10_intended_diff_only": 0.01900005340576172,
336
+ "tpp_threshold_10_unintended_diff_only": 0.0052499920129776,
337
+ "tpp_threshold_20_total_metric": 0.025499999523162842,
338
+ "tpp_threshold_20_intended_diff_only": 0.02799999713897705,
339
+ "tpp_threshold_20_unintended_diff_only": 0.002499997615814209,
340
+ "tpp_threshold_50_total_metric": 0.06825001537799835,
341
+ "tpp_threshold_50_intended_diff_only": 0.07700002193450928,
342
+ "tpp_threshold_50_unintended_diff_only": 0.008750006556510925,
343
+ "tpp_threshold_100_total_metric": 0.10425005853176117,
344
+ "tpp_threshold_100_intended_diff_only": 0.12200003862380981,
345
+ "tpp_threshold_100_unintended_diff_only": 0.017749980092048645,
346
+ "tpp_threshold_500_total_metric": 0.25950005650520325,
347
+ "tpp_threshold_500_intended_diff_only": 0.27500003576278687,
348
+ "tpp_threshold_500_unintended_diff_only": 0.015499979257583618
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.009249985218048096,
352
+ "tpp_threshold_2_intended_diff_only": 0.010999977588653564,
353
+ "tpp_threshold_2_unintended_diff_only": 0.0017499923706054688,
354
+ "tpp_threshold_5_total_metric": 0.006749987602233887,
355
+ "tpp_threshold_5_intended_diff_only": 0.0059999823570251465,
356
+ "tpp_threshold_5_unintended_diff_only": -0.0007500052452087402,
357
+ "tpp_threshold_10_total_metric": 0.016749978065490723,
358
  "tpp_threshold_10_intended_diff_only": 0.018999993801116943,
359
+ "tpp_threshold_10_unintended_diff_only": 0.0022500157356262207,
360
+ "tpp_threshold_20_total_metric": 0.010499969124794006,
361
+ "tpp_threshold_20_intended_diff_only": 0.014999985694885254,
362
+ "tpp_threshold_20_unintended_diff_only": 0.0045000165700912476,
363
+ "tpp_threshold_50_total_metric": 0.054000020027160645,
364
+ "tpp_threshold_50_intended_diff_only": 0.054000020027160645,
365
+ "tpp_threshold_50_unintended_diff_only": 0.0,
366
+ "tpp_threshold_100_total_metric": 0.08474999666213989,
367
+ "tpp_threshold_100_intended_diff_only": 0.09399998188018799,
368
+ "tpp_threshold_100_unintended_diff_only": 0.009249985218048096,
369
+ "tpp_threshold_500_total_metric": 0.2877500206232071,
370
+ "tpp_threshold_500_intended_diff_only": 0.30900001525878906,
371
+ "tpp_threshold_500_unintended_diff_only": 0.02124999463558197
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.005249977111816406,
375
+ "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
376
+ "tpp_threshold_2_unintended_diff_only": -0.0002499818801879883,
377
+ "tpp_threshold_5_total_metric": 0.013999998569488525,
378
+ "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
379
+ "tpp_threshold_5_unintended_diff_only": 0.0,
380
+ "tpp_threshold_10_total_metric": 0.017499983310699463,
381
+ "tpp_threshold_10_intended_diff_only": 0.02499997615814209,
382
+ "tpp_threshold_10_unintended_diff_only": 0.007499992847442627,
383
+ "tpp_threshold_20_total_metric": 0.027249976992607117,
384
+ "tpp_threshold_20_intended_diff_only": 0.02899998426437378,
385
+ "tpp_threshold_20_unintended_diff_only": 0.0017500072717666626,
386
+ "tpp_threshold_50_total_metric": 0.037249982357025146,
387
+ "tpp_threshold_50_intended_diff_only": 0.042999982833862305,
388
+ "tpp_threshold_50_unintended_diff_only": 0.005750000476837158,
389
+ "tpp_threshold_100_total_metric": 0.088749960064888,
390
+ "tpp_threshold_100_intended_diff_only": 0.09999996423721313,
391
+ "tpp_threshold_100_unintended_diff_only": 0.011250004172325134,
392
+ "tpp_threshold_500_total_metric": 0.26224999129772186,
393
+ "tpp_threshold_500_intended_diff_only": 0.28200000524520874,
394
+ "tpp_threshold_500_unintended_diff_only": 0.019750013947486877
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.013249993324279785,
398
+ "tpp_threshold_2_intended_diff_only": 0.015999972820281982,
399
+ "tpp_threshold_2_unintended_diff_only": 0.0027499794960021973,
400
+ "tpp_threshold_5_total_metric": 0.03824999928474426,
401
+ "tpp_threshold_5_intended_diff_only": 0.03799998760223389,
402
+ "tpp_threshold_5_unintended_diff_only": -0.000250011682510376,
403
+ "tpp_threshold_10_total_metric": 0.06575000286102295,
404
+ "tpp_threshold_10_intended_diff_only": 0.06999999284744263,
405
+ "tpp_threshold_10_unintended_diff_only": 0.004249989986419678,
406
+ "tpp_threshold_20_total_metric": 0.11149996519088745,
407
+ "tpp_threshold_20_intended_diff_only": 0.11799997091293335,
408
+ "tpp_threshold_20_unintended_diff_only": 0.0065000057220458984,
409
+ "tpp_threshold_50_total_metric": 0.2134999930858612,
410
+ "tpp_threshold_50_intended_diff_only": 0.22699999809265137,
411
+ "tpp_threshold_50_unintended_diff_only": 0.013500005006790161,
412
+ "tpp_threshold_100_total_metric": 0.26524996757507324,
413
+ "tpp_threshold_100_intended_diff_only": 0.2799999713897705,
414
+ "tpp_threshold_100_unintended_diff_only": 0.014750003814697266,
415
+ "tpp_threshold_500_total_metric": 0.34574997425079346,
416
+ "tpp_threshold_500_intended_diff_only": 0.37199997901916504,
417
+ "tpp_threshold_500_unintended_diff_only": 0.026250004768371582
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-24414-tokens-100000000/scr/saebench_scr_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,8 +69,8 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "6983652a-37fc-44b7-b030-946aed57cff6",
69
- "datetime_epoch_millis": 1776885496694,
70
  "eval_result_metrics": {
71
  "scr_metrics": {
72
  "scr_dir1_threshold_2": 0.21195278907653772,
@@ -78,18 +82,18 @@
78
  "scr_dir1_threshold_10": 0.2732106130356535,
79
  "scr_metric_threshold_10": 0.2643277530959982,
80
  "scr_dir2_threshold_10": 0.27417727552003496,
81
- "scr_dir1_threshold_20": null,
82
- "scr_metric_threshold_20": null,
83
- "scr_dir2_threshold_20": null,
84
- "scr_dir1_threshold_50": null,
85
- "scr_metric_threshold_50": null,
86
- "scr_dir2_threshold_50": null,
87
- "scr_dir1_threshold_100": null,
88
- "scr_metric_threshold_100": null,
89
- "scr_dir2_threshold_100": null,
90
- "scr_dir1_threshold_500": null,
91
- "scr_metric_threshold_500": null,
92
- "scr_dir2_threshold_500": null
93
  }
94
  },
95
  "eval_result_details": [
@@ -104,18 +108,18 @@
104
  "scr_dir1_threshold_10": 0.6153850268564743,
105
  "scr_metric_threshold_10": 0.12176162282740319,
106
  "scr_dir2_threshold_10": 0.12176162282740319,
107
- "scr_dir1_threshold_20": null,
108
- "scr_metric_threshold_20": null,
109
- "scr_dir2_threshold_20": null,
110
- "scr_dir1_threshold_50": null,
111
- "scr_metric_threshold_50": null,
112
- "scr_dir2_threshold_50": null,
113
- "scr_dir1_threshold_100": null,
114
- "scr_metric_threshold_100": null,
115
- "scr_dir2_threshold_100": null,
116
- "scr_dir1_threshold_500": null,
117
- "scr_metric_threshold_500": null,
118
- "scr_dir2_threshold_500": null
119
  },
120
  {
121
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
@@ -128,18 +132,18 @@
128
  "scr_dir1_threshold_10": 0.32575782547854165,
129
  "scr_metric_threshold_10": 0.4128439798401429,
130
  "scr_dir2_threshold_10": 0.4128439798401429,
131
- "scr_dir1_threshold_20": null,
132
- "scr_metric_threshold_20": null,
133
- "scr_dir2_threshold_20": null,
134
- "scr_dir1_threshold_50": null,
135
- "scr_metric_threshold_50": null,
136
- "scr_dir2_threshold_50": null,
137
- "scr_dir1_threshold_100": null,
138
- "scr_metric_threshold_100": null,
139
- "scr_dir2_threshold_100": null,
140
- "scr_dir1_threshold_500": null,
141
- "scr_metric_threshold_500": null,
142
- "scr_dir2_threshold_500": null
143
  },
144
  {
145
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
@@ -152,18 +156,18 @@
152
  "scr_dir1_threshold_10": 0.5156247962732202,
153
  "scr_metric_threshold_10": 0.10301508102112025,
154
  "scr_dir2_threshold_10": 0.10301508102112025,
155
- "scr_dir1_threshold_20": null,
156
- "scr_metric_threshold_20": null,
157
- "scr_dir2_threshold_20": null,
158
- "scr_dir1_threshold_50": null,
159
- "scr_metric_threshold_50": null,
160
- "scr_dir2_threshold_50": null,
161
- "scr_dir1_threshold_100": null,
162
- "scr_metric_threshold_100": null,
163
- "scr_dir2_threshold_100": null,
164
- "scr_dir1_threshold_500": null,
165
- "scr_metric_threshold_500": null,
166
- "scr_dir2_threshold_500": null
167
  },
168
  {
169
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
@@ -176,18 +180,18 @@
176
  "scr_dir1_threshold_10": 0.22147647516506289,
177
  "scr_metric_threshold_10": 0.31847140286929915,
178
  "scr_dir2_threshold_10": 0.31847140286929915,
179
- "scr_dir1_threshold_20": null,
180
- "scr_metric_threshold_20": null,
181
- "scr_dir2_threshold_20": null,
182
- "scr_dir1_threshold_50": null,
183
- "scr_metric_threshold_50": null,
184
- "scr_dir2_threshold_50": null,
185
- "scr_dir1_threshold_100": null,
186
- "scr_metric_threshold_100": null,
187
- "scr_dir2_threshold_100": null,
188
- "scr_dir1_threshold_500": null,
189
- "scr_metric_threshold_500": null,
190
- "scr_dir2_threshold_500": null
191
  },
192
  {
193
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
@@ -200,18 +204,18 @@
200
  "scr_dir1_threshold_10": 0.08421069803000372,
201
  "scr_metric_threshold_10": 0.4999998817368318,
202
  "scr_dir2_threshold_10": 0.4999998817368318,
203
- "scr_dir1_threshold_20": null,
204
- "scr_metric_threshold_20": null,
205
- "scr_dir2_threshold_20": null,
206
- "scr_dir1_threshold_50": null,
207
- "scr_metric_threshold_50": null,
208
- "scr_dir2_threshold_50": null,
209
- "scr_dir1_threshold_100": null,
210
- "scr_metric_threshold_100": null,
211
- "scr_dir2_threshold_100": null,
212
- "scr_dir1_threshold_500": null,
213
- "scr_metric_threshold_500": null,
214
- "scr_dir2_threshold_500": null
215
  },
216
  {
217
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
@@ -224,18 +228,18 @@
224
  "scr_dir1_threshold_10": 0.21857906410920355,
225
  "scr_metric_threshold_10": 0.11450392446499717,
226
  "scr_dir2_threshold_10": 0.11450392446499717,
227
- "scr_dir1_threshold_20": null,
228
- "scr_metric_threshold_20": null,
229
- "scr_dir2_threshold_20": null,
230
- "scr_dir1_threshold_50": null,
231
- "scr_metric_threshold_50": null,
232
- "scr_dir2_threshold_50": null,
233
- "scr_dir1_threshold_100": null,
234
- "scr_metric_threshold_100": null,
235
- "scr_dir2_threshold_100": null,
236
- "scr_dir1_threshold_500": null,
237
- "scr_metric_threshold_500": null,
238
- "scr_dir2_threshold_500": null
239
  },
240
  {
241
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
@@ -248,18 +252,18 @@
248
  "scr_dir1_threshold_10": 0.10697673644937127,
249
  "scr_metric_threshold_10": 0.4463518500848407,
250
  "scr_dir2_threshold_10": 0.4463518500848407,
251
- "scr_dir1_threshold_20": null,
252
- "scr_metric_threshold_20": null,
253
- "scr_dir2_threshold_20": null,
254
- "scr_dir1_threshold_50": null,
255
- "scr_metric_threshold_50": null,
256
- "scr_dir2_threshold_50": null,
257
- "scr_dir1_threshold_100": null,
258
- "scr_metric_threshold_100": null,
259
- "scr_dir2_threshold_100": null,
260
- "scr_dir1_threshold_500": null,
261
- "scr_metric_threshold_500": null,
262
- "scr_dir2_threshold_500": null
263
  },
264
  {
265
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
@@ -272,18 +276,18 @@
272
  "scr_dir1_threshold_10": 0.09767428192335063,
273
  "scr_metric_threshold_10": 0.09767428192335063,
274
  "scr_dir2_threshold_10": 0.17647046131564434,
275
- "scr_dir1_threshold_20": null,
276
- "scr_metric_threshold_20": null,
277
- "scr_dir2_threshold_20": null,
278
- "scr_dir1_threshold_50": null,
279
- "scr_metric_threshold_50": null,
280
- "scr_dir2_threshold_50": null,
281
- "scr_dir1_threshold_100": null,
282
- "scr_metric_threshold_100": null,
283
- "scr_dir2_threshold_100": null,
284
- "scr_dir1_threshold_500": null,
285
- "scr_metric_threshold_500": null,
286
- "scr_dir2_threshold_500": null
287
  }
288
  ],
289
  "sae_bench_commit_hash": "Unknown",
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "c9a879e4-cf40-4841-b84b-c5f618d46953",
73
+ "datetime_epoch_millis": 1777316667993,
74
  "eval_result_metrics": {
75
  "scr_metrics": {
76
  "scr_dir1_threshold_2": 0.21195278907653772,
 
82
  "scr_dir1_threshold_10": 0.2732106130356535,
83
  "scr_metric_threshold_10": 0.2643277530959982,
84
  "scr_dir2_threshold_10": 0.27417727552003496,
85
+ "scr_dir1_threshold_20": 0.254990915046506,
86
+ "scr_metric_threshold_20": 0.3350846355160032,
87
+ "scr_dir2_threshold_20": 0.3471492516707594,
88
+ "scr_dir1_threshold_50": 0.1993347158036075,
89
+ "scr_metric_threshold_50": 0.41213895336215817,
90
+ "scr_dir2_threshold_50": 0.41827912914356763,
91
+ "scr_dir1_threshold_100": 0.23447883885121396,
92
+ "scr_metric_threshold_100": 0.43812244114237103,
93
+ "scr_dir2_threshold_100": 0.45387801191240573,
94
+ "scr_dir1_threshold_500": 0.008625756773020434,
95
+ "scr_metric_threshold_500": 0.330668632184563,
96
+ "scr_dir2_threshold_500": 0.34271482165810296
97
  }
98
  },
99
  "eval_result_details": [
 
108
  "scr_dir1_threshold_10": 0.6153850268564743,
109
  "scr_metric_threshold_10": 0.12176162282740319,
110
  "scr_dir2_threshold_10": 0.12176162282740319,
111
+ "scr_dir1_threshold_20": 0.6153850268564743,
112
+ "scr_metric_threshold_20": 0.21761653710635845,
113
+ "scr_dir2_threshold_20": 0.21761653710635845,
114
+ "scr_dir1_threshold_50": 0.5897435701496917,
115
+ "scr_metric_threshold_50": 0.26683931423164214,
116
+ "scr_dir2_threshold_50": 0.26683931423164214,
117
+ "scr_dir1_threshold_100": 0.5384614208981502,
118
+ "scr_metric_threshold_100": 0.27461138856264183,
119
+ "scr_dir2_threshold_100": 0.27461138856264183,
120
+ "scr_dir1_threshold_500": 0.24359039998532808,
121
+ "scr_metric_threshold_500": 0.28497410286525365,
122
+ "scr_dir2_threshold_500": 0.28497410286525365
123
  },
124
  {
125
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
 
132
  "scr_dir1_threshold_10": 0.32575782547854165,
133
  "scr_metric_threshold_10": 0.4128439798401429,
134
  "scr_dir2_threshold_10": 0.4128439798401429,
135
+ "scr_dir1_threshold_20": 0.3484848005931564,
136
+ "scr_metric_threshold_20": 0.47706424359523525,
137
+ "scr_dir2_threshold_20": 0.47706424359523525,
138
+ "scr_dir1_threshold_50": 0.2045454853329709,
139
+ "scr_metric_threshold_50": 0.5412845073503276,
140
+ "scr_dir2_threshold_50": 0.5412845073503276,
141
+ "scr_dir1_threshold_100": 0.18939416858989438,
142
+ "scr_metric_threshold_100": 0.7064219899200714,
143
+ "scr_dir2_threshold_100": 0.7064219899200714,
144
+ "scr_dir1_threshold_500": -0.09090880355893836,
145
+ "scr_metric_threshold_500": 0.27522944141155453,
146
+ "scr_dir2_threshold_500": 0.27522944141155453
147
  },
148
  {
149
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
 
156
  "scr_dir1_threshold_10": 0.5156247962732202,
157
  "scr_metric_threshold_10": 0.10301508102112025,
158
  "scr_dir2_threshold_10": 0.10301508102112025,
159
+ "scr_dir1_threshold_20": 0.42187508731147705,
160
+ "scr_metric_threshold_20": 0.15829149152812957,
161
+ "scr_dir2_threshold_20": 0.15829149152812957,
162
+ "scr_dir1_threshold_50": 0.14062502910382568,
163
+ "scr_metric_threshold_50": 0.24874377240134646,
164
+ "scr_dir2_threshold_50": 0.24874377240134646,
165
+ "scr_dir1_threshold_100": 0.3593749708961743,
166
+ "scr_metric_threshold_100": 0.04522614043660845,
167
+ "scr_dir2_threshold_100": 0.04522614043660845,
168
+ "scr_dir1_threshold_500": -0.42187508731147705,
169
+ "scr_metric_threshold_500": 0.03517587036620758,
170
+ "scr_dir2_threshold_500": 0.03517587036620758
171
  },
172
  {
173
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
 
180
  "scr_dir1_threshold_10": 0.22147647516506289,
181
  "scr_metric_threshold_10": 0.31847140286929915,
182
  "scr_dir2_threshold_10": 0.31847140286929915,
183
+ "scr_dir1_threshold_20": 0.053690987897055974,
184
+ "scr_metric_threshold_20": 0.4267515887294834,
185
+ "scr_dir2_threshold_20": 0.4267515887294834,
186
+ "scr_dir1_threshold_50": 0.03355701744736089,
187
+ "scr_metric_threshold_50": 0.5541401878419436,
188
+ "scr_dir2_threshold_50": 0.5541401878419436,
189
+ "scr_dir1_threshold_100": 0.25503349261242375,
190
+ "scr_metric_threshold_100": 0.6369428057385983,
191
+ "scr_dir2_threshold_100": 0.6369428057385983,
192
+ "scr_dir1_threshold_500": -0.38926196243306976,
193
+ "scr_metric_threshold_500": 0.14649682254103324,
194
+ "scr_dir2_threshold_500": 0.14649682254103324
195
  },
196
  {
197
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
 
204
  "scr_dir1_threshold_10": 0.08421069803000372,
205
  "scr_metric_threshold_10": 0.4999998817368318,
206
  "scr_dir2_threshold_10": 0.4999998817368318,
207
+ "scr_dir1_threshold_20": 0.08947378822971379,
208
+ "scr_metric_threshold_20": 0.5992063595309115,
209
+ "scr_dir2_threshold_20": 0.5992063595309115,
210
+ "scr_dir1_threshold_50": -0.12631573333634494,
211
+ "scr_metric_threshold_50": 0.6865079937623274,
212
+ "scr_dir2_threshold_50": 0.6865079937623274,
213
+ "scr_dir1_threshold_100": -0.12105264313663487,
214
+ "scr_metric_threshold_100": 0.6904761961077699,
215
+ "scr_dir2_threshold_100": 0.6904761961077699,
216
+ "scr_dir1_threshold_500": -0.06842111372221286,
217
+ "scr_metric_threshold_500": 0.6190476077844602,
218
+ "scr_dir2_threshold_500": 0.6190476077844602
219
  },
220
  {
221
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
 
228
  "scr_dir1_threshold_10": 0.21857906410920355,
229
  "scr_metric_threshold_10": 0.11450392446499717,
230
  "scr_dir2_threshold_10": 0.11450392446499717,
231
+ "scr_dir1_threshold_20": 0.245901447122854,
232
+ "scr_metric_threshold_20": 0.17938926174649328,
233
+ "scr_dir2_threshold_20": 0.17938926174649328,
234
+ "scr_dir1_threshold_50": 0.3060109503196829,
235
+ "scr_metric_threshold_50": 0.25954203162185363,
236
+ "scr_dir2_threshold_50": 0.25954203162185363,
237
+ "scr_dir1_threshold_100": 0.2732241558450018,
238
+ "scr_metric_threshold_100": 0.38549618928431434,
239
+ "scr_dir2_threshold_100": 0.38549618928431434,
240
+ "scr_dir1_threshold_500": 0.4098360709132541,
241
+ "scr_metric_threshold_500": 0.47709919235713827,
242
+ "scr_dir2_threshold_500": 0.47709919235713827
243
  },
244
  {
245
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
 
252
  "scr_dir1_threshold_10": 0.10697673644937127,
253
  "scr_metric_threshold_10": 0.4463518500848407,
254
  "scr_dir2_threshold_10": 0.4463518500848407,
255
+ "scr_dir1_threshold_20": 0.15348831277851305,
256
+ "scr_metric_threshold_20": 0.5107297323086109,
257
+ "scr_dir2_threshold_20": 0.5107297323086109,
258
+ "scr_dir1_threshold_50": 0.25581394704114274,
259
+ "scr_metric_threshold_50": 0.549356359317294,
260
+ "scr_dir2_threshold_50": 0.549356359317294,
261
+ "scr_dir1_threshold_100": 0.19069768473417165,
262
+ "scr_metric_threshold_100": 0.5751073587184335,
263
+ "scr_dir2_threshold_100": 0.5751073587184335,
264
+ "scr_dir1_threshold_500": 0.10232563426262967,
265
+ "scr_metric_threshold_500": 0.5236051041022068,
266
+ "scr_dir2_threshold_500": 0.5236051041022068
267
  },
268
  {
269
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
 
276
  "scr_dir1_threshold_10": 0.09767428192335063,
277
  "scr_metric_threshold_10": 0.09767428192335063,
278
  "scr_dir2_threshold_10": 0.17647046131564434,
279
+ "scr_dir1_threshold_20": 0.11162786958280348,
280
+ "scr_metric_threshold_20": 0.11162786958280348,
281
+ "scr_dir2_threshold_20": 0.208144798820853,
282
+ "scr_dir1_threshold_50": 0.19069746037053023,
283
+ "scr_metric_threshold_50": 0.19069746037053023,
284
+ "scr_dir2_threshold_50": 0.23981886662180585,
285
+ "scr_dir1_threshold_100": 0.19069746037053023,
286
+ "scr_metric_threshold_100": 0.19069746037053023,
287
+ "scr_dir2_threshold_100": 0.31674202653080763,
288
+ "scr_dir1_threshold_500": 0.2837209160486496,
289
+ "scr_metric_threshold_500": 0.2837209160486496,
290
+ "scr_dir2_threshold_500": 0.38009043183696917
291
  }
292
  ],
293
  "sae_bench_commit_hash": "Unknown",
gemma-2-2b-btk/k-100/seed-0/snapshots/step-24414-tokens-100000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "5fca8cee-c948-46e9-a4d9-ffb5369182a7",
69
- "datetime_epoch_millis": 1776885656359,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.011775010824203493,
73
- "tpp_threshold_2_intended_diff_only": 0.01190001368522644,
74
- "tpp_threshold_2_unintended_diff_only": 0.0001250028610229492,
75
- "tpp_threshold_5_total_metric": 0.016574998199939725,
76
- "tpp_threshold_5_intended_diff_only": 0.019100004434585573,
77
- "tpp_threshold_5_unintended_diff_only": 0.0025250062346458433,
78
- "tpp_threshold_10_total_metric": 0.027750010788440707,
79
- "tpp_threshold_10_intended_diff_only": 0.03140001893043518,
80
- "tpp_threshold_10_unintended_diff_only": 0.0036500081419944762,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.01425001621246338,
99
- "tpp_threshold_2_intended_diff_only": 0.01380002498626709,
100
- "tpp_threshold_2_unintended_diff_only": -0.0004499912261962891,
101
- "tpp_threshold_5_total_metric": 0.017799994349479674,
102
- "tpp_threshold_5_intended_diff_only": 0.02040001153945923,
103
- "tpp_threshold_5_unintended_diff_only": 0.0026000171899795534,
104
- "tpp_threshold_10_total_metric": 0.03145001232624054,
105
- "tpp_threshold_10_intended_diff_only": 0.0350000262260437,
106
- "tpp_threshold_10_unintended_diff_only": 0.0035500138998031615,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.009300005435943604,
123
- "tpp_threshold_2_intended_diff_only": 0.010000002384185792,
124
- "tpp_threshold_2_unintended_diff_only": 0.0006999969482421875,
125
- "tpp_threshold_5_total_metric": 0.01535000205039978,
126
- "tpp_threshold_5_intended_diff_only": 0.017799997329711915,
127
- "tpp_threshold_5_unintended_diff_only": 0.0024499952793121337,
128
- "tpp_threshold_10_total_metric": 0.02405000925064087,
129
- "tpp_threshold_10_intended_diff_only": 0.02780001163482666,
130
- "tpp_threshold_10_unintended_diff_only": 0.003750002384185791,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.0022500157356262207,
190
  "tpp_threshold_10_total_metric": 0.004999950528144836,
191
  "tpp_threshold_10_intended_diff_only": 0.006999969482421875,
192
- "tpp_threshold_10_unintended_diff_only": 0.0020000189542770386
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": 0.006000012159347534,
196
- "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
197
- "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162,
198
- "tpp_threshold_5_total_metric": 0.009500011801719666,
199
- "tpp_threshold_5_intended_diff_only": 0.012000024318695068,
200
- "tpp_threshold_5_unintended_diff_only": 0.002500012516975403,
201
- "tpp_threshold_10_total_metric": 0.009749993681907654,
202
- "tpp_threshold_10_intended_diff_only": 0.013000011444091797,
203
- "tpp_threshold_10_unintended_diff_only": 0.003250017762184143
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.0467500239610672,
207
- "tpp_threshold_2_intended_diff_only": 0.04400002956390381,
208
- "tpp_threshold_2_unintended_diff_only": -0.002749994397163391,
209
- "tpp_threshold_5_total_metric": 0.05849997699260712,
210
- "tpp_threshold_5_intended_diff_only": 0.06000000238418579,
211
- "tpp_threshold_5_unintended_diff_only": 0.0015000253915786743,
212
- "tpp_threshold_10_total_metric": 0.08375003933906555,
213
- "tpp_threshold_10_intended_diff_only": 0.08400005102157593,
214
- "tpp_threshold_10_unintended_diff_only": 0.000250011682510376
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.010000035166740417,
218
  "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
219
- "tpp_threshold_2_unintended_diff_only": -0.003000006079673767,
220
- "tpp_threshold_5_total_metric": 0.004249989986419678,
221
- "tpp_threshold_5_intended_diff_only": 0.009000003337860107,
222
- "tpp_threshold_5_unintended_diff_only": 0.00475001335144043,
223
- "tpp_threshold_10_total_metric": 0.0102500319480896,
224
- "tpp_threshold_10_intended_diff_only": 0.01500004529953003,
225
- "tpp_threshold_10_unintended_diff_only": 0.00475001335144043
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.00025004148483276367,
229
- "tpp_threshold_2_intended_diff_only": 0.001000046730041504,
230
- "tpp_threshold_2_unintended_diff_only": 0.0007500052452087402,
231
- "tpp_threshold_5_total_metric": 0.008000031113624573,
232
- "tpp_threshold_5_intended_diff_only": 0.010000050067901611,
233
- "tpp_threshold_5_unintended_diff_only": 0.0020000189542770386,
234
- "tpp_threshold_10_total_metric": 0.048500046133995056,
235
- "tpp_threshold_10_intended_diff_only": 0.05600005388259888,
236
- "tpp_threshold_10_unintended_diff_only": 0.007500007748603821
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.010249987244606018,
242
- "tpp_threshold_2_intended_diff_only": 0.0059999823570251465,
243
- "tpp_threshold_2_unintended_diff_only": -0.004250004887580872,
244
- "tpp_threshold_5_total_metric": 0.016500011086463928,
245
- "tpp_threshold_5_intended_diff_only": 0.013000011444091797,
246
- "tpp_threshold_5_unintended_diff_only": -0.0034999996423721313,
247
- "tpp_threshold_10_total_metric": 0.00700002908706665,
248
- "tpp_threshold_10_intended_diff_only": 0.012000024318695068,
249
- "tpp_threshold_10_unintended_diff_only": 0.004999995231628418
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": 0.004000052809715271,
253
- "tpp_threshold_2_intended_diff_only": 0.010000050067901611,
254
- "tpp_threshold_2_unintended_diff_only": 0.00599999725818634,
255
- "tpp_threshold_5_total_metric": 0.018000036478042603,
256
- "tpp_threshold_5_intended_diff_only": 0.026000022888183594,
257
- "tpp_threshold_5_unintended_diff_only": 0.007999986410140991,
258
- "tpp_threshold_10_total_metric": 0.044250041246414185,
259
- "tpp_threshold_10_intended_diff_only": 0.04700005054473877,
260
- "tpp_threshold_10_unintended_diff_only": 0.002750009298324585
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.016249999403953552,
264
- "tpp_threshold_2_intended_diff_only": 0.009999990463256836,
265
- "tpp_threshold_2_unintended_diff_only": -0.006250008940696716,
266
- "tpp_threshold_5_total_metric": 0.01524999737739563,
267
- "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
268
- "tpp_threshold_5_unintended_diff_only": -0.0012499988079071045,
269
- "tpp_threshold_10_total_metric": 0.01975002884864807,
270
- "tpp_threshold_10_intended_diff_only": 0.021000027656555176,
271
- "tpp_threshold_10_unintended_diff_only": 0.0012499988079071045
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.006750017404556274,
275
- "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
276
- "tpp_threshold_2_unintended_diff_only": 0.007750004529953003,
277
- "tpp_threshold_5_total_metric": 0.0029999762773513794,
278
- "tpp_threshold_5_intended_diff_only": 0.006999969482421875,
279
- "tpp_threshold_5_unintended_diff_only": 0.003999993205070496,
280
- "tpp_threshold_10_total_metric": 0.005249962210655212,
281
- "tpp_threshold_10_intended_diff_only": 0.010999977588653564,
282
- "tpp_threshold_10_unintended_diff_only": 0.005750015377998352
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.02275000512599945,
286
- "tpp_threshold_2_intended_diff_only": 0.023000001907348633,
287
- "tpp_threshold_2_unintended_diff_only": 0.00024999678134918213,
288
- "tpp_threshold_5_total_metric": 0.02399998903274536,
289
- "tpp_threshold_5_intended_diff_only": 0.02899998426437378,
290
- "tpp_threshold_5_unintended_diff_only": 0.004999995231628418,
291
- "tpp_threshold_10_total_metric": 0.04399998486042023,
292
- "tpp_threshold_10_intended_diff_only": 0.04799997806549072,
293
- "tpp_threshold_10_unintended_diff_only": 0.003999993205070496
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "96d34a9f-9a36-44b1-920b-b6eab8594ada",
73
+ "datetime_epoch_millis": 1777316957490,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.010125000774860383,
77
+ "tpp_threshold_2_intended_diff_only": 0.011500006914138794,
78
+ "tpp_threshold_2_unintended_diff_only": 0.001375006139278412,
79
+ "tpp_threshold_5_total_metric": 0.015449994802474977,
80
+ "tpp_threshold_5_intended_diff_only": 0.01690000295639038,
81
+ "tpp_threshold_5_unintended_diff_only": 0.0014500081539154052,
82
+ "tpp_threshold_10_total_metric": 0.02767498642206192,
83
+ "tpp_threshold_10_intended_diff_only": 0.031699997186660764,
84
+ "tpp_threshold_10_unintended_diff_only": 0.004025010764598847,
85
+ "tpp_threshold_20_total_metric": 0.05050000697374344,
86
+ "tpp_threshold_20_intended_diff_only": 0.05460001230239868,
87
+ "tpp_threshold_20_unintended_diff_only": 0.004100005328655242,
88
+ "tpp_threshold_50_total_metric": 0.11895001530647278,
89
+ "tpp_threshold_50_intended_diff_only": 0.12460001111030579,
90
+ "tpp_threshold_50_unintended_diff_only": 0.005649995803833008,
91
+ "tpp_threshold_100_total_metric": 0.20270000994205473,
92
+ "tpp_threshold_100_intended_diff_only": 0.21150001287460327,
93
+ "tpp_threshold_100_unintended_diff_only": 0.008800002932548522,
94
+ "tpp_threshold_500_total_metric": 0.3938000306487084,
95
+ "tpp_threshold_500_intended_diff_only": 0.4196000337600708,
96
+ "tpp_threshold_500_unintended_diff_only": 0.025800003111362456
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.012350007891654968,
103
+ "tpp_threshold_2_intended_diff_only": 0.013400018215179443,
104
+ "tpp_threshold_2_unintended_diff_only": 0.001050010323524475,
105
+ "tpp_threshold_5_total_metric": 0.018299993872642518,
106
+ "tpp_threshold_5_intended_diff_only": 0.01960000991821289,
107
+ "tpp_threshold_5_unintended_diff_only": 0.0013000160455703736,
108
+ "tpp_threshold_10_total_metric": 0.031249985098838806,
109
+ "tpp_threshold_10_intended_diff_only": 0.0346000075340271,
110
+ "tpp_threshold_10_unintended_diff_only": 0.0033500224351882936,
111
+ "tpp_threshold_20_total_metric": 0.05729999542236328,
112
+ "tpp_threshold_20_intended_diff_only": 0.06020001173019409,
113
+ "tpp_threshold_20_unintended_diff_only": 0.0029000163078308104,
114
+ "tpp_threshold_50_total_metric": 0.13355001509189607,
115
+ "tpp_threshold_50_intended_diff_only": 0.13940001726150514,
116
+ "tpp_threshold_50_unintended_diff_only": 0.00585000216960907,
117
+ "tpp_threshold_100_total_metric": 0.22655001580715178,
118
+ "tpp_threshold_100_intended_diff_only": 0.232200026512146,
119
+ "tpp_threshold_100_unintended_diff_only": 0.0056500107049942015,
120
+ "tpp_threshold_500_total_metric": 0.4307000309228897,
121
+ "tpp_threshold_500_intended_diff_only": 0.4522000432014465,
122
+ "tpp_threshold_500_unintended_diff_only": 0.021500012278556822
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.007899993658065796,
127
+ "tpp_threshold_2_intended_diff_only": 0.009599995613098145,
128
+ "tpp_threshold_2_unintended_diff_only": 0.0017000019550323486,
129
+ "tpp_threshold_5_total_metric": 0.012599995732307434,
130
+ "tpp_threshold_5_intended_diff_only": 0.01419999599456787,
131
+ "tpp_threshold_5_unintended_diff_only": 0.001600000262260437,
132
+ "tpp_threshold_10_total_metric": 0.024099987745285035,
133
+ "tpp_threshold_10_intended_diff_only": 0.028799986839294432,
134
+ "tpp_threshold_10_unintended_diff_only": 0.004699999094009399,
135
+ "tpp_threshold_20_total_metric": 0.0437000185251236,
136
+ "tpp_threshold_20_intended_diff_only": 0.049000012874603274,
137
+ "tpp_threshold_20_unintended_diff_only": 0.005299994349479675,
138
+ "tpp_threshold_50_total_metric": 0.1043500155210495,
139
+ "tpp_threshold_50_intended_diff_only": 0.10980000495910644,
140
+ "tpp_threshold_50_unintended_diff_only": 0.0054499894380569455,
141
+ "tpp_threshold_100_total_metric": 0.1788500040769577,
142
+ "tpp_threshold_100_intended_diff_only": 0.19079999923706054,
143
+ "tpp_threshold_100_unintended_diff_only": 0.011949995160102844,
144
+ "tpp_threshold_500_total_metric": 0.356900030374527,
145
+ "tpp_threshold_500_intended_diff_only": 0.38700002431869507,
146
+ "tpp_threshold_500_unintended_diff_only": 0.03009999394416809
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.0022500157356262207,
194
  "tpp_threshold_10_total_metric": 0.004999950528144836,
195
  "tpp_threshold_10_intended_diff_only": 0.006999969482421875,
196
+ "tpp_threshold_10_unintended_diff_only": 0.0020000189542770386,
197
+ "tpp_threshold_20_total_metric": 0.017499983310699463,
198
+ "tpp_threshold_20_intended_diff_only": 0.018000006675720215,
199
+ "tpp_threshold_20_unintended_diff_only": 0.000500023365020752,
200
+ "tpp_threshold_50_total_metric": 0.07524995505809784,
201
+ "tpp_threshold_50_intended_diff_only": 0.08099997043609619,
202
+ "tpp_threshold_50_unintended_diff_only": 0.005750015377998352,
203
+ "tpp_threshold_100_total_metric": 0.15574996173381805,
204
+ "tpp_threshold_100_intended_diff_only": 0.1589999794960022,
205
+ "tpp_threshold_100_unintended_diff_only": 0.003250017762184143,
206
+ "tpp_threshold_500_total_metric": 0.4347500056028366,
207
+ "tpp_threshold_500_intended_diff_only": 0.44700002670288086,
208
+ "tpp_threshold_500_unintended_diff_only": 0.01225002110004425
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.009499981999397278,
212
+ "tpp_threshold_2_intended_diff_only": 0.009999990463256836,
213
+ "tpp_threshold_2_unintended_diff_only": 0.0005000084638595581,
214
+ "tpp_threshold_5_total_metric": 0.006250008940696716,
215
+ "tpp_threshold_5_intended_diff_only": 0.00700002908706665,
216
+ "tpp_threshold_5_unintended_diff_only": 0.0007500201463699341,
217
+ "tpp_threshold_10_total_metric": 0.015249952673912048,
218
+ "tpp_threshold_10_intended_diff_only": 0.014999985694885254,
219
+ "tpp_threshold_10_unintended_diff_only": -0.00024996697902679443,
220
+ "tpp_threshold_20_total_metric": 0.024749964475631714,
221
+ "tpp_threshold_20_intended_diff_only": 0.02899998426437378,
222
+ "tpp_threshold_20_unintended_diff_only": 0.004250019788742065,
223
+ "tpp_threshold_50_total_metric": 0.11525003612041473,
224
+ "tpp_threshold_50_intended_diff_only": 0.12200003862380981,
225
+ "tpp_threshold_50_unintended_diff_only": 0.006750002503395081,
226
+ "tpp_threshold_100_total_metric": 0.2240000069141388,
227
+ "tpp_threshold_100_intended_diff_only": 0.23000001907348633,
228
+ "tpp_threshold_100_unintended_diff_only": 0.006000012159347534,
229
+ "tpp_threshold_500_total_metric": 0.434750035405159,
230
+ "tpp_threshold_500_intended_diff_only": 0.4590000510215759,
231
+ "tpp_threshold_500_unintended_diff_only": 0.02425001561641693
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.03475002944469452,
235
+ "tpp_threshold_2_intended_diff_only": 0.03800004720687866,
236
+ "tpp_threshold_2_unintended_diff_only": 0.003250017762184143,
237
+ "tpp_threshold_5_total_metric": 0.05400003492832184,
238
+ "tpp_threshold_5_intended_diff_only": 0.05600005388259888,
239
+ "tpp_threshold_5_unintended_diff_only": 0.0020000189542770386,
240
+ "tpp_threshold_10_total_metric": 0.08175002038478851,
241
+ "tpp_threshold_10_intended_diff_only": 0.08500003814697266,
242
+ "tpp_threshold_10_unintended_diff_only": 0.003250017762184143,
243
+ "tpp_threshold_20_total_metric": 0.12000003457069397,
244
+ "tpp_threshold_20_intended_diff_only": 0.12100005149841309,
245
+ "tpp_threshold_20_unintended_diff_only": 0.0010000169277191162,
246
+ "tpp_threshold_50_total_metric": 0.20500002801418304,
247
+ "tpp_threshold_50_intended_diff_only": 0.2070000171661377,
248
+ "tpp_threshold_50_unintended_diff_only": 0.001999989151954651,
249
+ "tpp_threshold_100_total_metric": 0.2952500432729721,
250
+ "tpp_threshold_100_intended_diff_only": 0.29700005054473877,
251
+ "tpp_threshold_100_unintended_diff_only": 0.0017500072717666626,
252
+ "tpp_threshold_500_total_metric": 0.4150000214576721,
253
+ "tpp_threshold_500_intended_diff_only": 0.44300001859664917,
254
+ "tpp_threshold_500_unintended_diff_only": 0.02799999713897705
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.007750019431114197,
258
  "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0007499903440475464,
260
+ "tpp_threshold_5_total_metric": 0.010749980807304382,
261
+ "tpp_threshold_5_intended_diff_only": 0.009999990463256836,
262
+ "tpp_threshold_5_unintended_diff_only": -0.0007499903440475464,
263
+ "tpp_threshold_10_total_metric": 0.013750001788139343,
264
+ "tpp_threshold_10_intended_diff_only": 0.016000032424926758,
265
+ "tpp_threshold_10_unintended_diff_only": 0.0022500306367874146,
266
+ "tpp_threshold_20_total_metric": 0.01374998688697815,
267
+ "tpp_threshold_20_intended_diff_only": 0.013999998569488525,
268
+ "tpp_threshold_20_unintended_diff_only": 0.000250011682510376,
269
+ "tpp_threshold_50_total_metric": 0.03700004518032074,
270
+ "tpp_threshold_50_intended_diff_only": 0.03800004720687866,
271
+ "tpp_threshold_50_unintended_diff_only": 0.0010000020265579224,
272
+ "tpp_threshold_100_total_metric": 0.07475000619888306,
273
+ "tpp_threshold_100_intended_diff_only": 0.07700002193450928,
274
+ "tpp_threshold_100_unintended_diff_only": 0.0022500157356262207,
275
+ "tpp_threshold_500_total_metric": 0.42750002443790436,
276
+ "tpp_threshold_500_intended_diff_only": 0.4360000491142273,
277
+ "tpp_threshold_500_unintended_diff_only": 0.008500024676322937
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.0015000402927398682,
281
+ "tpp_threshold_2_intended_diff_only": 0.0020000338554382324,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0004999935626983643,
283
+ "tpp_threshold_5_total_metric": 0.011749982833862305,
284
+ "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
285
+ "tpp_threshold_5_unintended_diff_only": 0.0022500157356262207,
286
+ "tpp_threshold_10_total_metric": 0.04050000011920929,
287
+ "tpp_threshold_10_intended_diff_only": 0.050000011920928955,
288
+ "tpp_threshold_10_unintended_diff_only": 0.009500011801719666,
289
+ "tpp_threshold_20_total_metric": 0.11050000786781311,
290
+ "tpp_threshold_20_intended_diff_only": 0.11900001764297485,
291
+ "tpp_threshold_20_unintended_diff_only": 0.008500009775161743,
292
+ "tpp_threshold_50_total_metric": 0.23525001108646393,
293
+ "tpp_threshold_50_intended_diff_only": 0.24900001287460327,
294
+ "tpp_threshold_50_unintended_diff_only": 0.013750001788139343,
295
+ "tpp_threshold_100_total_metric": 0.38300006091594696,
296
+ "tpp_threshold_100_intended_diff_only": 0.3980000615119934,
297
+ "tpp_threshold_100_unintended_diff_only": 0.015000000596046448,
298
+ "tpp_threshold_500_total_metric": 0.44150006771087646,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.03450000286102295
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.004000023007392883,
306
+ "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
307
+ "tpp_threshold_2_unintended_diff_only": 0.003999993205070496,
308
+ "tpp_threshold_5_total_metric": 0.008249998092651367,
309
+ "tpp_threshold_5_intended_diff_only": 0.009000003337860107,
310
+ "tpp_threshold_5_unintended_diff_only": 0.0007500052452087402,
311
+ "tpp_threshold_10_total_metric": 0.008999988436698914,
312
+ "tpp_threshold_10_intended_diff_only": 0.010999977588653564,
313
+ "tpp_threshold_10_unintended_diff_only": 0.001999989151954651,
314
+ "tpp_threshold_20_total_metric": 0.015750035643577576,
315
+ "tpp_threshold_20_intended_diff_only": 0.021000027656555176,
316
+ "tpp_threshold_20_unintended_diff_only": 0.0052499920129776,
317
+ "tpp_threshold_50_total_metric": 0.0805000513792038,
318
+ "tpp_threshold_50_intended_diff_only": 0.08100003004074097,
319
+ "tpp_threshold_50_unintended_diff_only": 0.0004999786615371704,
320
+ "tpp_threshold_100_total_metric": 0.13700000941753387,
321
+ "tpp_threshold_100_intended_diff_only": 0.14300000667572021,
322
+ "tpp_threshold_100_unintended_diff_only": 0.00599999725818634,
323
+ "tpp_threshold_500_total_metric": 0.3920000493526459,
324
+ "tpp_threshold_500_intended_diff_only": 0.39500004053115845,
325
+ "tpp_threshold_500_unintended_diff_only": 0.0029999911785125732
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.012500032782554626,
329
+ "tpp_threshold_2_intended_diff_only": 0.012000024318695068,
330
+ "tpp_threshold_2_unintended_diff_only": -0.0005000084638595581,
331
+ "tpp_threshold_5_total_metric": 0.013250023126602173,
332
+ "tpp_threshold_5_intended_diff_only": 0.022000014781951904,
333
+ "tpp_threshold_5_unintended_diff_only": 0.008749991655349731,
334
+ "tpp_threshold_10_total_metric": 0.03925003111362457,
335
+ "tpp_threshold_10_intended_diff_only": 0.04900002479553223,
336
+ "tpp_threshold_10_unintended_diff_only": 0.009749993681907654,
337
+ "tpp_threshold_20_total_metric": 0.067500039935112,
338
+ "tpp_threshold_20_intended_diff_only": 0.07100003957748413,
339
+ "tpp_threshold_20_unintended_diff_only": 0.0034999996423721313,
340
+ "tpp_threshold_50_total_metric": 0.12450003623962402,
341
+ "tpp_threshold_50_intended_diff_only": 0.13200002908706665,
342
+ "tpp_threshold_50_unintended_diff_only": 0.007499992847442627,
343
+ "tpp_threshold_100_total_metric": 0.28025004267692566,
344
+ "tpp_threshold_100_intended_diff_only": 0.30800002813339233,
345
+ "tpp_threshold_100_unintended_diff_only": 0.027749985456466675,
346
+ "tpp_threshold_500_total_metric": 0.3997500538825989,
347
+ "tpp_threshold_500_intended_diff_only": 0.4360000491142273,
348
+ "tpp_threshold_500_unintended_diff_only": 0.03624999523162842
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.005499988794326782,
352
+ "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
353
+ "tpp_threshold_2_unintended_diff_only": -0.0004999935626983643,
354
+ "tpp_threshold_5_total_metric": 0.00849999487400055,
355
+ "tpp_threshold_5_intended_diff_only": 0.004999995231628418,
356
+ "tpp_threshold_5_unintended_diff_only": -0.0034999996423721313,
357
+ "tpp_threshold_10_total_metric": 0.021249979734420776,
358
+ "tpp_threshold_10_intended_diff_only": 0.018999993801116943,
359
+ "tpp_threshold_10_unintended_diff_only": -0.002249985933303833,
360
+ "tpp_threshold_20_total_metric": 0.013999998569488525,
361
+ "tpp_threshold_20_intended_diff_only": 0.014999985694885254,
362
+ "tpp_threshold_20_unintended_diff_only": 0.0009999871253967285,
363
+ "tpp_threshold_50_total_metric": 0.056250035762786865,
364
+ "tpp_threshold_50_intended_diff_only": 0.054000020027160645,
365
+ "tpp_threshold_50_unintended_diff_only": -0.0022500157356262207,
366
+ "tpp_threshold_100_total_metric": 0.09549999237060547,
367
+ "tpp_threshold_100_intended_diff_only": 0.09799998998641968,
368
+ "tpp_threshold_100_unintended_diff_only": 0.002499997615814209,
369
+ "tpp_threshold_500_total_metric": 0.33650003373622894,
370
+ "tpp_threshold_500_intended_diff_only": 0.3450000286102295,
371
+ "tpp_threshold_500_unintended_diff_only": 0.00849999487400055
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.005499973893165588,
375
+ "tpp_threshold_2_intended_diff_only": 0.0059999823570251465,
376
+ "tpp_threshold_2_unintended_diff_only": 0.0005000084638595581,
377
+ "tpp_threshold_5_total_metric": 0.01199999451637268,
378
+ "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
379
+ "tpp_threshold_5_unintended_diff_only": 0.0020000040531158447,
380
+ "tpp_threshold_10_total_metric": -0.005500033497810364,
381
+ "tpp_threshold_10_intended_diff_only": 0.0029999613761901855,
382
+ "tpp_threshold_10_unintended_diff_only": 0.00849999487400055,
383
+ "tpp_threshold_20_total_metric": 0.02449999749660492,
384
+ "tpp_threshold_20_intended_diff_only": 0.02799999713897705,
385
+ "tpp_threshold_20_unintended_diff_only": 0.0034999996423721313,
386
+ "tpp_threshold_50_total_metric": 0.08624997735023499,
387
+ "tpp_threshold_50_intended_diff_only": 0.09399998188018799,
388
+ "tpp_threshold_50_unintended_diff_only": 0.007750004529953003,
389
+ "tpp_threshold_100_total_metric": 0.1289999783039093,
390
+ "tpp_threshold_100_intended_diff_only": 0.13599997758865356,
391
+ "tpp_threshold_100_unintended_diff_only": 0.006999999284744263,
392
+ "tpp_threshold_500_total_metric": 0.31325002014636993,
393
+ "tpp_threshold_500_intended_diff_only": 0.3880000114440918,
394
+ "tpp_threshold_500_unintended_diff_only": 0.07474999129772186
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.011999949812889099,
398
+ "tpp_threshold_2_intended_diff_only": 0.01699995994567871,
399
+ "tpp_threshold_2_unintended_diff_only": 0.005000010132789612,
400
+ "tpp_threshold_5_total_metric": 0.0209999680519104,
401
+ "tpp_threshold_5_intended_diff_only": 0.0209999680519104,
402
+ "tpp_threshold_5_unintended_diff_only": 0.0,
403
+ "tpp_threshold_10_total_metric": 0.05649997293949127,
404
+ "tpp_threshold_10_intended_diff_only": 0.06199997663497925,
405
+ "tpp_threshold_10_unintended_diff_only": 0.005500003695487976,
406
+ "tpp_threshold_20_total_metric": 0.09675002098083496,
407
+ "tpp_threshold_20_intended_diff_only": 0.11000001430511475,
408
+ "tpp_threshold_20_unintended_diff_only": 0.013249993324279785,
409
+ "tpp_threshold_50_total_metric": 0.17424997687339783,
410
+ "tpp_threshold_50_intended_diff_only": 0.18799996376037598,
411
+ "tpp_threshold_50_unintended_diff_only": 0.01374998688697815,
412
+ "tpp_threshold_100_total_metric": 0.2524999976158142,
413
+ "tpp_threshold_100_intended_diff_only": 0.26899999380111694,
414
+ "tpp_threshold_100_unintended_diff_only": 0.016499996185302734,
415
+ "tpp_threshold_500_total_metric": 0.34299999475479126,
416
+ "tpp_threshold_500_intended_diff_only": 0.3709999918937683,
417
+ "tpp_threshold_500_unintended_diff_only": 0.02799999713897705
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-244140-tokens-1000000000/scr/saebench_scr_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,8 +69,8 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "7b769476-9026-4538-89a9-a67496f617b5",
69
- "datetime_epoch_millis": 1776885520377,
70
  "eval_result_metrics": {
71
  "scr_metrics": {
72
  "scr_dir1_threshold_2": 0.2840545467631945,
@@ -78,18 +82,18 @@
78
  "scr_dir1_threshold_10": 0.21232264677068308,
79
  "scr_metric_threshold_10": 0.20882376972801514,
80
  "scr_dir2_threshold_10": 0.20197067998020166,
81
- "scr_dir1_threshold_20": null,
82
- "scr_metric_threshold_20": null,
83
- "scr_dir2_threshold_20": null,
84
- "scr_dir1_threshold_50": null,
85
- "scr_metric_threshold_50": null,
86
- "scr_dir2_threshold_50": null,
87
- "scr_dir1_threshold_100": null,
88
- "scr_metric_threshold_100": null,
89
- "scr_dir2_threshold_100": null,
90
- "scr_dir1_threshold_500": null,
91
- "scr_metric_threshold_500": null,
92
- "scr_dir2_threshold_500": null
93
  }
94
  },
95
  "eval_result_details": [
@@ -104,18 +108,18 @@
104
  "scr_dir1_threshold_10": 0.24359039998532808,
105
  "scr_metric_threshold_10": 0.010362714302611827,
106
  "scr_dir2_threshold_10": 0.010362714302611827,
107
- "scr_dir1_threshold_20": null,
108
- "scr_metric_threshold_20": null,
109
- "scr_dir2_threshold_20": null,
110
- "scr_dir1_threshold_50": null,
111
- "scr_metric_threshold_50": null,
112
- "scr_dir2_threshold_50": null,
113
- "scr_dir1_threshold_100": null,
114
- "scr_metric_threshold_100": null,
115
- "scr_dir2_threshold_100": null,
116
- "scr_dir1_threshold_500": null,
117
- "scr_metric_threshold_500": null,
118
- "scr_dir2_threshold_500": null
119
  },
120
  {
121
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
@@ -128,18 +132,18 @@
128
  "scr_dir1_threshold_10": 0.31060605718522544,
129
  "scr_metric_threshold_10": 0.29969413887699026,
130
  "scr_dir2_threshold_10": 0.29969413887699026,
131
- "scr_dir1_threshold_20": null,
132
- "scr_metric_threshold_20": null,
133
- "scr_dir2_threshold_20": null,
134
- "scr_dir1_threshold_50": null,
135
- "scr_metric_threshold_50": null,
136
- "scr_dir2_threshold_50": null,
137
- "scr_dir1_threshold_100": null,
138
- "scr_metric_threshold_100": null,
139
- "scr_dir2_threshold_100": null,
140
- "scr_dir1_threshold_500": null,
141
- "scr_metric_threshold_500": null,
142
- "scr_dir2_threshold_500": null
143
  },
144
  {
145
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
@@ -152,18 +156,18 @@
152
  "scr_dir1_threshold_10": 0.1250002328306055,
153
  "scr_metric_threshold_10": 0.14070348146483033,
154
  "scr_dir2_threshold_10": 0.14070348146483033,
155
- "scr_dir1_threshold_20": null,
156
- "scr_metric_threshold_20": null,
157
- "scr_dir2_threshold_20": null,
158
- "scr_dir1_threshold_50": null,
159
- "scr_metric_threshold_50": null,
160
- "scr_dir2_threshold_50": null,
161
- "scr_dir1_threshold_100": null,
162
- "scr_metric_threshold_100": null,
163
- "scr_dir2_threshold_100": null,
164
- "scr_dir1_threshold_500": null,
165
- "scr_metric_threshold_500": null,
166
- "scr_dir2_threshold_500": null
167
  },
168
  {
169
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
@@ -176,18 +180,18 @@
176
  "scr_dir1_threshold_10": -0.0805370818923876,
177
  "scr_metric_threshold_10": -0.009554016802434907,
178
  "scr_dir2_threshold_10": -0.009554016802434907,
179
- "scr_dir1_threshold_20": null,
180
- "scr_metric_threshold_20": null,
181
- "scr_dir2_threshold_20": null,
182
- "scr_dir1_threshold_50": null,
183
- "scr_metric_threshold_50": null,
184
- "scr_dir2_threshold_50": null,
185
- "scr_dir1_threshold_100": null,
186
- "scr_metric_threshold_100": null,
187
- "scr_dir2_threshold_100": null,
188
- "scr_dir1_threshold_500": null,
189
- "scr_metric_threshold_500": null,
190
- "scr_dir2_threshold_500": null
191
  },
192
  {
193
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
@@ -200,18 +204,18 @@
200
  "scr_dir1_threshold_10": 0.09473687842942387,
201
  "scr_metric_threshold_10": 0.4285715299398585,
202
  "scr_dir2_threshold_10": 0.4285715299398585,
203
- "scr_dir1_threshold_20": null,
204
- "scr_metric_threshold_20": null,
205
- "scr_dir2_threshold_20": null,
206
- "scr_dir1_threshold_50": null,
207
- "scr_metric_threshold_50": null,
208
- "scr_dir2_threshold_50": null,
209
- "scr_dir1_threshold_100": null,
210
- "scr_metric_threshold_100": null,
211
- "scr_dir2_threshold_100": null,
212
- "scr_dir1_threshold_500": null,
213
- "scr_metric_threshold_500": null,
214
- "scr_dir2_threshold_500": null
215
  },
216
  {
217
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
@@ -224,18 +228,18 @@
224
  "scr_dir1_threshold_10": 0.4098360709132541,
225
  "scr_metric_threshold_10": 0.16412205665125212,
226
  "scr_dir2_threshold_10": 0.16412205665125212,
227
- "scr_dir1_threshold_20": null,
228
- "scr_metric_threshold_20": null,
229
- "scr_dir2_threshold_20": null,
230
- "scr_dir1_threshold_50": null,
231
- "scr_metric_threshold_50": null,
232
- "scr_dir2_threshold_50": null,
233
- "scr_dir1_threshold_100": null,
234
- "scr_metric_threshold_100": null,
235
- "scr_dir2_threshold_100": null,
236
- "scr_dir1_threshold_500": null,
237
- "scr_metric_threshold_500": null,
238
- "scr_dir2_threshold_500": null
239
  },
240
  {
241
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
@@ -248,18 +252,18 @@
248
  "scr_dir1_threshold_10": 0.4093022598196558,
249
  "scr_metric_threshold_10": 0.4506438964966536,
250
  "scr_dir2_threshold_10": 0.4506438964966536,
251
- "scr_dir1_threshold_20": null,
252
- "scr_metric_threshold_20": null,
253
- "scr_dir2_threshold_20": null,
254
- "scr_dir1_threshold_50": null,
255
- "scr_metric_threshold_50": null,
256
- "scr_dir2_threshold_50": null,
257
- "scr_dir1_threshold_100": null,
258
- "scr_metric_threshold_100": null,
259
- "scr_dir2_threshold_100": null,
260
- "scr_dir1_threshold_500": null,
261
- "scr_metric_threshold_500": null,
262
- "scr_dir2_threshold_500": null
263
  },
264
  {
265
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
@@ -272,18 +276,18 @@
272
  "scr_dir1_threshold_10": 0.18604635689435922,
273
  "scr_metric_threshold_10": 0.18604635689435922,
274
  "scr_dir2_threshold_10": 0.1312216389118512,
275
- "scr_dir1_threshold_20": null,
276
- "scr_metric_threshold_20": null,
277
- "scr_dir2_threshold_20": null,
278
- "scr_dir1_threshold_50": null,
279
- "scr_metric_threshold_50": null,
280
- "scr_dir2_threshold_50": null,
281
- "scr_dir1_threshold_100": null,
282
- "scr_metric_threshold_100": null,
283
- "scr_dir2_threshold_100": null,
284
- "scr_dir1_threshold_500": null,
285
- "scr_metric_threshold_500": null,
286
- "scr_dir2_threshold_500": null
287
  }
288
  ],
289
  "sae_bench_commit_hash": "Unknown",
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "e4e83998-fec8-45e5-bdbc-3730c9454491",
73
+ "datetime_epoch_millis": 1777317034894,
74
  "eval_result_metrics": {
75
  "scr_metrics": {
76
  "scr_dir1_threshold_2": 0.2840545467631945,
 
82
  "scr_dir1_threshold_10": 0.21232264677068308,
83
  "scr_metric_threshold_10": 0.20882376972801514,
84
  "scr_dir2_threshold_10": 0.20197067998020166,
85
+ "scr_dir1_threshold_20": 0.13545519102935433,
86
+ "scr_metric_threshold_20": 0.23108328808010634,
87
+ "scr_dir2_threshold_20": 0.22685304974797876,
88
+ "scr_dir1_threshold_50": 0.13788349135316502,
89
+ "scr_metric_threshold_50": 0.3072251125637197,
90
+ "scr_dir2_threshold_50": 0.2971335714465668,
91
+ "scr_dir1_threshold_100": -0.09793706365514628,
92
+ "scr_metric_threshold_100": 0.30912104268525553,
93
+ "scr_dir2_threshold_100": 0.3019522602916574,
94
+ "scr_dir1_threshold_500": -1.0374360799353397,
95
+ "scr_metric_threshold_500": 0.18726127052226596,
96
+ "scr_dir2_threshold_500": 0.17635157269187982
97
  }
98
  },
99
  "eval_result_details": [
 
108
  "scr_dir1_threshold_10": 0.24359039998532808,
109
  "scr_metric_threshold_10": 0.010362714302611827,
110
  "scr_dir2_threshold_10": 0.010362714302611827,
111
+ "scr_dir1_threshold_20": 0.15384644775462453,
112
+ "scr_metric_threshold_20": 0.08290156000473131,
113
+ "scr_dir2_threshold_20": 0.08290156000473131,
114
+ "scr_dir1_threshold_50": 0.38461573730554943,
115
+ "scr_metric_threshold_50": 0.17616583431207444,
116
+ "scr_dir2_threshold_50": 0.17616583431207444,
117
+ "scr_dir1_threshold_100": 0.11538464477546245,
118
+ "scr_metric_threshold_100": 0.2797926685058661,
119
+ "scr_dir2_threshold_100": 0.2797926685058661,
120
+ "scr_dir1_threshold_500": -2.1538456835926008,
121
+ "scr_metric_threshold_500": -0.06735756575889522,
122
+ "scr_dir2_threshold_500": -0.06735756575889522
123
  },
124
  {
125
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
 
132
  "scr_dir1_threshold_10": 0.31060605718522544,
133
  "scr_metric_threshold_10": 0.29969413887699026,
134
  "scr_dir2_threshold_10": 0.29969413887699026,
135
+ "scr_dir1_threshold_20": 0.3484848005931564,
136
+ "scr_metric_threshold_20": 0.22018355312924365,
137
+ "scr_dir2_threshold_20": 0.22018355312924365,
138
+ "scr_dir1_threshold_50": 0.5530302859261272,
139
+ "scr_metric_threshold_50": 0.3241590186196148,
140
+ "scr_dir2_threshold_50": 0.3241590186196148,
141
+ "scr_dir1_threshold_100": 0.3181817155567637,
142
+ "scr_metric_threshold_100": 0.30581044995124085,
143
+ "scr_dir2_threshold_100": 0.30581044995124085,
144
+ "scr_dir1_threshold_500": -1.8560602331895748,
145
+ "scr_metric_threshold_500": -0.006116128797061692,
146
+ "scr_dir2_threshold_500": -0.006116128797061692
147
  },
148
  {
149
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
 
156
  "scr_dir1_threshold_10": 0.1250002328306055,
157
  "scr_metric_threshold_10": 0.14070348146483033,
158
  "scr_dir2_threshold_10": 0.14070348146483033,
159
+ "scr_dir1_threshold_20": 0.28125005820765137,
160
+ "scr_metric_threshold_20": 0.15577896145062706,
161
+ "scr_dir2_threshold_20": 0.15577896145062706,
162
+ "scr_dir1_threshold_50": -0.7968748544808716,
163
+ "scr_metric_threshold_50": 0.15577896145062706,
164
+ "scr_dir2_threshold_50": 0.15577896145062706,
165
+ "scr_dir1_threshold_100": -0.9374998835846973,
166
+ "scr_metric_threshold_100": 0.027638280133700087,
167
+ "scr_dir2_threshold_100": 0.027638280133700087,
168
+ "scr_dir1_threshold_500": -3.9218741559890553,
169
+ "scr_metric_threshold_500": 0.07537695064781104,
170
+ "scr_dir2_threshold_500": 0.07537695064781104
171
  },
172
  {
173
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
 
180
  "scr_dir1_threshold_10": -0.0805370818923876,
181
  "scr_metric_threshold_10": -0.009554016802434907,
182
  "scr_dir2_threshold_10": -0.009554016802434907,
183
+ "scr_dir1_threshold_20": -0.9395976886197124,
184
+ "scr_metric_threshold_20": -0.12420373702127886,
185
+ "scr_dir2_threshold_20": -0.12420373702127886,
186
+ "scr_dir1_threshold_50": -0.1610741637847752,
187
+ "scr_metric_threshold_50": 0.006369534358659739,
188
+ "scr_dir2_threshold_50": 0.006369534358659739,
189
+ "scr_dir1_threshold_100": -1.3355709745360138,
190
+ "scr_metric_threshold_100": -0.14649682254103324,
191
+ "scr_dir2_threshold_100": -0.14649682254103324,
192
+ "scr_dir1_threshold_500": -1.51006738525605,
193
+ "scr_metric_threshold_500": -0.5414013089483273,
194
+ "scr_dir2_threshold_500": -0.5414013089483273
195
  },
196
  {
197
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
 
204
  "scr_dir1_threshold_10": 0.09473687842942387,
205
  "scr_metric_threshold_10": 0.4285715299398585,
206
  "scr_dir2_threshold_10": 0.4285715299398585,
207
+ "scr_dir1_threshold_20": 0.08947378822971379,
208
+ "scr_metric_threshold_20": 0.5396826147702655,
209
+ "scr_dir2_threshold_20": 0.5396826147702655,
210
+ "scr_dir1_threshold_50": -0.13157882353605502,
211
+ "scr_metric_threshold_50": 0.6269840124753451,
212
+ "scr_dir2_threshold_50": 0.6269840124753451,
213
+ "scr_dir1_threshold_100": -0.15789458824326602,
214
+ "scr_metric_threshold_100": 0.6587301042915575,
215
+ "scr_dir2_threshold_100": 0.6587301042915575,
216
+ "scr_dir1_threshold_500": -0.12631573333634494,
217
+ "scr_metric_threshold_500": 0.773809391467407,
218
+ "scr_dir2_threshold_500": 0.773809391467407
219
  },
220
  {
221
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
 
228
  "scr_dir1_threshold_10": 0.4098360709132541,
229
  "scr_metric_threshold_10": 0.16412205665125212,
230
  "scr_dir2_threshold_10": 0.16412205665125212,
231
+ "scr_dir1_threshold_20": 0.4153004823742847,
232
+ "scr_metric_threshold_20": 0.22137413263306224,
233
+ "scr_dir2_threshold_20": 0.22137413263306224,
234
+ "scr_dir1_threshold_50": 0.4316940424658739,
235
+ "scr_metric_threshold_50": 0.28625946991455836,
236
+ "scr_dir2_threshold_50": 0.28625946991455836,
237
+ "scr_dir1_threshold_100": 0.4371584539269045,
238
+ "scr_metric_threshold_100": 0.48473290865407037,
239
+ "scr_dir2_threshold_100": 0.48473290865407037,
240
+ "scr_dir1_threshold_500": 0.3989072479911928,
241
+ "scr_metric_threshold_500": 0.35877852349298656,
242
+ "scr_dir2_threshold_500": 0.35877852349298656
243
  },
244
  {
245
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
 
252
  "scr_dir1_threshold_10": 0.4093022598196558,
253
  "scr_metric_threshold_10": 0.4506438964966536,
254
  "scr_dir2_threshold_10": 0.4506438964966536,
255
+ "scr_dir1_threshold_20": 0.48837210591771457,
256
+ "scr_metric_threshold_20": 0.506437685896798,
257
+ "scr_dir2_threshold_20": 0.506437685896798,
258
+ "scr_dir1_threshold_50": 0.5162789962690271,
259
+ "scr_metric_threshold_50": 0.5751073587184335,
260
+ "scr_dir2_threshold_50": 0.5751073587184335,
261
+ "scr_dir1_threshold_100": 0.49767431029119774,
262
+ "scr_metric_threshold_100": 0.5836909399141642,
263
+ "scr_dir2_threshold_100": 0.5836909399141642,
264
+ "scr_dir1_threshold_500": 0.48837210591771457,
265
+ "scr_metric_threshold_500": 0.5236051041022068,
266
+ "scr_dir2_threshold_500": 0.5236051041022068
267
  },
268
  {
269
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
 
276
  "scr_dir1_threshold_10": 0.18604635689435922,
277
  "scr_metric_threshold_10": 0.18604635689435922,
278
  "scr_dir2_threshold_10": 0.1312216389118512,
279
+ "scr_dir1_threshold_20": 0.24651153377740184,
280
+ "scr_metric_threshold_20": 0.24651153377740184,
281
+ "scr_dir2_threshold_20": 0.21266962712038115,
282
+ "scr_dir1_threshold_50": 0.3069767106604445,
283
+ "scr_metric_threshold_50": 0.3069767106604445,
284
+ "scr_dir2_threshold_50": 0.2262443817232214,
285
+ "scr_dir1_threshold_100": 0.27906981257247854,
286
+ "scr_metric_threshold_100": 0.27906981257247854,
287
+ "scr_dir2_threshold_100": 0.22171955342369326,
288
+ "scr_dir1_threshold_500": 0.3813951979720002,
289
+ "scr_metric_threshold_500": 0.3813951979720002,
290
+ "scr_dir2_threshold_500": 0.29411761532891106
291
  }
292
  ],
293
  "sae_bench_commit_hash": "Unknown",
gemma-2-2b-btk/k-100/seed-0/snapshots/step-244140-tokens-1000000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "f9684801-165e-4650-91a4-1f8bd9b21619",
69
- "datetime_epoch_millis": 1776885682543,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.02392500936985016,
73
- "tpp_threshold_2_intended_diff_only": 0.02620001435279846,
74
- "tpp_threshold_2_unintended_diff_only": 0.0022750049829483032,
75
- "tpp_threshold_5_total_metric": 0.10072501301765441,
76
- "tpp_threshold_5_intended_diff_only": 0.13040001392364503,
77
- "tpp_threshold_5_unintended_diff_only": 0.0296750009059906,
78
- "tpp_threshold_10_total_metric": 0.10860000550746918,
79
- "tpp_threshold_10_intended_diff_only": 0.1636000096797943,
80
- "tpp_threshold_10_unintended_diff_only": 0.05500000417232514,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.03755001127719879,
99
- "tpp_threshold_2_intended_diff_only": 0.04180002212524414,
100
- "tpp_threshold_2_unintended_diff_only": 0.0042500108480453495,
101
- "tpp_threshold_5_total_metric": 0.18545002043247222,
102
- "tpp_threshold_5_intended_diff_only": 0.24300003051757812,
103
- "tpp_threshold_5_unintended_diff_only": 0.05755001008510589,
104
- "tpp_threshold_10_total_metric": 0.19265002012252808,
105
  "tpp_threshold_10_intended_diff_only": 0.29980002641677855,
106
- "tpp_threshold_10_unintended_diff_only": 0.10715000629425049,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.010300007462501527,
123
- "tpp_threshold_2_intended_diff_only": 0.010600006580352784,
124
- "tpp_threshold_2_unintended_diff_only": 0.0002999991178512573,
125
- "tpp_threshold_5_total_metric": 0.016000005602836608,
126
- "tpp_threshold_5_intended_diff_only": 0.017799997329711915,
127
- "tpp_threshold_5_unintended_diff_only": 0.001799991726875305,
128
- "tpp_threshold_10_total_metric": 0.02454999089241028,
129
- "tpp_threshold_10_intended_diff_only": 0.027399992942810057,
130
- "tpp_threshold_10_unintended_diff_only": 0.00285000205039978,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.019500017166137695,
190
  "tpp_threshold_10_total_metric": 0.1617499738931656,
191
  "tpp_threshold_10_intended_diff_only": 0.1809999942779541,
192
- "tpp_threshold_10_unintended_diff_only": 0.019250020384788513
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": 0.04900000989437103,
196
  "tpp_threshold_2_intended_diff_only": 0.05900001525878906,
197
- "tpp_threshold_2_unintended_diff_only": 0.01000000536441803,
198
- "tpp_threshold_5_total_metric": 0.1902499943971634,
199
  "tpp_threshold_5_intended_diff_only": 0.2630000114440918,
200
- "tpp_threshold_5_unintended_diff_only": 0.0727500170469284,
201
- "tpp_threshold_10_total_metric": 0.20275001227855682,
202
  "tpp_threshold_10_intended_diff_only": 0.32200002670288086,
203
- "tpp_threshold_10_unintended_diff_only": 0.11925001442432404
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.022750020027160645,
207
- "tpp_threshold_2_intended_diff_only": 0.025000035762786865,
208
- "tpp_threshold_2_unintended_diff_only": 0.0022500157356262207,
209
- "tpp_threshold_5_total_metric": 0.19025000929832458,
210
- "tpp_threshold_5_intended_diff_only": 0.24800002574920654,
211
- "tpp_threshold_5_unintended_diff_only": 0.05775001645088196,
212
- "tpp_threshold_10_total_metric": 0.130000039935112,
213
- "tpp_threshold_10_intended_diff_only": 0.33900004625320435,
214
- "tpp_threshold_10_unintended_diff_only": 0.20900000631809235
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.011500000953674316,
218
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
219
- "tpp_threshold_2_unintended_diff_only": -0.002499997615814209,
220
- "tpp_threshold_5_total_metric": 0.25700004398822784,
221
- "tpp_threshold_5_intended_diff_only": 0.33000004291534424,
222
- "tpp_threshold_5_unintended_diff_only": 0.0729999989271164,
223
- "tpp_threshold_10_total_metric": 0.2820000648498535,
224
  "tpp_threshold_10_intended_diff_only": 0.3940000534057617,
225
- "tpp_threshold_10_unintended_diff_only": 0.1119999885559082
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.005250036716461182,
229
- "tpp_threshold_2_intended_diff_only": 0.006000041961669922,
230
- "tpp_threshold_2_unintended_diff_only": 0.0007500052452087402,
231
- "tpp_threshold_5_total_metric": 0.153250053524971,
232
- "tpp_threshold_5_intended_diff_only": 0.21800005435943604,
233
- "tpp_threshold_5_unintended_diff_only": 0.06475000083446503,
234
- "tpp_threshold_10_total_metric": 0.18675000965595245,
235
- "tpp_threshold_10_intended_diff_only": 0.2630000114440918,
236
- "tpp_threshold_10_unintended_diff_only": 0.07625000178813934
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.008000016212463379,
242
- "tpp_threshold_2_intended_diff_only": 0.003000020980834961,
243
- "tpp_threshold_2_unintended_diff_only": -0.004999995231628418,
244
- "tpp_threshold_5_total_metric": 0.00974997878074646,
245
- "tpp_threshold_5_intended_diff_only": 0.006999969482421875,
246
- "tpp_threshold_5_unintended_diff_only": -0.002750009298324585,
247
- "tpp_threshold_10_total_metric": -0.001749977469444275,
248
- "tpp_threshold_10_intended_diff_only": 0.003000020980834961,
249
- "tpp_threshold_10_unintended_diff_only": 0.004749998450279236
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": 0.004250004887580872,
253
- "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
254
- "tpp_threshold_2_unintended_diff_only": 0.004749998450279236,
255
- "tpp_threshold_5_total_metric": 0.013250067830085754,
256
- "tpp_threshold_5_intended_diff_only": 0.01900005340576172,
257
- "tpp_threshold_5_unintended_diff_only": 0.005749985575675964,
258
- "tpp_threshold_10_total_metric": 0.03425005078315735,
259
  "tpp_threshold_10_intended_diff_only": 0.03800004720687866,
260
- "tpp_threshold_10_unintended_diff_only": 0.0037499964237213135
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.01000000536441803,
264
- "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
265
- "tpp_threshold_2_unintended_diff_only": -0.00599999725818634,
266
- "tpp_threshold_5_total_metric": 0.01875002682209015,
267
- "tpp_threshold_5_intended_diff_only": 0.017000019550323486,
268
- "tpp_threshold_5_unintended_diff_only": -0.0017500072717666626,
269
- "tpp_threshold_10_total_metric": 0.014749974012374878,
270
- "tpp_threshold_10_intended_diff_only": 0.015999972820281982,
271
- "tpp_threshold_10_unintended_diff_only": 0.0012499988079071045
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.007500007748603821,
275
- "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
276
- "tpp_threshold_2_unintended_diff_only": 0.00849999487400055,
277
- "tpp_threshold_5_total_metric": 0.0029999464750289917,
278
- "tpp_threshold_5_intended_diff_only": 0.003999948501586914,
279
- "tpp_threshold_5_unintended_diff_only": 0.0010000020265579224,
280
- "tpp_threshold_10_total_metric": 0.010249942541122437,
281
- "tpp_threshold_10_intended_diff_only": 0.012999951839447021,
282
- "tpp_threshold_10_unintended_diff_only": 0.002750009298324585
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.03675001859664917,
286
- "tpp_threshold_2_intended_diff_only": 0.03600001335144043,
287
- "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402,
288
- "tpp_threshold_5_total_metric": 0.03525000810623169,
289
- "tpp_threshold_5_intended_diff_only": 0.041999995708465576,
290
- "tpp_threshold_5_unintended_diff_only": 0.006749987602233887,
291
- "tpp_threshold_10_total_metric": 0.065249964594841,
292
- "tpp_threshold_10_intended_diff_only": 0.06699997186660767,
293
- "tpp_threshold_10_unintended_diff_only": 0.0017500072717666626
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "194fbc19-bfbb-4a57-9342-0a022d09d4d0",
73
+ "datetime_epoch_millis": 1777317310395,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.02252500206232071,
77
+ "tpp_threshold_2_intended_diff_only": 0.025500005483627318,
78
+ "tpp_threshold_2_unintended_diff_only": 0.0029750034213066098,
79
+ "tpp_threshold_5_total_metric": 0.10015000998973846,
80
+ "tpp_threshold_5_intended_diff_only": 0.12900001406669617,
81
+ "tpp_threshold_5_unintended_diff_only": 0.028850004076957703,
82
+ "tpp_threshold_10_total_metric": 0.1085500031709671,
83
+ "tpp_threshold_10_intended_diff_only": 0.16440001130104065,
84
+ "tpp_threshold_10_unintended_diff_only": 0.05585000813007354,
85
+ "tpp_threshold_20_total_metric": 0.13464999347925186,
86
+ "tpp_threshold_20_intended_diff_only": 0.21290000081062316,
87
+ "tpp_threshold_20_unintended_diff_only": 0.07825000733137132,
88
+ "tpp_threshold_50_total_metric": 0.14915000647306442,
89
+ "tpp_threshold_50_intended_diff_only": 0.26150001883506774,
90
+ "tpp_threshold_50_unintended_diff_only": 0.11235001236200333,
91
+ "tpp_threshold_100_total_metric": 0.1778000056743622,
92
+ "tpp_threshold_100_intended_diff_only": 0.32670001983642577,
93
+ "tpp_threshold_100_unintended_diff_only": 0.1489000141620636,
94
+ "tpp_threshold_500_total_metric": 0.2337000250816345,
95
+ "tpp_threshold_500_intended_diff_only": 0.43680003881454466,
96
+ "tpp_threshold_500_unintended_diff_only": 0.20310001373291015
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.03645000159740448,
103
+ "tpp_threshold_2_intended_diff_only": 0.041400015354156494,
104
+ "tpp_threshold_2_unintended_diff_only": 0.004950013756752014,
105
+ "tpp_threshold_5_total_metric": 0.1860500156879425,
106
+ "tpp_threshold_5_intended_diff_only": 0.24320002794265747,
107
+ "tpp_threshold_5_unintended_diff_only": 0.057150012254714964,
108
+ "tpp_threshold_10_total_metric": 0.19125000536441802,
109
  "tpp_threshold_10_intended_diff_only": 0.29980002641677855,
110
+ "tpp_threshold_10_unintended_diff_only": 0.10855002105236053,
111
+ "tpp_threshold_20_total_metric": 0.21470000445842743,
112
+ "tpp_threshold_20_intended_diff_only": 0.3666000247001648,
113
+ "tpp_threshold_20_unintended_diff_only": 0.15190002024173738,
114
+ "tpp_threshold_50_total_metric": 0.19535000622272491,
115
+ "tpp_threshold_50_intended_diff_only": 0.4150000333786011,
116
+ "tpp_threshold_50_unintended_diff_only": 0.21965002715587617,
117
+ "tpp_threshold_100_total_metric": 0.1574999988079071,
118
+ "tpp_threshold_100_intended_diff_only": 0.4402000308036804,
119
+ "tpp_threshold_100_unintended_diff_only": 0.2827000319957733,
120
+ "tpp_threshold_500_total_metric": 0.09405001103878022,
121
+ "tpp_threshold_500_intended_diff_only": 0.4670000433921814,
122
+ "tpp_threshold_500_unintended_diff_only": 0.3729500323534012
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.008600002527236939,
127
+ "tpp_threshold_2_intended_diff_only": 0.009599995613098145,
128
+ "tpp_threshold_2_unintended_diff_only": 0.000999993085861206,
129
+ "tpp_threshold_5_total_metric": 0.014250004291534423,
130
+ "tpp_threshold_5_intended_diff_only": 0.014800000190734863,
131
+ "tpp_threshold_5_unintended_diff_only": 0.0005499958992004394,
132
+ "tpp_threshold_10_total_metric": 0.025850000977516174,
133
+ "tpp_threshold_10_intended_diff_only": 0.028999996185302735,
134
+ "tpp_threshold_10_unintended_diff_only": 0.00314999520778656,
135
+ "tpp_threshold_20_total_metric": 0.05459998250007629,
136
+ "tpp_threshold_20_intended_diff_only": 0.059199976921081546,
137
+ "tpp_threshold_20_unintended_diff_only": 0.004599994421005249,
138
+ "tpp_threshold_50_total_metric": 0.10295000672340393,
139
+ "tpp_threshold_50_intended_diff_only": 0.10800000429153442,
140
+ "tpp_threshold_50_unintended_diff_only": 0.005049997568130493,
141
+ "tpp_threshold_100_total_metric": 0.19810001254081727,
142
+ "tpp_threshold_100_intended_diff_only": 0.21320000886917115,
143
+ "tpp_threshold_100_unintended_diff_only": 0.015099996328353881,
144
+ "tpp_threshold_500_total_metric": 0.37335003912448883,
145
+ "tpp_threshold_500_intended_diff_only": 0.40660003423690794,
146
+ "tpp_threshold_500_unintended_diff_only": 0.03324999511241913
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.019500017166137695,
194
  "tpp_threshold_10_total_metric": 0.1617499738931656,
195
  "tpp_threshold_10_intended_diff_only": 0.1809999942779541,
196
+ "tpp_threshold_10_unintended_diff_only": 0.019250020384788513,
197
+ "tpp_threshold_20_total_metric": 0.21324995160102844,
198
+ "tpp_threshold_20_intended_diff_only": 0.36299997568130493,
199
+ "tpp_threshold_20_unintended_diff_only": 0.1497500240802765,
200
+ "tpp_threshold_50_total_metric": 0.21424999833106995,
201
+ "tpp_threshold_50_intended_diff_only": 0.4190000295639038,
202
+ "tpp_threshold_50_unintended_diff_only": 0.20475003123283386,
203
+ "tpp_threshold_100_total_metric": 0.17949995398521423,
204
+ "tpp_threshold_100_intended_diff_only": 0.44099998474121094,
205
+ "tpp_threshold_100_unintended_diff_only": 0.2615000307559967,
206
+ "tpp_threshold_500_total_metric": 0.08949998021125793,
207
+ "tpp_threshold_500_intended_diff_only": 0.45600003004074097,
208
+ "tpp_threshold_500_unintended_diff_only": 0.36650004982948303
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.05224999785423279,
212
  "tpp_threshold_2_intended_diff_only": 0.05900001525878906,
213
+ "tpp_threshold_2_unintended_diff_only": 0.006750017404556274,
214
+ "tpp_threshold_5_total_metric": 0.19199998676776886,
215
  "tpp_threshold_5_intended_diff_only": 0.2630000114440918,
216
+ "tpp_threshold_5_unintended_diff_only": 0.07100002467632294,
217
+ "tpp_threshold_10_total_metric": 0.1952500194311142,
218
  "tpp_threshold_10_intended_diff_only": 0.32200002670288086,
219
+ "tpp_threshold_10_unintended_diff_only": 0.12675000727176666,
220
+ "tpp_threshold_20_total_metric": 0.2110000103712082,
221
+ "tpp_threshold_20_intended_diff_only": 0.3540000319480896,
222
+ "tpp_threshold_20_unintended_diff_only": 0.1430000215768814,
223
+ "tpp_threshold_50_total_metric": 0.16900001466274261,
224
+ "tpp_threshold_50_intended_diff_only": 0.3940000534057617,
225
+ "tpp_threshold_50_unintended_diff_only": 0.2250000387430191,
226
+ "tpp_threshold_100_total_metric": 0.14649996161460876,
227
+ "tpp_threshold_100_intended_diff_only": 0.4259999990463257,
228
+ "tpp_threshold_100_unintended_diff_only": 0.2795000374317169,
229
+ "tpp_threshold_500_total_metric": 0.08175002038478851,
230
+ "tpp_threshold_500_intended_diff_only": 0.4590000510215759,
231
+ "tpp_threshold_500_unintended_diff_only": 0.3772500306367874
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.01400001347064972,
235
+ "tpp_threshold_2_intended_diff_only": 0.021000027656555176,
236
+ "tpp_threshold_2_unintended_diff_only": 0.0070000141859054565,
237
+ "tpp_threshold_5_total_metric": 0.19050003588199615,
238
+ "tpp_threshold_5_intended_diff_only": 0.24700003862380981,
239
+ "tpp_threshold_5_unintended_diff_only": 0.05650000274181366,
240
+ "tpp_threshold_10_total_metric": 0.132750004529953,
241
+ "tpp_threshold_10_intended_diff_only": 0.3400000333786011,
242
+ "tpp_threshold_10_unintended_diff_only": 0.20725002884864807,
243
+ "tpp_threshold_20_total_metric": 0.12600001692771912,
244
+ "tpp_threshold_20_intended_diff_only": 0.3580000400543213,
245
+ "tpp_threshold_20_unintended_diff_only": 0.23200002312660217,
246
+ "tpp_threshold_50_total_metric": 0.10300001502037048,
247
+ "tpp_threshold_50_intended_diff_only": 0.37700003385543823,
248
+ "tpp_threshold_50_unintended_diff_only": 0.27400001883506775,
249
+ "tpp_threshold_100_total_metric": 0.05925004184246063,
250
+ "tpp_threshold_100_intended_diff_only": 0.4020000696182251,
251
+ "tpp_threshold_100_unintended_diff_only": 0.34275002777576447,
252
+ "tpp_threshold_500_total_metric": 0.05025000870227814,
253
+ "tpp_threshold_500_intended_diff_only": 0.45100003480911255,
254
+ "tpp_threshold_500_unintended_diff_only": 0.4007500261068344
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.009499981999397278,
258
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704,
260
+ "tpp_threshold_5_total_metric": 0.26100002229213715,
261
+ "tpp_threshold_5_intended_diff_only": 0.33100003004074097,
262
+ "tpp_threshold_5_unintended_diff_only": 0.07000000774860382,
263
+ "tpp_threshold_10_total_metric": 0.2827500253915787,
264
  "tpp_threshold_10_intended_diff_only": 0.3940000534057617,
265
+ "tpp_threshold_10_unintended_diff_only": 0.11125002801418304,
266
+ "tpp_threshold_20_total_metric": 0.30000002682209015,
267
+ "tpp_threshold_20_intended_diff_only": 0.4410000443458557,
268
+ "tpp_threshold_20_unintended_diff_only": 0.14100001752376556,
269
+ "tpp_threshold_50_total_metric": 0.265500009059906,
270
+ "tpp_threshold_50_intended_diff_only": 0.4790000319480896,
271
+ "tpp_threshold_50_unintended_diff_only": 0.2135000228881836,
272
+ "tpp_threshold_100_total_metric": 0.20400001108646393,
273
+ "tpp_threshold_100_intended_diff_only": 0.4830000400543213,
274
+ "tpp_threshold_100_unintended_diff_only": 0.27900002896785736,
275
+ "tpp_threshold_500_total_metric": 0.1287499964237213,
276
+ "tpp_threshold_500_intended_diff_only": 0.4930000305175781,
277
+ "tpp_threshold_500_unintended_diff_only": 0.3642500340938568
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.0072500258684158325,
281
+ "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464,
283
+ "tpp_threshold_5_total_metric": 0.15025003254413605,
284
+ "tpp_threshold_5_intended_diff_only": 0.21900004148483276,
285
+ "tpp_threshold_5_unintended_diff_only": 0.06875000894069672,
286
+ "tpp_threshold_10_total_metric": 0.1837500035762787,
287
+ "tpp_threshold_10_intended_diff_only": 0.26200002431869507,
288
+ "tpp_threshold_10_unintended_diff_only": 0.07825002074241638,
289
+ "tpp_threshold_20_total_metric": 0.22325001657009125,
290
+ "tpp_threshold_20_intended_diff_only": 0.31700003147125244,
291
+ "tpp_threshold_20_unintended_diff_only": 0.0937500149011612,
292
+ "tpp_threshold_50_total_metric": 0.22499999403953552,
293
+ "tpp_threshold_50_intended_diff_only": 0.406000018119812,
294
+ "tpp_threshold_50_unintended_diff_only": 0.1810000240802765,
295
+ "tpp_threshold_100_total_metric": 0.19825002551078796,
296
+ "tpp_threshold_100_intended_diff_only": 0.4490000605583191,
297
+ "tpp_threshold_100_unintended_diff_only": 0.25075003504753113,
298
+ "tpp_threshold_500_total_metric": 0.12000004947185516,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.35600002110004425
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.0015000104904174805,
306
+ "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
307
+ "tpp_threshold_2_unintended_diff_only": 0.0034999847412109375,
308
+ "tpp_threshold_5_total_metric": 0.002750009298324585,
309
+ "tpp_threshold_5_intended_diff_only": 0.0040000081062316895,
310
+ "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045,
311
+ "tpp_threshold_10_total_metric": -0.0007500350475311279,
312
+ "tpp_threshold_10_intended_diff_only": 0.001999974250793457,
313
+ "tpp_threshold_10_unintended_diff_only": 0.002750009298324585,
314
+ "tpp_threshold_20_total_metric": 0.011749997735023499,
315
+ "tpp_threshold_20_intended_diff_only": 0.014999985694885254,
316
+ "tpp_threshold_20_unintended_diff_only": 0.0032499879598617554,
317
+ "tpp_threshold_50_total_metric": 0.04874999821186066,
318
+ "tpp_threshold_50_intended_diff_only": 0.046000003814697266,
319
+ "tpp_threshold_50_unintended_diff_only": -0.002749994397163391,
320
+ "tpp_threshold_100_total_metric": 0.11000002920627594,
321
+ "tpp_threshold_100_intended_diff_only": 0.11900001764297485,
322
+ "tpp_threshold_100_unintended_diff_only": 0.008999988436698914,
323
+ "tpp_threshold_500_total_metric": 0.3965000510215759,
324
+ "tpp_threshold_500_intended_diff_only": 0.409000039100647,
325
+ "tpp_threshold_500_unintended_diff_only": 0.012499988079071045
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.013250038027763367,
329
+ "tpp_threshold_2_intended_diff_only": 0.01100003719329834,
330
+ "tpp_threshold_2_unintended_diff_only": -0.002250000834465027,
331
+ "tpp_threshold_5_total_metric": 0.007500007748603821,
332
+ "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
333
+ "tpp_threshold_5_unintended_diff_only": 0.006499990820884705,
334
+ "tpp_threshold_10_total_metric": 0.03250005841255188,
335
  "tpp_threshold_10_intended_diff_only": 0.03800004720687866,
336
+ "tpp_threshold_10_unintended_diff_only": 0.005499988794326782,
337
+ "tpp_threshold_20_total_metric": 0.09225001931190491,
338
+ "tpp_threshold_20_intended_diff_only": 0.09700000286102295,
339
+ "tpp_threshold_20_unintended_diff_only": 0.004749983549118042,
340
+ "tpp_threshold_50_total_metric": 0.1572500616312027,
341
+ "tpp_threshold_50_intended_diff_only": 0.16800004243850708,
342
+ "tpp_threshold_50_unintended_diff_only": 0.010749980807304382,
343
+ "tpp_threshold_100_total_metric": 0.27675004303455353,
344
+ "tpp_threshold_100_intended_diff_only": 0.2980000376701355,
345
+ "tpp_threshold_100_unintended_diff_only": 0.02124999463558197,
346
+ "tpp_threshold_500_total_metric": 0.403000071644783,
347
+ "tpp_threshold_500_intended_diff_only": 0.43900007009506226,
348
+ "tpp_threshold_500_unintended_diff_only": 0.035999998450279236
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.0014999955892562866,
352
+ "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
353
+ "tpp_threshold_2_unintended_diff_only": -0.0005000084638595581,
354
+ "tpp_threshold_5_total_metric": 0.012000039219856262,
355
+ "tpp_threshold_5_intended_diff_only": 0.00700002908706665,
356
+ "tpp_threshold_5_unintended_diff_only": -0.005000010132789612,
357
+ "tpp_threshold_10_total_metric": 0.015499994158744812,
358
+ "tpp_threshold_10_intended_diff_only": 0.013999998569488525,
359
+ "tpp_threshold_10_unintended_diff_only": -0.0014999955892562866,
360
+ "tpp_threshold_20_total_metric": 0.00849999487400055,
361
+ "tpp_threshold_20_intended_diff_only": 0.010999977588653564,
362
+ "tpp_threshold_20_unintended_diff_only": 0.002499982714653015,
363
+ "tpp_threshold_50_total_metric": 0.036500006914138794,
364
+ "tpp_threshold_50_intended_diff_only": 0.03299999237060547,
365
+ "tpp_threshold_50_unintended_diff_only": -0.003500014543533325,
366
+ "tpp_threshold_100_total_metric": 0.10525000095367432,
367
+ "tpp_threshold_100_intended_diff_only": 0.12099999189376831,
368
+ "tpp_threshold_100_unintended_diff_only": 0.015749990940093994,
369
+ "tpp_threshold_500_total_metric": 0.3632500320672989,
370
+ "tpp_threshold_500_intended_diff_only": 0.3960000276565552,
371
+ "tpp_threshold_500_unintended_diff_only": 0.03274999558925629
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.001749977469444275,
375
+ "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
376
+ "tpp_threshold_2_unintended_diff_only": 0.00024999678134918213,
377
+ "tpp_threshold_5_total_metric": 0.010499998927116394,
378
+ "tpp_threshold_5_intended_diff_only": 0.009999990463256836,
379
+ "tpp_threshold_5_unintended_diff_only": -0.0005000084638595581,
380
+ "tpp_threshold_10_total_metric": 0.0032499581575393677,
381
+ "tpp_threshold_10_intended_diff_only": 0.008999943733215332,
382
+ "tpp_threshold_10_unintended_diff_only": 0.005749985575675964,
383
+ "tpp_threshold_20_total_metric": 0.02599993348121643,
384
+ "tpp_threshold_20_intended_diff_only": 0.026999950408935547,
385
+ "tpp_threshold_20_unintended_diff_only": 0.0010000169277191162,
386
+ "tpp_threshold_50_total_metric": 0.05699998140335083,
387
+ "tpp_threshold_50_intended_diff_only": 0.06599998474121094,
388
+ "tpp_threshold_50_unintended_diff_only": 0.009000003337860107,
389
+ "tpp_threshold_100_total_metric": 0.19049999117851257,
390
+ "tpp_threshold_100_intended_diff_only": 0.20399999618530273,
391
+ "tpp_threshold_100_unintended_diff_only": 0.013500005006790161,
392
+ "tpp_threshold_500_total_metric": 0.375,
393
+ "tpp_threshold_500_intended_diff_only": 0.41600000858306885,
394
+ "tpp_threshold_500_unintended_diff_only": 0.04100000858306885
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.024999991059303284,
398
+ "tpp_threshold_2_intended_diff_only": 0.02899998426437378,
399
+ "tpp_threshold_2_unintended_diff_only": 0.003999993205070496,
400
+ "tpp_threshold_5_total_metric": 0.03849996626377106,
401
+ "tpp_threshold_5_intended_diff_only": 0.038999974727630615,
402
+ "tpp_threshold_5_unintended_diff_only": 0.0005000084638595581,
403
+ "tpp_threshold_10_total_metric": 0.07875002920627594,
404
+ "tpp_threshold_10_intended_diff_only": 0.0820000171661377,
405
+ "tpp_threshold_10_unintended_diff_only": 0.0032499879598617554,
406
+ "tpp_threshold_20_total_metric": 0.13449996709823608,
407
+ "tpp_threshold_20_intended_diff_only": 0.1459999680519104,
408
+ "tpp_threshold_20_unintended_diff_only": 0.011500000953674316,
409
+ "tpp_threshold_50_total_metric": 0.21524998545646667,
410
+ "tpp_threshold_50_intended_diff_only": 0.22699999809265137,
411
+ "tpp_threshold_50_unintended_diff_only": 0.011750012636184692,
412
+ "tpp_threshold_100_total_metric": 0.30799999833106995,
413
+ "tpp_threshold_100_intended_diff_only": 0.3240000009536743,
414
+ "tpp_threshold_100_unintended_diff_only": 0.01600000262260437,
415
+ "tpp_threshold_500_total_metric": 0.3290000408887863,
416
+ "tpp_threshold_500_intended_diff_only": 0.37300002574920654,
417
+ "tpp_threshold_500_unintended_diff_only": 0.04399998486042023
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-268554-tokens-1100000000/scr/saebench_scr_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,8 +69,8 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "ae563e9f-2c3f-40c3-8da8-85ecdc76f324",
69
- "datetime_epoch_millis": 1776885575680,
70
  "eval_result_metrics": {
71
  "scr_metrics": {
72
  "scr_dir1_threshold_2": 0.27799565814935673,
@@ -78,18 +82,18 @@
78
  "scr_dir1_threshold_10": 0.09746747674747715,
79
  "scr_metric_threshold_10": 0.20395231432564953,
80
  "scr_dir2_threshold_10": 0.19763325838028084,
81
- "scr_dir1_threshold_20": null,
82
- "scr_metric_threshold_20": null,
83
- "scr_dir2_threshold_20": null,
84
- "scr_dir1_threshold_50": null,
85
- "scr_metric_threshold_50": null,
86
- "scr_dir2_threshold_50": null,
87
- "scr_dir1_threshold_100": null,
88
- "scr_metric_threshold_100": null,
89
- "scr_dir2_threshold_100": null,
90
- "scr_dir1_threshold_500": null,
91
- "scr_metric_threshold_500": null,
92
- "scr_dir2_threshold_500": null
93
  }
94
  },
95
  "eval_result_details": [
@@ -104,18 +108,18 @@
104
  "scr_dir1_threshold_10": 0.3589742805987668,
105
  "scr_metric_threshold_10": 0.046632137153671566,
106
  "scr_dir2_threshold_10": 0.046632137153671566,
107
- "scr_dir1_threshold_20": null,
108
- "scr_metric_threshold_20": null,
109
- "scr_dir2_threshold_20": null,
110
- "scr_dir1_threshold_50": null,
111
- "scr_metric_threshold_50": null,
112
- "scr_dir2_threshold_50": null,
113
- "scr_dir1_threshold_100": null,
114
- "scr_metric_threshold_100": null,
115
- "scr_dir2_threshold_100": null,
116
- "scr_dir1_threshold_500": null,
117
- "scr_metric_threshold_500": null,
118
- "scr_dir2_threshold_500": null
119
  },
120
  {
121
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
@@ -128,18 +132,18 @@
128
  "scr_dir1_threshold_10": 0.15151542518196345,
129
  "scr_metric_threshold_10": 0.14984716057708958,
130
  "scr_dir2_threshold_10": 0.14984716057708958,
131
- "scr_dir1_threshold_20": null,
132
- "scr_metric_threshold_20": null,
133
- "scr_dir2_threshold_20": null,
134
- "scr_dir1_threshold_50": null,
135
- "scr_metric_threshold_50": null,
136
- "scr_dir2_threshold_50": null,
137
- "scr_dir1_threshold_100": null,
138
- "scr_metric_threshold_100": null,
139
- "scr_dir2_threshold_100": null,
140
- "scr_dir1_threshold_500": null,
141
- "scr_metric_threshold_500": null,
142
- "scr_dir2_threshold_500": null
143
  },
144
  {
145
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
@@ -152,18 +156,18 @@
152
  "scr_dir1_threshold_10": 0.1093754365573853,
153
  "scr_metric_threshold_10": 0.10804029093651613,
154
  "scr_dir2_threshold_10": 0.10804029093651613,
155
- "scr_dir1_threshold_20": null,
156
- "scr_metric_threshold_20": null,
157
- "scr_dir2_threshold_20": null,
158
- "scr_dir1_threshold_50": null,
159
- "scr_metric_threshold_50": null,
160
- "scr_dir2_threshold_50": null,
161
- "scr_dir1_threshold_100": null,
162
- "scr_metric_threshold_100": null,
163
- "scr_dir2_threshold_100": null,
164
- "scr_dir1_threshold_500": null,
165
- "scr_metric_threshold_500": null,
166
- "scr_dir2_threshold_500": null
167
  },
168
  {
169
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
@@ -176,18 +180,18 @@
176
  "scr_dir1_threshold_10": -0.9127519946555831,
177
  "scr_metric_threshold_10": 0.06369439446808171,
178
  "scr_dir2_threshold_10": 0.06369439446808171,
179
- "scr_dir1_threshold_20": null,
180
- "scr_metric_threshold_20": null,
181
- "scr_dir2_threshold_20": null,
182
- "scr_dir1_threshold_50": null,
183
- "scr_metric_threshold_50": null,
184
- "scr_dir2_threshold_50": null,
185
- "scr_dir1_threshold_100": null,
186
- "scr_metric_threshold_100": null,
187
- "scr_dir2_threshold_100": null,
188
- "scr_dir1_threshold_500": null,
189
- "scr_metric_threshold_500": null,
190
- "scr_dir2_threshold_500": null
191
  },
192
  {
193
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
@@ -200,18 +204,18 @@
200
  "scr_dir1_threshold_10": 0.09999996862913393,
201
  "scr_metric_threshold_10": 0.4761904311378407,
202
  "scr_dir2_threshold_10": 0.4761904311378407,
203
- "scr_dir1_threshold_20": null,
204
- "scr_metric_threshold_20": null,
205
- "scr_dir2_threshold_20": null,
206
- "scr_dir1_threshold_50": null,
207
- "scr_metric_threshold_50": null,
208
- "scr_dir2_threshold_50": null,
209
- "scr_dir1_threshold_100": null,
210
- "scr_metric_threshold_100": null,
211
- "scr_dir2_threshold_100": null,
212
- "scr_dir1_threshold_500": null,
213
- "scr_metric_threshold_500": null,
214
- "scr_dir2_threshold_500": null
215
  },
216
  {
217
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
@@ -224,18 +228,18 @@
224
  "scr_dir1_threshold_10": 0.4098360709132541,
225
  "scr_metric_threshold_10": 0.14122136275770192,
226
  "scr_dir2_threshold_10": 0.14122136275770192,
227
- "scr_dir1_threshold_20": null,
228
- "scr_metric_threshold_20": null,
229
- "scr_dir2_threshold_20": null,
230
- "scr_dir1_threshold_50": null,
231
- "scr_metric_threshold_50": null,
232
- "scr_dir2_threshold_50": null,
233
- "scr_dir1_threshold_100": null,
234
- "scr_metric_threshold_100": null,
235
- "scr_dir2_threshold_100": null,
236
- "scr_dir1_threshold_500": null,
237
- "scr_metric_threshold_500": null,
238
- "scr_dir2_threshold_500": null
239
  },
240
  {
241
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
@@ -248,18 +252,18 @@
248
  "scr_dir1_threshold_10": 0.3674417856772556,
249
  "scr_metric_threshold_10": 0.4506438964966536,
250
  "scr_dir2_threshold_10": 0.4506438964966536,
251
- "scr_dir1_threshold_20": null,
252
- "scr_metric_threshold_20": null,
253
- "scr_dir2_threshold_20": null,
254
- "scr_dir1_threshold_50": null,
255
- "scr_metric_threshold_50": null,
256
- "scr_dir2_threshold_50": null,
257
- "scr_dir1_threshold_100": null,
258
- "scr_metric_threshold_100": null,
259
- "scr_dir2_threshold_100": null,
260
- "scr_dir1_threshold_500": null,
261
- "scr_metric_threshold_500": null,
262
- "scr_dir2_threshold_500": null
263
  },
264
  {
265
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
@@ -272,18 +276,18 @@
272
  "scr_dir1_threshold_10": 0.19534884107764103,
273
  "scr_metric_threshold_10": 0.19534884107764103,
274
  "scr_dir2_threshold_10": 0.14479639351469145,
275
- "scr_dir1_threshold_20": null,
276
- "scr_metric_threshold_20": null,
277
- "scr_dir2_threshold_20": null,
278
- "scr_dir1_threshold_50": null,
279
- "scr_metric_threshold_50": null,
280
- "scr_dir2_threshold_50": null,
281
- "scr_dir1_threshold_100": null,
282
- "scr_metric_threshold_100": null,
283
- "scr_dir2_threshold_100": null,
284
- "scr_dir1_threshold_500": null,
285
- "scr_metric_threshold_500": null,
286
- "scr_dir2_threshold_500": null
287
  }
288
  ],
289
  "sae_bench_commit_hash": "Unknown",
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "8490fe9e-8ae3-4400-9250-cd9ba5aa8d6e",
73
+ "datetime_epoch_millis": 1777317133212,
74
  "eval_result_metrics": {
75
  "scr_metrics": {
76
  "scr_dir1_threshold_2": 0.27799565814935673,
 
82
  "scr_dir1_threshold_10": 0.09746747674747715,
83
  "scr_metric_threshold_10": 0.20395231432564953,
84
  "scr_dir2_threshold_10": 0.19763325838028084,
85
+ "scr_dir1_threshold_20": 0.06295881921345468,
86
+ "scr_metric_threshold_20": 0.22838102912176494,
87
+ "scr_dir2_threshold_20": 0.21502211012524783,
88
+ "scr_dir1_threshold_50": -0.05904648623138321,
89
+ "scr_metric_threshold_50": 0.3063793988798543,
90
+ "scr_dir2_threshold_50": 0.29803205622013307,
91
+ "scr_dir1_threshold_100": -0.10039104355690062,
92
+ "scr_metric_threshold_100": 0.31131066188410583,
93
+ "scr_dir2_threshold_100": 0.30588607794793926,
94
+ "scr_dir1_threshold_500": -0.9007161627968477,
95
+ "scr_metric_threshold_500": 0.2019885197553268,
96
+ "scr_dir2_threshold_500": 0.19177070157986006
97
  }
98
  },
99
  "eval_result_details": [
 
108
  "scr_dir1_threshold_10": 0.3589742805987668,
109
  "scr_metric_threshold_10": 0.046632137153671566,
110
  "scr_dir2_threshold_10": 0.046632137153671566,
111
+ "scr_dir1_threshold_20": -0.07692284179630039,
112
+ "scr_metric_threshold_20": 0.14248705143262683,
113
+ "scr_dir2_threshold_20": 0.14248705143262683,
114
+ "scr_dir1_threshold_50": 0.39743608357792887,
115
+ "scr_metric_threshold_50": 0.2720207485910297,
116
+ "scr_dir2_threshold_50": 0.2720207485910297,
117
+ "scr_dir1_threshold_100": 0.20512859700616604,
118
+ "scr_metric_threshold_100": 0.2461140400425818,
119
+ "scr_dir2_threshold_100": 0.2461140400425818,
120
+ "scr_dir1_threshold_500": -2.589742805987668,
121
+ "scr_metric_threshold_500": -0.12953369715840288,
122
+ "scr_dir2_threshold_500": -0.12953369715840288
123
  },
124
  {
125
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
 
132
  "scr_dir1_threshold_10": 0.15151542518196345,
133
  "scr_metric_threshold_10": 0.14984716057708958,
134
  "scr_dir2_threshold_10": 0.14984716057708958,
135
+ "scr_dir1_threshold_20": -0.03030308503639269,
136
+ "scr_metric_threshold_20": 0.030581008539686302,
137
+ "scr_dir2_threshold_20": 0.030581008539686302,
138
+ "scr_dir1_threshold_50": 0.03030308503639269,
139
+ "scr_metric_threshold_50": 0.3700305314291443,
140
+ "scr_dir2_threshold_50": 0.3700305314291443,
141
+ "scr_dir1_threshold_100": 0.0,
142
+ "scr_metric_threshold_100": 0.5993884600311694,
143
+ "scr_dir2_threshold_100": 0.5993884600311694,
144
+ "scr_dir1_threshold_500": -1.4924236643031021,
145
+ "scr_metric_threshold_500": 0.1743120403197142,
146
+ "scr_dir2_threshold_500": 0.1743120403197142
147
  },
148
  {
149
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
 
156
  "scr_dir1_threshold_10": 0.1093754365573853,
157
  "scr_metric_threshold_10": 0.10804029093651613,
158
  "scr_dir2_threshold_10": 0.10804029093651613,
159
+ "scr_dir1_threshold_20": 0.17187462165026607,
160
+ "scr_metric_threshold_20": 0.17085429167603294,
161
+ "scr_dir2_threshold_20": 0.17085429167603294,
162
+ "scr_dir1_threshold_50": -1.6874994179234863,
163
+ "scr_metric_threshold_50": 0.16080402160563206,
164
+ "scr_dir2_threshold_50": 0.16080402160563206,
165
+ "scr_dir1_threshold_100": -0.8906245634426148,
166
+ "scr_metric_threshold_100": 0.03266334028870509,
167
+ "scr_dir2_threshold_100": 0.03266334028870509,
168
+ "scr_dir1_threshold_500": -2.5,
169
+ "scr_metric_threshold_500": 0.08291454088031854,
170
+ "scr_dir2_threshold_500": 0.08291454088031854
171
  },
172
  {
173
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
 
180
  "scr_dir1_threshold_10": -0.9127519946555831,
181
  "scr_metric_threshold_10": 0.06369439446808171,
182
  "scr_dir2_threshold_10": 0.06369439446808171,
183
+ "scr_dir1_threshold_20": -0.8322149127631955,
184
+ "scr_metric_threshold_20": -0.05413999801824054,
185
+ "scr_dir2_threshold_20": -0.05413999801824054,
186
+ "scr_dir1_threshold_50": -0.2953022335742188,
187
+ "scr_metric_threshold_50": -0.05732486010942198,
188
+ "scr_dir2_threshold_50": -0.05732486010942198,
189
+ "scr_dir1_threshold_100": -1.2080538281985995,
190
+ "scr_metric_threshold_100": -0.2070063549179335,
191
+ "scr_dir2_threshold_100": -0.2070063549179335,
192
+ "scr_dir1_threshold_500": -1.8053696188302688,
193
+ "scr_metric_threshold_500": -0.5254777577872327,
194
+ "scr_dir2_threshold_500": -0.5254777577872327
195
  },
196
  {
197
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
 
204
  "scr_dir1_threshold_10": 0.09999996862913393,
205
  "scr_metric_threshold_10": 0.4761904311378407,
206
  "scr_dir2_threshold_10": 0.4761904311378407,
207
+ "scr_dir1_threshold_20": 0.12105264313663487,
208
+ "scr_metric_threshold_20": 0.5277777712076017,
209
+ "scr_dir2_threshold_20": 0.5277777712076017,
210
+ "scr_dir1_threshold_50": -0.16842108235134678,
211
+ "scr_metric_threshold_50": 0.6507936996006726,
212
+ "scr_dir2_threshold_50": 0.6507936996006726,
213
+ "scr_dir1_threshold_100": -0.15789458824326602,
214
+ "scr_metric_threshold_100": 0.714285646706761,
215
+ "scr_dir2_threshold_100": 0.714285646706761,
216
+ "scr_dir1_threshold_500": -0.12105264313663487,
217
+ "scr_metric_threshold_500": 0.7857142350300708,
218
+ "scr_dir2_threshold_500": 0.7857142350300708
219
  },
220
  {
221
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
 
228
  "scr_dir1_threshold_10": 0.4098360709132541,
229
  "scr_metric_threshold_10": 0.14122136275770192,
230
  "scr_dir2_threshold_10": 0.14122136275770192,
231
+ "scr_dir1_threshold_20": 0.4153004823742847,
232
+ "scr_metric_threshold_20": 0.19465646684173443,
233
+ "scr_dir2_threshold_20": 0.19465646684173443,
234
+ "scr_dir1_threshold_50": 0.4371584539269045,
235
+ "scr_metric_threshold_50": 0.25954203162185363,
236
+ "scr_dir2_threshold_50": 0.25954203162185363,
237
+ "scr_dir1_threshold_100": 0.4808743970321442,
238
+ "scr_metric_threshold_100": 0.3206106245041952,
239
+ "scr_dir2_threshold_100": 0.3206106245041952,
240
+ "scr_dir1_threshold_500": 0.4098360709132541,
241
+ "scr_metric_threshold_500": 0.31297713570588614,
242
+ "scr_dir2_threshold_500": 0.31297713570588614
243
  },
244
  {
245
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
 
252
  "scr_dir1_threshold_10": 0.3674417856772556,
253
  "scr_metric_threshold_10": 0.4506438964966536,
254
  "scr_dir2_threshold_10": 0.4506438964966536,
255
+ "scr_dir1_threshold_20": 0.46511631775314366,
256
+ "scr_metric_threshold_20": 0.5450643129054811,
257
+ "scr_dir2_threshold_20": 0.5450643129054811,
258
+ "scr_dir1_threshold_50": 0.5209300984557687,
259
+ "scr_metric_threshold_50": 0.5021458952989327,
260
+ "scr_dir2_threshold_50": 0.5021458952989327,
261
+ "scr_dir1_threshold_100": 0.5023254124779394,
262
+ "scr_metric_threshold_100": 0.5193133135043415,
263
+ "scr_dir2_threshold_100": 0.5193133135043415,
264
+ "scr_dir1_threshold_500": 0.548837266037944,
265
+ "scr_metric_threshold_500": 0.5708155681205682,
266
+ "scr_dir2_threshold_500": 0.5708155681205682
267
  },
268
  {
269
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
 
276
  "scr_dir1_threshold_10": 0.19534884107764103,
277
  "scr_metric_threshold_10": 0.19534884107764103,
278
  "scr_dir2_threshold_10": 0.14479639351469145,
279
+ "scr_dir1_threshold_20": 0.26976732838919676,
280
+ "scr_metric_threshold_20": 0.26976732838919676,
281
+ "scr_dir2_threshold_20": 0.16289597641705988,
282
+ "scr_dir1_threshold_50": 0.29302312300099165,
283
+ "scr_metric_threshold_50": 0.29302312300099165,
284
+ "scr_dir2_threshold_50": 0.2262443817232214,
285
+ "scr_dir1_threshold_100": 0.2651162249130257,
286
+ "scr_metric_threshold_100": 0.2651162249130257,
287
+ "scr_dir2_threshold_100": 0.22171955342369326,
288
+ "scr_dir1_threshold_500": 0.3441860929316922,
289
+ "scr_metric_threshold_500": 0.3441860929316922,
290
+ "scr_dir2_threshold_500": 0.26244354752795823
291
  }
292
  ],
293
  "sae_bench_commit_hash": "Unknown",
gemma-2-2b-btk/k-100/seed-0/snapshots/step-268554-tokens-1100000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "22271bd6-3e5d-48e4-ad56-7bdadb5c08f7",
69
- "datetime_epoch_millis": 1776885726256,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.04750001132488251,
73
- "tpp_threshold_2_intended_diff_only": 0.05780001282691956,
74
- "tpp_threshold_2_unintended_diff_only": 0.010300001502037049,
75
- "tpp_threshold_5_total_metric": 0.099875009059906,
76
- "tpp_threshold_5_intended_diff_only": 0.13090001344680785,
77
- "tpp_threshold_5_unintended_diff_only": 0.031025004386901853,
78
- "tpp_threshold_10_total_metric": 0.11487500071525573,
79
- "tpp_threshold_10_intended_diff_only": 0.17110000848770143,
80
- "tpp_threshold_10_unintended_diff_only": 0.05622500777244568,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.08625001311302186,
99
  "tpp_threshold_2_intended_diff_only": 0.10660002231597901,
100
- "tpp_threshold_2_unintended_diff_only": 0.020350009202957153,
101
- "tpp_threshold_5_total_metric": 0.1866000175476074,
102
- "tpp_threshold_5_intended_diff_only": 0.24660003185272217,
103
- "tpp_threshold_5_unintended_diff_only": 0.06000001430511474,
104
- "tpp_threshold_10_total_metric": 0.20535000264644623,
105
  "tpp_threshold_10_intended_diff_only": 0.31500002145767214,
106
- "tpp_threshold_10_unintended_diff_only": 0.1096500188112259,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.008750009536743163,
123
- "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
124
- "tpp_threshold_2_unintended_diff_only": 0.00024999380111694335,
125
- "tpp_threshold_5_total_metric": 0.01315000057220459,
126
- "tpp_threshold_5_intended_diff_only": 0.015199995040893555,
127
- "tpp_threshold_5_unintended_diff_only": 0.0020499944686889647,
128
- "tpp_threshold_10_total_metric": 0.024399998784065246,
129
- "tpp_threshold_10_intended_diff_only": 0.027199995517730714,
130
- "tpp_threshold_10_unintended_diff_only": 0.0027999967336654664,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.018750011920928955,
190
  "tpp_threshold_10_total_metric": 0.18799994885921478,
191
  "tpp_threshold_10_intended_diff_only": 0.21999996900558472,
192
- "tpp_threshold_10_unintended_diff_only": 0.032000020146369934
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": 0.1820000261068344,
196
- "tpp_threshold_2_intended_diff_only": 0.2510000467300415,
197
- "tpp_threshold_2_unintended_diff_only": 0.06900002062320709,
198
- "tpp_threshold_5_total_metric": 0.1885000318288803,
199
  "tpp_threshold_5_intended_diff_only": 0.26500004529953003,
200
- "tpp_threshold_5_unintended_diff_only": 0.07650001347064972,
201
- "tpp_threshold_10_total_metric": 0.21524998545646667,
202
  "tpp_threshold_10_intended_diff_only": 0.3320000171661377,
203
- "tpp_threshold_10_unintended_diff_only": 0.11675003170967102
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.1340000480413437,
207
- "tpp_threshold_2_intended_diff_only": 0.15800005197525024,
208
- "tpp_threshold_2_unintended_diff_only": 0.024000003933906555,
209
- "tpp_threshold_5_total_metric": 0.19050003588199615,
210
- "tpp_threshold_5_intended_diff_only": 0.23200005292892456,
211
- "tpp_threshold_5_unintended_diff_only": 0.041500017046928406,
212
- "tpp_threshold_10_total_metric": 0.1575000137090683,
213
- "tpp_threshold_10_intended_diff_only": 0.3400000333786011,
214
- "tpp_threshold_10_unintended_diff_only": 0.18250001966953278
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.011749997735023499,
218
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
219
- "tpp_threshold_2_unintended_diff_only": -0.002749994397163391,
220
- "tpp_threshold_5_total_metric": 0.2600000500679016,
221
- "tpp_threshold_5_intended_diff_only": 0.3380000591278076,
222
- "tpp_threshold_5_unintended_diff_only": 0.078000009059906,
223
- "tpp_threshold_10_total_metric": 0.27775004506111145,
224
  "tpp_threshold_10_intended_diff_only": 0.3980000615119934,
225
- "tpp_threshold_10_unintended_diff_only": 0.12025001645088196
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.003999993205070496,
229
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
230
- "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224,
231
- "tpp_threshold_5_total_metric": 0.16175001859664917,
232
- "tpp_threshold_5_intended_diff_only": 0.24700003862380981,
233
- "tpp_threshold_5_unintended_diff_only": 0.08525002002716064,
234
- "tpp_threshold_10_total_metric": 0.18825002014636993,
235
- "tpp_threshold_10_intended_diff_only": 0.2850000262260437,
236
- "tpp_threshold_10_unintended_diff_only": 0.09675000607967377
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.008000031113624573,
242
- "tpp_threshold_2_intended_diff_only": 0.003000020980834961,
243
- "tpp_threshold_2_unintended_diff_only": -0.005000010132789612,
244
- "tpp_threshold_5_total_metric": 0.008499979972839355,
245
- "tpp_threshold_5_intended_diff_only": 0.0059999823570251465,
246
- "tpp_threshold_5_unintended_diff_only": -0.002499997615814209,
247
- "tpp_threshold_10_total_metric": 0.0014999806880950928,
248
- "tpp_threshold_10_intended_diff_only": 0.0059999823570251465,
249
- "tpp_threshold_10_unintended_diff_only": 0.004500001668930054
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": 0.0022500455379486084,
253
- "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
254
- "tpp_threshold_2_unintended_diff_only": 0.004749983549118042,
255
- "tpp_threshold_5_total_metric": 0.004500061273574829,
256
- "tpp_threshold_5_intended_diff_only": 0.01100003719329834,
257
- "tpp_threshold_5_unintended_diff_only": 0.006499975919723511,
258
- "tpp_threshold_10_total_metric": 0.027250036597251892,
259
- "tpp_threshold_10_intended_diff_only": 0.030000030994415283,
260
- "tpp_threshold_10_unintended_diff_only": 0.002749994397163391
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.01400001347064972,
264
- "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
265
- "tpp_threshold_2_unintended_diff_only": -0.00599999725818634,
266
- "tpp_threshold_5_total_metric": 0.017999976873397827,
267
- "tpp_threshold_5_intended_diff_only": 0.015999972820281982,
268
- "tpp_threshold_5_unintended_diff_only": -0.0020000040531158447,
269
- "tpp_threshold_10_total_metric": 0.020750001072883606,
270
- "tpp_threshold_10_intended_diff_only": 0.023000001907348633,
271
- "tpp_threshold_10_unintended_diff_only": 0.002250000834465027
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.003750041127204895,
275
  "tpp_threshold_2_intended_diff_only": 0.003999948501586914,
276
- "tpp_threshold_2_unintended_diff_only": 0.007749989628791809,
277
- "tpp_threshold_5_total_metric": 0.0057499706745147705,
278
- "tpp_threshold_5_intended_diff_only": 0.006999969482421875,
279
- "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045,
280
- "tpp_threshold_10_total_metric": 0.018249958753585815,
281
- "tpp_threshold_10_intended_diff_only": 0.02199995517730713,
282
- "tpp_threshold_10_unintended_diff_only": 0.0037499964237213135
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.023249998688697815,
286
- "tpp_threshold_2_intended_diff_only": 0.023000001907348633,
287
- "tpp_threshold_2_unintended_diff_only": -0.00024999678134918213,
288
- "tpp_threshold_5_total_metric": 0.029000014066696167,
289
- "tpp_threshold_5_intended_diff_only": 0.03600001335144043,
290
- "tpp_threshold_5_unintended_diff_only": 0.006999999284744263,
291
- "tpp_threshold_10_total_metric": 0.05425001680850983,
292
- "tpp_threshold_10_intended_diff_only": 0.05500000715255737,
293
- "tpp_threshold_10_unintended_diff_only": 0.0007499903440475464
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "ca94a942-63bd-47ee-a0d0-ad74ff107731",
73
+ "datetime_epoch_millis": 1777317407490,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.046325007081031794,
77
+ "tpp_threshold_2_intended_diff_only": 0.05740000605583191,
78
+ "tpp_threshold_2_unintended_diff_only": 0.01107499897480011,
79
+ "tpp_threshold_5_total_metric": 0.0999000072479248,
80
+ "tpp_threshold_5_intended_diff_only": 0.13030001521110535,
81
+ "tpp_threshold_5_unintended_diff_only": 0.030400007963180542,
82
+ "tpp_threshold_10_total_metric": 0.115425007045269,
83
+ "tpp_threshold_10_intended_diff_only": 0.17250001430511475,
84
+ "tpp_threshold_10_unintended_diff_only": 0.05707500725984573,
85
+ "tpp_threshold_20_total_metric": 0.13175000250339508,
86
+ "tpp_threshold_20_intended_diff_only": 0.2121000111103058,
87
+ "tpp_threshold_20_unintended_diff_only": 0.0803500086069107,
88
+ "tpp_threshold_50_total_metric": 0.15572500377893447,
89
+ "tpp_threshold_50_intended_diff_only": 0.27340001463890073,
90
+ "tpp_threshold_50_unintended_diff_only": 0.11767501085996628,
91
+ "tpp_threshold_100_total_metric": 0.17822501808404922,
92
+ "tpp_threshold_100_intended_diff_only": 0.32950002551078794,
93
+ "tpp_threshold_100_unintended_diff_only": 0.15127500742673874,
94
+ "tpp_threshold_500_total_metric": 0.23397501409053803,
95
+ "tpp_threshold_500_intended_diff_only": 0.43740002512931825,
96
+ "tpp_threshold_500_unintended_diff_only": 0.20342501103878022
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.08535001575946807,
103
  "tpp_threshold_2_intended_diff_only": 0.10660002231597901,
104
+ "tpp_threshold_2_unintended_diff_only": 0.021250006556510926,
105
+ "tpp_threshold_5_total_metric": 0.1865000069141388,
106
+ "tpp_threshold_5_intended_diff_only": 0.24720002412796022,
107
+ "tpp_threshold_5_unintended_diff_only": 0.060700017213821414,
108
+ "tpp_threshold_10_total_metric": 0.20345000326633453,
109
  "tpp_threshold_10_intended_diff_only": 0.31500002145767214,
110
+ "tpp_threshold_10_unintended_diff_only": 0.11155001819133759,
111
+ "tpp_threshold_20_total_metric": 0.2124500036239624,
112
+ "tpp_threshold_20_intended_diff_only": 0.3694000244140625,
113
+ "tpp_threshold_20_unintended_diff_only": 0.1569500207901001,
114
+ "tpp_threshold_50_total_metric": 0.1884000062942505,
115
+ "tpp_threshold_50_intended_diff_only": 0.4172000288963318,
116
+ "tpp_threshold_50_unintended_diff_only": 0.2288000226020813,
117
+ "tpp_threshold_100_total_metric": 0.15235000848770142,
118
+ "tpp_threshold_100_intended_diff_only": 0.44200003147125244,
119
+ "tpp_threshold_100_unintended_diff_only": 0.289650022983551,
120
+ "tpp_threshold_500_total_metric": 0.0913000077009201,
121
+ "tpp_threshold_500_intended_diff_only": 0.4674000382423401,
122
+ "tpp_threshold_500_unintended_diff_only": 0.37610003054142
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.00729999840259552,
127
+ "tpp_threshold_2_intended_diff_only": 0.008199989795684814,
128
+ "tpp_threshold_2_unintended_diff_only": 0.0008999913930892945,
129
+ "tpp_threshold_5_total_metric": 0.013300007581710816,
130
+ "tpp_threshold_5_intended_diff_only": 0.013400006294250488,
131
+ "tpp_threshold_5_unintended_diff_only": 9.999871253967285e-05,
132
+ "tpp_threshold_10_total_metric": 0.027400010824203493,
133
+ "tpp_threshold_10_intended_diff_only": 0.03000000715255737,
134
+ "tpp_threshold_10_unintended_diff_only": 0.0025999963283538817,
135
+ "tpp_threshold_20_total_metric": 0.05105000138282776,
136
+ "tpp_threshold_20_intended_diff_only": 0.054799997806549074,
137
+ "tpp_threshold_20_unintended_diff_only": 0.0037499964237213135,
138
+ "tpp_threshold_50_total_metric": 0.12305000126361847,
139
+ "tpp_threshold_50_intended_diff_only": 0.12960000038146974,
140
+ "tpp_threshold_50_unintended_diff_only": 0.006549999117851257,
141
+ "tpp_threshold_100_total_metric": 0.20410002768039703,
142
+ "tpp_threshold_100_intended_diff_only": 0.2170000195503235,
143
+ "tpp_threshold_100_unintended_diff_only": 0.012899991869926453,
144
+ "tpp_threshold_500_total_metric": 0.37665002048015594,
145
+ "tpp_threshold_500_intended_diff_only": 0.4074000120162964,
146
+ "tpp_threshold_500_unintended_diff_only": 0.030749991536140442
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.018750011920928955,
194
  "tpp_threshold_10_total_metric": 0.18799994885921478,
195
  "tpp_threshold_10_intended_diff_only": 0.21999996900558472,
196
+ "tpp_threshold_10_unintended_diff_only": 0.032000020146369934,
197
+ "tpp_threshold_20_total_metric": 0.21149998903274536,
198
+ "tpp_threshold_20_intended_diff_only": 0.35500001907348633,
199
+ "tpp_threshold_20_unintended_diff_only": 0.14350003004074097,
200
+ "tpp_threshold_50_total_metric": 0.18724992871284485,
201
+ "tpp_threshold_50_intended_diff_only": 0.4139999747276306,
202
+ "tpp_threshold_50_unintended_diff_only": 0.22675004601478577,
203
+ "tpp_threshold_100_total_metric": 0.14524996280670166,
204
+ "tpp_threshold_100_intended_diff_only": 0.4359999895095825,
205
+ "tpp_threshold_100_unintended_diff_only": 0.29075002670288086,
206
+ "tpp_threshold_500_total_metric": 0.08874998986721039,
207
+ "tpp_threshold_500_intended_diff_only": 0.45600003004074097,
208
+ "tpp_threshold_500_unintended_diff_only": 0.3672500401735306
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.18300001323223114,
212
+ "tpp_threshold_2_intended_diff_only": 0.24800002574920654,
213
+ "tpp_threshold_2_unintended_diff_only": 0.0650000125169754,
214
+ "tpp_threshold_5_total_metric": 0.18825002014636993,
215
  "tpp_threshold_5_intended_diff_only": 0.26500004529953003,
216
+ "tpp_threshold_5_unintended_diff_only": 0.0767500251531601,
217
+ "tpp_threshold_10_total_metric": 0.2095000147819519,
218
  "tpp_threshold_10_intended_diff_only": 0.3320000171661377,
219
+ "tpp_threshold_10_unintended_diff_only": 0.12250000238418579,
220
+ "tpp_threshold_20_total_metric": 0.20500001311302185,
221
+ "tpp_threshold_20_intended_diff_only": 0.3540000319480896,
222
+ "tpp_threshold_20_unintended_diff_only": 0.14900001883506775,
223
+ "tpp_threshold_50_total_metric": 0.19350001215934753,
224
+ "tpp_threshold_50_intended_diff_only": 0.3960000276565552,
225
+ "tpp_threshold_50_unintended_diff_only": 0.20250001549720764,
226
+ "tpp_threshold_100_total_metric": 0.1745000034570694,
227
+ "tpp_threshold_100_intended_diff_only": 0.437000036239624,
228
+ "tpp_threshold_100_unintended_diff_only": 0.2625000327825546,
229
+ "tpp_threshold_500_total_metric": 0.08075001835823059,
230
+ "tpp_threshold_500_intended_diff_only": 0.4590000510215759,
231
+ "tpp_threshold_500_unintended_diff_only": 0.37825003266334534
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.12850002944469452,
235
+ "tpp_threshold_2_intended_diff_only": 0.15900003910064697,
236
+ "tpp_threshold_2_unintended_diff_only": 0.030500009655952454,
237
+ "tpp_threshold_5_total_metric": 0.1912500262260437,
238
+ "tpp_threshold_5_intended_diff_only": 0.2330000400543213,
239
+ "tpp_threshold_5_unintended_diff_only": 0.04175001382827759,
240
+ "tpp_threshold_10_total_metric": 0.15574999153614044,
241
+ "tpp_threshold_10_intended_diff_only": 0.3410000205039978,
242
+ "tpp_threshold_10_unintended_diff_only": 0.18525002896785736,
243
+ "tpp_threshold_20_total_metric": 0.13650000095367432,
244
+ "tpp_threshold_20_intended_diff_only": 0.36000001430511475,
245
+ "tpp_threshold_20_unintended_diff_only": 0.22350001335144043,
246
+ "tpp_threshold_50_total_metric": 0.09500002861022949,
247
+ "tpp_threshold_50_intended_diff_only": 0.3850000500679016,
248
+ "tpp_threshold_50_unintended_diff_only": 0.2900000214576721,
249
+ "tpp_threshold_100_total_metric": 0.0700000524520874,
250
+ "tpp_threshold_100_intended_diff_only": 0.4070000648498535,
251
+ "tpp_threshold_100_unintended_diff_only": 0.3370000123977661,
252
+ "tpp_threshold_500_total_metric": 0.0495000034570694,
253
+ "tpp_threshold_500_intended_diff_only": 0.4520000219345093,
254
+ "tpp_threshold_500_unintended_diff_only": 0.4025000184774399
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.009749993681907654,
258
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0007499903440475464,
260
+ "tpp_threshold_5_total_metric": 0.26350001990795135,
261
+ "tpp_threshold_5_intended_diff_only": 0.33900004625320435,
262
+ "tpp_threshold_5_unintended_diff_only": 0.07550002634525299,
263
+ "tpp_threshold_10_total_metric": 0.27900004386901855,
264
  "tpp_threshold_10_intended_diff_only": 0.3980000615119934,
265
+ "tpp_threshold_10_unintended_diff_only": 0.11900001764297485,
266
+ "tpp_threshold_20_total_metric": 0.28725001215934753,
267
+ "tpp_threshold_20_intended_diff_only": 0.45100003480911255,
268
+ "tpp_threshold_20_unintended_diff_only": 0.16375002264976501,
269
+ "tpp_threshold_50_total_metric": 0.24950005114078522,
270
+ "tpp_threshold_50_intended_diff_only": 0.48100006580352783,
271
+ "tpp_threshold_50_unintended_diff_only": 0.23150001466274261,
272
+ "tpp_threshold_100_total_metric": 0.19350002706050873,
273
+ "tpp_threshold_100_intended_diff_only": 0.4830000400543213,
274
+ "tpp_threshold_100_unintended_diff_only": 0.28950001299381256,
275
+ "tpp_threshold_500_total_metric": 0.12299998104572296,
276
+ "tpp_threshold_500_intended_diff_only": 0.49400001764297485,
277
+ "tpp_threshold_500_unintended_diff_only": 0.3710000365972519
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.006000041961669922,
281
+ "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0009999871253967285,
283
+ "tpp_threshold_5_total_metric": 0.15725001692771912,
284
+ "tpp_threshold_5_intended_diff_only": 0.24800002574920654,
285
+ "tpp_threshold_5_unintended_diff_only": 0.09075000882148743,
286
+ "tpp_threshold_10_total_metric": 0.18500001728534698,
287
+ "tpp_threshold_10_intended_diff_only": 0.284000039100647,
288
+ "tpp_threshold_10_unintended_diff_only": 0.09900002181529999,
289
+ "tpp_threshold_20_total_metric": 0.22200000286102295,
290
+ "tpp_threshold_20_intended_diff_only": 0.3270000219345093,
291
+ "tpp_threshold_20_unintended_diff_only": 0.10500001907348633,
292
+ "tpp_threshold_50_total_metric": 0.21675001084804535,
293
+ "tpp_threshold_50_intended_diff_only": 0.4100000262260437,
294
+ "tpp_threshold_50_unintended_diff_only": 0.19325001537799835,
295
+ "tpp_threshold_100_total_metric": 0.1784999966621399,
296
+ "tpp_threshold_100_intended_diff_only": 0.44700002670288086,
297
+ "tpp_threshold_100_unintended_diff_only": 0.26850003004074097,
298
+ "tpp_threshold_500_total_metric": 0.11450004577636719,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.3615000247955322
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.002499997615814209,
306
+ "tpp_threshold_2_intended_diff_only": 0.0059999823570251465,
307
+ "tpp_threshold_2_unintended_diff_only": 0.0034999847412109375,
308
+ "tpp_threshold_5_total_metric": 0.0022500157356262207,
309
+ "tpp_threshold_5_intended_diff_only": 0.003000020980834961,
310
+ "tpp_threshold_5_unintended_diff_only": 0.0007500052452087402,
311
+ "tpp_threshold_10_total_metric": 0.0029999911785125732,
312
+ "tpp_threshold_10_intended_diff_only": 0.004999995231628418,
313
+ "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447,
314
+ "tpp_threshold_20_total_metric": 0.01199999451637268,
315
+ "tpp_threshold_20_intended_diff_only": 0.013999998569488525,
316
+ "tpp_threshold_20_unintended_diff_only": 0.0020000040531158447,
317
+ "tpp_threshold_50_total_metric": 0.036500006914138794,
318
+ "tpp_threshold_50_intended_diff_only": 0.03600001335144043,
319
+ "tpp_threshold_50_unintended_diff_only": -0.0004999935626983643,
320
+ "tpp_threshold_100_total_metric": 0.09850002825260162,
321
+ "tpp_threshold_100_intended_diff_only": 0.1040000319480896,
322
+ "tpp_threshold_100_unintended_diff_only": 0.005500003695487976,
323
+ "tpp_threshold_500_total_metric": 0.4007500559091568,
324
+ "tpp_threshold_500_intended_diff_only": 0.4140000343322754,
325
+ "tpp_threshold_500_unintended_diff_only": 0.013249978423118591
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.011250019073486328,
329
+ "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
330
+ "tpp_threshold_2_unintended_diff_only": -0.0022500157356262207,
331
+ "tpp_threshold_5_total_metric": 0.006750032305717468,
332
+ "tpp_threshold_5_intended_diff_only": 0.013000011444091797,
333
+ "tpp_threshold_5_unintended_diff_only": 0.006249979138374329,
334
+ "tpp_threshold_10_total_metric": 0.028500065207481384,
335
+ "tpp_threshold_10_intended_diff_only": 0.033000051975250244,
336
+ "tpp_threshold_10_unintended_diff_only": 0.00449998676776886,
337
+ "tpp_threshold_20_total_metric": 0.08800001442432404,
338
+ "tpp_threshold_20_intended_diff_only": 0.09200000762939453,
339
+ "tpp_threshold_20_unintended_diff_only": 0.003999993205070496,
340
+ "tpp_threshold_50_total_metric": 0.16875003278255463,
341
+ "tpp_threshold_50_intended_diff_only": 0.18000000715255737,
342
+ "tpp_threshold_50_unintended_diff_only": 0.011249974370002747,
343
+ "tpp_threshold_100_total_metric": 0.26725006103515625,
344
+ "tpp_threshold_100_intended_diff_only": 0.28300005197525024,
345
+ "tpp_threshold_100_unintended_diff_only": 0.015749990940093994,
346
+ "tpp_threshold_500_total_metric": 0.4047500491142273,
347
+ "tpp_threshold_500_intended_diff_only": 0.437000036239624,
348
+ "tpp_threshold_500_unintended_diff_only": 0.03224998712539673
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.003750026226043701,
352
+ "tpp_threshold_2_intended_diff_only": 0.003000020980834961,
353
+ "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402,
354
+ "tpp_threshold_5_total_metric": 0.013500005006790161,
355
+ "tpp_threshold_5_intended_diff_only": 0.008000016212463379,
356
+ "tpp_threshold_5_unintended_diff_only": -0.005499988794326782,
357
+ "tpp_threshold_10_total_metric": 0.02124999463558197,
358
+ "tpp_threshold_10_intended_diff_only": 0.018999993801116943,
359
+ "tpp_threshold_10_unintended_diff_only": -0.002250000834465027,
360
+ "tpp_threshold_20_total_metric": 0.010000035166740417,
361
+ "tpp_threshold_20_intended_diff_only": 0.012000024318695068,
362
+ "tpp_threshold_20_unintended_diff_only": 0.001999989151954651,
363
+ "tpp_threshold_50_total_metric": 0.06225000321865082,
364
+ "tpp_threshold_50_intended_diff_only": 0.06400001049041748,
365
+ "tpp_threshold_50_unintended_diff_only": 0.0017500072717666626,
366
+ "tpp_threshold_100_total_metric": 0.12650001049041748,
367
+ "tpp_threshold_100_intended_diff_only": 0.14300000667572021,
368
+ "tpp_threshold_100_unintended_diff_only": 0.016499996185302734,
369
+ "tpp_threshold_500_total_metric": 0.3659999966621399,
370
+ "tpp_threshold_500_intended_diff_only": 0.39399999380111694,
371
+ "tpp_threshold_500_unintended_diff_only": 0.02799999713897705
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.004249945282936096,
375
  "tpp_threshold_2_intended_diff_only": 0.003999948501586914,
376
+ "tpp_threshold_2_unintended_diff_only": -0.00024999678134918213,
377
+ "tpp_threshold_5_total_metric": 0.011499971151351929,
378
+ "tpp_threshold_5_intended_diff_only": 0.010999977588653564,
379
+ "tpp_threshold_5_unintended_diff_only": -0.0004999935626983643,
380
+ "tpp_threshold_10_total_metric": 0.016250014305114746,
381
+ "tpp_threshold_10_intended_diff_only": 0.023000001907348633,
382
+ "tpp_threshold_10_unintended_diff_only": 0.006749987602233887,
383
+ "tpp_threshold_20_total_metric": 0.04149998724460602,
384
+ "tpp_threshold_20_intended_diff_only": 0.042999982833862305,
385
+ "tpp_threshold_20_unintended_diff_only": 0.0014999955892562866,
386
+ "tpp_threshold_50_total_metric": 0.15074999630451202,
387
+ "tpp_threshold_50_intended_diff_only": 0.16200000047683716,
388
+ "tpp_threshold_50_unintended_diff_only": 0.011250004172325134,
389
+ "tpp_threshold_100_total_metric": 0.24775002896785736,
390
+ "tpp_threshold_100_intended_diff_only": 0.2630000114440918,
391
+ "tpp_threshold_100_unintended_diff_only": 0.015249982476234436,
392
+ "tpp_threshold_500_total_metric": 0.37599997222423553,
393
+ "tpp_threshold_500_intended_diff_only": 0.41899996995925903,
394
+ "tpp_threshold_500_unintended_diff_only": 0.0429999977350235
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.014750003814697266,
398
+ "tpp_threshold_2_intended_diff_only": 0.018999993801116943,
399
+ "tpp_threshold_2_unintended_diff_only": 0.004249989986419678,
400
+ "tpp_threshold_5_total_metric": 0.0325000137090683,
401
+ "tpp_threshold_5_intended_diff_only": 0.03200000524520874,
402
+ "tpp_threshold_5_unintended_diff_only": -0.0005000084638595581,
403
+ "tpp_threshold_10_total_metric": 0.06799998879432678,
404
+ "tpp_threshold_10_intended_diff_only": 0.06999999284744263,
405
+ "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447,
406
+ "tpp_threshold_20_total_metric": 0.10374997556209564,
407
+ "tpp_threshold_20_intended_diff_only": 0.11299997568130493,
408
+ "tpp_threshold_20_unintended_diff_only": 0.00925000011920929,
409
+ "tpp_threshold_50_total_metric": 0.19699996709823608,
410
+ "tpp_threshold_50_intended_diff_only": 0.2059999704360962,
411
+ "tpp_threshold_50_unintended_diff_only": 0.009000003337860107,
412
+ "tpp_threshold_100_total_metric": 0.28050000965595245,
413
+ "tpp_threshold_100_intended_diff_only": 0.2919999957084656,
414
+ "tpp_threshold_100_unintended_diff_only": 0.011499986052513123,
415
+ "tpp_threshold_500_total_metric": 0.3357500284910202,
416
+ "tpp_threshold_500_intended_diff_only": 0.37300002574920654,
417
+ "tpp_threshold_500_unintended_diff_only": 0.03724999725818634
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-292968-tokens-1200000000/scr/saebench_scr_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,8 +69,8 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "0bf9147d-f57c-4585-96a0-148907394839",
69
- "datetime_epoch_millis": 1776885956411,
70
  "eval_result_metrics": {
71
  "scr_metrics": {
72
  "scr_dir1_threshold_2": 0.2829853198662318,
@@ -78,18 +82,18 @@
78
  "scr_dir1_threshold_10": 0.27287744671552183,
79
  "scr_metric_threshold_10": 0.19882804859098052,
80
  "scr_dir2_threshold_10": 0.18966515684829438,
81
- "scr_dir1_threshold_20": null,
82
- "scr_metric_threshold_20": null,
83
- "scr_dir2_threshold_20": null,
84
- "scr_dir1_threshold_50": null,
85
- "scr_metric_threshold_50": null,
86
- "scr_dir2_threshold_50": null,
87
- "scr_dir1_threshold_100": null,
88
- "scr_metric_threshold_100": null,
89
- "scr_dir2_threshold_100": null,
90
- "scr_dir1_threshold_500": null,
91
- "scr_metric_threshold_500": null,
92
- "scr_dir2_threshold_500": null
93
  }
94
  },
95
  "eval_result_details": [
@@ -104,18 +108,18 @@
104
  "scr_dir1_threshold_10": 0.43589788655709094,
105
  "scr_metric_threshold_10": 0.03367878287944761,
106
  "scr_dir2_threshold_10": 0.03367878287944761,
107
- "scr_dir1_threshold_20": null,
108
- "scr_metric_threshold_20": null,
109
- "scr_dir2_threshold_20": null,
110
- "scr_dir1_threshold_50": null,
111
- "scr_metric_threshold_50": null,
112
- "scr_dir2_threshold_50": null,
113
- "scr_dir1_threshold_100": null,
114
- "scr_metric_threshold_100": null,
115
- "scr_dir2_threshold_100": null,
116
- "scr_dir1_threshold_500": null,
117
- "scr_metric_threshold_500": null,
118
- "scr_dir2_threshold_500": null
119
  },
120
  {
121
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
@@ -128,18 +132,18 @@
128
  "scr_dir1_threshold_10": 0.3484848005931564,
129
  "scr_metric_threshold_10": 0.25688069046599166,
130
  "scr_dir2_threshold_10": 0.25688069046599166,
131
- "scr_dir1_threshold_20": null,
132
- "scr_metric_threshold_20": null,
133
- "scr_dir2_threshold_20": null,
134
- "scr_dir1_threshold_50": null,
135
- "scr_metric_threshold_50": null,
136
- "scr_dir2_threshold_50": null,
137
- "scr_dir1_threshold_100": null,
138
- "scr_metric_threshold_100": null,
139
- "scr_dir2_threshold_100": null,
140
- "scr_dir1_threshold_500": null,
141
- "scr_metric_threshold_500": null,
142
- "scr_dir2_threshold_500": null
143
  },
144
  {
145
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
@@ -152,18 +156,18 @@
152
  "scr_dir1_threshold_10": 0.5,
153
  "scr_metric_threshold_10": 0.11055282101401862,
154
  "scr_dir2_threshold_10": 0.11055282101401862,
155
- "scr_dir1_threshold_20": null,
156
- "scr_metric_threshold_20": null,
157
- "scr_dir2_threshold_20": null,
158
- "scr_dir1_threshold_50": null,
159
- "scr_metric_threshold_50": null,
160
- "scr_dir2_threshold_50": null,
161
- "scr_dir1_threshold_100": null,
162
- "scr_metric_threshold_100": null,
163
- "scr_dir2_threshold_100": null,
164
- "scr_dir1_threshold_500": null,
165
- "scr_metric_threshold_500": null,
166
- "scr_dir2_threshold_500": null
167
  },
168
  {
169
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
@@ -176,18 +180,18 @@
176
  "scr_dir1_threshold_10": -0.1543624402703411,
177
  "scr_metric_threshold_10": 0.06687906673556002,
178
  "scr_dir2_threshold_10": 0.06687906673556002,
179
- "scr_dir1_threshold_20": null,
180
- "scr_metric_threshold_20": null,
181
- "scr_dir2_threshold_20": null,
182
- "scr_dir1_threshold_50": null,
183
- "scr_metric_threshold_50": null,
184
- "scr_dir2_threshold_50": null,
185
- "scr_dir1_threshold_100": null,
186
- "scr_metric_threshold_100": null,
187
- "scr_dir2_threshold_100": null,
188
- "scr_dir1_threshold_500": null,
189
- "scr_metric_threshold_500": null,
190
- "scr_dir2_threshold_500": null
191
  },
192
  {
193
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
@@ -200,18 +204,18 @@
200
  "scr_dir1_threshold_10": 0.08421069803000372,
201
  "scr_metric_threshold_10": 0.45238098053884956,
202
  "scr_dir2_threshold_10": 0.45238098053884956,
203
- "scr_dir1_threshold_20": null,
204
- "scr_metric_threshold_20": null,
205
- "scr_dir2_threshold_20": null,
206
- "scr_dir1_threshold_50": null,
207
- "scr_metric_threshold_50": null,
208
- "scr_dir2_threshold_50": null,
209
- "scr_dir1_threshold_100": null,
210
- "scr_metric_threshold_100": null,
211
- "scr_dir2_threshold_100": null,
212
- "scr_dir1_threshold_500": null,
213
- "scr_metric_threshold_500": null,
214
- "scr_dir2_threshold_500": null
215
  },
216
  {
217
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
@@ -224,18 +228,18 @@
224
  "scr_dir1_threshold_10": 0.4153004823742847,
225
  "scr_metric_threshold_10": 0.1183206688641517,
226
  "scr_dir2_threshold_10": 0.1183206688641517,
227
- "scr_dir1_threshold_20": null,
228
- "scr_metric_threshold_20": null,
229
- "scr_dir2_threshold_20": null,
230
- "scr_dir1_threshold_50": null,
231
- "scr_metric_threshold_50": null,
232
- "scr_dir2_threshold_50": null,
233
- "scr_dir1_threshold_100": null,
234
- "scr_metric_threshold_100": null,
235
- "scr_dir2_threshold_100": null,
236
- "scr_dir1_threshold_500": null,
237
- "scr_metric_threshold_500": null,
238
- "scr_dir2_threshold_500": null
239
  },
240
  {
241
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
@@ -248,18 +252,18 @@
248
  "scr_dir1_threshold_10": 0.3534882018861679,
249
  "scr_metric_threshold_10": 0.35193143367601315,
250
  "scr_dir2_threshold_10": 0.35193143367601315,
251
- "scr_dir1_threshold_20": null,
252
- "scr_metric_threshold_20": null,
253
- "scr_dir2_threshold_20": null,
254
- "scr_dir1_threshold_50": null,
255
- "scr_metric_threshold_50": null,
256
- "scr_dir2_threshold_50": null,
257
- "scr_dir1_threshold_100": null,
258
- "scr_metric_threshold_100": null,
259
- "scr_dir2_threshold_100": null,
260
- "scr_dir1_threshold_500": null,
261
- "scr_metric_threshold_500": null,
262
- "scr_dir2_threshold_500": null
263
  },
264
  {
265
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
@@ -272,18 +276,18 @@
272
  "scr_dir1_threshold_10": 0.19999994455381204,
273
  "scr_metric_threshold_10": 0.19999994455381204,
274
  "scr_dir2_threshold_10": 0.12669681061232305,
275
- "scr_dir1_threshold_20": null,
276
- "scr_metric_threshold_20": null,
277
- "scr_dir2_threshold_20": null,
278
- "scr_dir1_threshold_50": null,
279
- "scr_metric_threshold_50": null,
280
- "scr_dir2_threshold_50": null,
281
- "scr_dir1_threshold_100": null,
282
- "scr_metric_threshold_100": null,
283
- "scr_dir2_threshold_100": null,
284
- "scr_dir1_threshold_500": null,
285
- "scr_metric_threshold_500": null,
286
- "scr_dir2_threshold_500": null
287
  }
288
  ],
289
  "sae_bench_commit_hash": "Unknown",
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "a5f0acc4-c047-4749-a48d-7e4d190221e6",
73
+ "datetime_epoch_millis": 1777317150738,
74
  "eval_result_metrics": {
75
  "scr_metrics": {
76
  "scr_dir1_threshold_2": 0.2829853198662318,
 
82
  "scr_dir1_threshold_10": 0.27287744671552183,
83
  "scr_metric_threshold_10": 0.19882804859098052,
84
  "scr_dir2_threshold_10": 0.18966515684829438,
85
+ "scr_dir1_threshold_20": 0.2991269431541925,
86
+ "scr_metric_threshold_20": 0.21957360880872895,
87
+ "scr_dir2_threshold_20": 0.20910587974160583,
88
+ "scr_dir1_threshold_50": 0.15900130660025805,
89
+ "scr_metric_threshold_50": 0.26660734469158326,
90
+ "scr_dir2_threshold_50": 0.253608829247956,
91
+ "scr_dir1_threshold_100": -0.04134676619549471,
92
+ "scr_metric_threshold_100": 0.26784659140862055,
93
+ "scr_dir2_threshold_100": 0.25048102609187584,
94
+ "scr_dir1_threshold_500": -1.076120057194125,
95
+ "scr_metric_threshold_500": 0.2294901639413055,
96
+ "scr_dir2_threshold_500": 0.227914380472892
97
  }
98
  },
99
  "eval_result_details": [
 
108
  "scr_dir1_threshold_10": 0.43589788655709094,
109
  "scr_metric_threshold_10": 0.03367878287944761,
110
  "scr_dir2_threshold_10": 0.03367878287944761,
111
+ "scr_dir1_threshold_20": 0.24359039998532808,
112
+ "scr_metric_threshold_20": 0.16580312000946262,
113
+ "scr_dir2_threshold_20": 0.16580312000946262,
114
+ "scr_dir1_threshold_50": 0.6025646805840948,
115
+ "scr_metric_threshold_50": 0.25906739431680575,
116
+ "scr_dir2_threshold_50": 0.25906739431680575,
117
+ "scr_dir1_threshold_100": 0.0,
118
+ "scr_metric_threshold_100": 0.2461140400425818,
119
+ "scr_dir2_threshold_100": 0.2461140400425818,
120
+ "scr_dir1_threshold_500": -3.0384610388171382,
121
+ "scr_metric_threshold_500": 0.3108808114137016,
122
+ "scr_dir2_threshold_500": 0.3108808114137016
123
  },
124
  {
125
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
 
132
  "scr_dir1_threshold_10": 0.3484848005931564,
133
  "scr_metric_threshold_10": 0.25688069046599166,
134
  "scr_dir2_threshold_10": 0.25688069046599166,
135
+ "scr_dir1_threshold_20": 0.38636354400108736,
136
+ "scr_metric_threshold_20": 0.14373084950283896,
137
+ "scr_dir2_threshold_20": 0.14373084950283896,
138
+ "scr_dir1_threshold_50": 0.1439393152601855,
139
+ "scr_metric_threshold_50": 0.21406724205499303,
140
+ "scr_dir2_threshold_50": 0.21406724205499303,
141
+ "scr_dir1_threshold_100": -0.07575748681586188,
142
+ "scr_metric_threshold_100": 0.3944954111717689,
143
+ "scr_dir2_threshold_100": 0.3944954111717689,
144
+ "scr_dir1_threshold_500": -0.8484845748180365,
145
+ "scr_metric_threshold_500": 0.13761472070577727,
146
+ "scr_dir2_threshold_500": 0.13761472070577727
147
  },
148
  {
149
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
 
156
  "scr_dir1_threshold_10": 0.5,
157
  "scr_metric_threshold_10": 0.11055282101401862,
158
  "scr_dir2_threshold_10": 0.11055282101401862,
159
+ "scr_dir1_threshold_20": 0.4062502910382569,
160
+ "scr_metric_threshold_20": 0.17085429167603294,
161
+ "scr_dir2_threshold_20": 0.17085429167603294,
162
+ "scr_dir1_threshold_50": -0.7187499417923486,
163
+ "scr_metric_threshold_50": 0.16582923152102794,
164
+ "scr_dir2_threshold_50": 0.16582923152102794,
165
+ "scr_dir1_threshold_100": -0.3437501746229541,
166
+ "scr_metric_threshold_100": 0.05276388042950681,
167
+ "scr_dir2_threshold_100": 0.05276388042950681,
168
+ "scr_dir1_threshold_500": -4.124999301508184,
169
+ "scr_metric_threshold_500": 0.09799002086611526,
170
+ "scr_dir2_threshold_500": 0.09799002086611526
171
  },
172
  {
173
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
 
180
  "scr_dir1_threshold_10": -0.1543624402703411,
181
  "scr_metric_threshold_10": 0.06687906673556002,
182
  "scr_dir2_threshold_10": 0.06687906673556002,
183
+ "scr_dir1_threshold_20": 0.11409369930854607,
184
+ "scr_metric_threshold_20": -0.10191084132522761,
185
+ "scr_dir2_threshold_20": -0.10191084132522761,
186
+ "scr_dir1_threshold_50": 0.17449641072003616,
187
+ "scr_metric_threshold_50": -0.11464972021884395,
188
+ "scr_dir2_threshold_50": -0.11464972021884395,
189
+ "scr_dir1_threshold_100": -1.0536913879282583,
190
+ "scr_metric_threshold_100": -0.2388534572401228,
191
+ "scr_dir2_threshold_100": -0.2388534572401228,
192
+ "scr_dir1_threshold_500": -1.8389266362776298,
193
+ "scr_metric_threshold_500": -0.5222928956960512,
194
+ "scr_dir2_threshold_500": -0.5222928956960512
195
  },
196
  {
197
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
 
204
  "scr_dir1_threshold_10": 0.08421069803000372,
205
  "scr_metric_threshold_10": 0.45238098053884956,
206
  "scr_dir2_threshold_10": 0.45238098053884956,
207
+ "scr_dir1_threshold_20": 0.11578955293692479,
208
+ "scr_metric_threshold_20": 0.543650817115708,
209
+ "scr_dir2_threshold_20": 0.543650817115708,
210
+ "scr_dir1_threshold_50": -0.15263149804355594,
211
+ "scr_metric_threshold_50": 0.6587301042915575,
212
+ "scr_dir2_threshold_50": 0.6587301042915575,
213
+ "scr_dir1_threshold_100": -0.15263149804355594,
214
+ "scr_metric_threshold_100": 0.6785713525451061,
215
+ "scr_dir2_threshold_100": 0.6785713525451061,
216
+ "scr_dir1_threshold_500": -0.19999993725826787,
217
+ "scr_metric_threshold_500": 0.773809391467407,
218
+ "scr_dir2_threshold_500": 0.773809391467407
219
  },
220
  {
221
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
 
228
  "scr_dir1_threshold_10": 0.4153004823742847,
229
  "scr_metric_threshold_10": 0.1183206688641517,
230
  "scr_dir2_threshold_10": 0.1183206688641517,
231
+ "scr_dir1_threshold_20": 0.4153004823742847,
232
+ "scr_metric_threshold_20": 0.17557251734733875,
233
+ "scr_dir2_threshold_20": 0.17557251734733875,
234
+ "scr_dir1_threshold_50": 0.4316940424658739,
235
+ "scr_metric_threshold_50": 0.26335877602100816,
236
+ "scr_dir2_threshold_50": 0.26335877602100816,
237
+ "scr_dir1_threshold_100": 0.5136611915068252,
238
+ "scr_metric_threshold_100": 0.3396945739985909,
239
+ "scr_dir2_threshold_100": 0.3396945739985909,
240
+ "scr_dir1_threshold_500": 0.5628415460730956,
241
+ "scr_metric_threshold_500": 0.29389318621149046,
242
+ "scr_dir2_threshold_500": 0.29389318621149046
243
  },
244
  {
245
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
 
252
  "scr_dir1_threshold_10": 0.3534882018861679,
253
  "scr_metric_threshold_10": 0.35193143367601315,
254
  "scr_dir2_threshold_10": 0.35193143367601315,
255
+ "scr_dir1_threshold_20": 0.46046493833553914,
256
+ "scr_metric_threshold_20": 0.40772547889010513,
257
+ "scr_dir2_threshold_20": 0.40772547889010513,
258
+ "scr_dir1_threshold_50": 0.46046493833553914,
259
+ "scr_metric_threshold_50": 0.3562232242738785,
260
+ "scr_dir2_threshold_50": 0.3562232242738785,
261
+ "scr_dir1_threshold_100": 0.49767431029119774,
262
+ "scr_metric_threshold_100": 0.3862660142728834,
263
+ "scr_dir2_threshold_100": 0.3862660142728834,
264
+ "scr_dir1_threshold_500": 0.5813952585759982,
265
+ "scr_metric_threshold_500": 0.4463518500848407,
266
+ "scr_dir2_threshold_500": 0.4463518500848407
267
  },
268
  {
269
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
 
276
  "scr_dir1_threshold_10": 0.19999994455381204,
277
  "scr_metric_threshold_10": 0.19999994455381204,
278
  "scr_dir2_threshold_10": 0.12669681061232305,
279
+ "scr_dir1_threshold_20": 0.2511626372535729,
280
+ "scr_metric_threshold_20": 0.2511626372535729,
281
+ "scr_dir2_threshold_20": 0.16742080471658802,
282
+ "scr_dir1_threshold_50": 0.3302325052722394,
283
+ "scr_metric_threshold_50": 0.3302325052722394,
284
+ "scr_dir2_threshold_50": 0.2262443817232214,
285
+ "scr_dir1_threshold_100": 0.2837209160486496,
286
+ "scr_metric_threshold_100": 0.2837209160486496,
287
+ "scr_dir2_threshold_100": 0.14479639351469145,
288
+ "scr_dir1_threshold_500": 0.2976742264771627,
289
+ "scr_metric_threshold_500": 0.2976742264771627,
290
+ "scr_dir2_threshold_500": 0.2850679587298548
291
  }
292
  ],
293
  "sae_bench_commit_hash": "Unknown",
gemma-2-2b-btk/k-100/seed-0/snapshots/step-292968-tokens-1200000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "dfa66f69-74fd-4bcc-9a66-73be067e304e",
69
- "datetime_epoch_millis": 1776886123512,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.034550003707408905,
73
- "tpp_threshold_2_intended_diff_only": 0.039000004529953,
74
- "tpp_threshold_2_unintended_diff_only": 0.004450000822544098,
75
- "tpp_threshold_5_total_metric": 0.07345001846551895,
76
- "tpp_threshold_5_intended_diff_only": 0.09270002245903015,
77
- "tpp_threshold_5_unintended_diff_only": 0.019250003993511202,
78
- "tpp_threshold_10_total_metric": 0.10297500342130661,
79
- "tpp_threshold_10_intended_diff_only": 0.15680001378059386,
80
- "tpp_threshold_10_unintended_diff_only": 0.053825010359287266,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.061400017142295836,
99
- "tpp_threshold_2_intended_diff_only": 0.07020002603530884,
100
- "tpp_threshold_2_unintended_diff_only": 0.008800008893013,
101
- "tpp_threshold_5_total_metric": 0.1337500274181366,
102
- "tpp_threshold_5_intended_diff_only": 0.17000004053115844,
103
- "tpp_threshold_5_unintended_diff_only": 0.03625001311302185,
104
- "tpp_threshold_10_total_metric": 0.1850000023841858,
105
  "tpp_threshold_10_intended_diff_only": 0.28980002403259275,
106
- "tpp_threshold_10_unintended_diff_only": 0.10480002164840699,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.007699990272521972,
123
- "tpp_threshold_2_intended_diff_only": 0.007799983024597168,
124
- "tpp_threshold_2_unintended_diff_only": 9.999275207519531e-05,
125
- "tpp_threshold_5_total_metric": 0.013150009512901305,
126
- "tpp_threshold_5_intended_diff_only": 0.015400004386901856,
127
- "tpp_threshold_5_unintended_diff_only": 0.0022499948740005494,
128
- "tpp_threshold_10_total_metric": 0.02095000445842743,
129
- "tpp_threshold_10_intended_diff_only": 0.02380000352859497,
130
- "tpp_threshold_10_unintended_diff_only": 0.0028499990701675417,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.012250006198883057,
190
  "tpp_threshold_10_total_metric": 0.1407499760389328,
191
  "tpp_threshold_10_intended_diff_only": 0.15700000524520874,
192
- "tpp_threshold_10_unintended_diff_only": 0.01625002920627594
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": 0.046500012278556824,
196
  "tpp_threshold_2_intended_diff_only": 0.058000028133392334,
197
- "tpp_threshold_2_unintended_diff_only": 0.01150001585483551,
198
- "tpp_threshold_5_total_metric": 0.1862500160932541,
199
  "tpp_threshold_5_intended_diff_only": 0.27000004053115845,
200
- "tpp_threshold_5_unintended_diff_only": 0.08375002443790436,
201
- "tpp_threshold_10_total_metric": 0.1875000149011612,
202
  "tpp_threshold_10_intended_diff_only": 0.31700003147125244,
203
- "tpp_threshold_10_unintended_diff_only": 0.12950001657009125
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.1585000604391098,
207
  "tpp_threshold_2_intended_diff_only": 0.18500006198883057,
208
- "tpp_threshold_2_unintended_diff_only": 0.026500001549720764,
209
- "tpp_threshold_5_total_metric": 0.19850002229213715,
210
- "tpp_threshold_5_intended_diff_only": 0.2420000433921814,
211
- "tpp_threshold_5_unintended_diff_only": 0.04350002110004425,
212
- "tpp_threshold_10_total_metric": 0.1470000147819519,
213
- "tpp_threshold_10_intended_diff_only": 0.34400004148483276,
214
- "tpp_threshold_10_unintended_diff_only": 0.19700002670288086
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.011749997735023499,
218
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
219
- "tpp_threshold_2_unintended_diff_only": -0.002749994397163391,
220
- "tpp_threshold_5_total_metric": 0.15700004994869232,
221
- "tpp_threshold_5_intended_diff_only": 0.18100005388259888,
222
- "tpp_threshold_5_unintended_diff_only": 0.024000003933906555,
223
- "tpp_threshold_10_total_metric": 0.2667500078678131,
224
  "tpp_threshold_10_intended_diff_only": 0.3680000305175781,
225
- "tpp_threshold_10_unintended_diff_only": 0.10125002264976501
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.006000027060508728,
229
- "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
230
- "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224,
231
- "tpp_threshold_5_total_metric": 0.029250040650367737,
232
- "tpp_threshold_5_intended_diff_only": 0.04700005054473877,
233
- "tpp_threshold_5_unintended_diff_only": 0.017750009894371033,
234
- "tpp_threshold_10_total_metric": 0.18299999833106995,
235
- "tpp_threshold_10_intended_diff_only": 0.2630000114440918,
236
- "tpp_threshold_10_unintended_diff_only": 0.08000001311302185
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.011499986052513123,
242
- "tpp_threshold_2_intended_diff_only": 0.0059999823570251465,
243
- "tpp_threshold_2_unintended_diff_only": -0.005500003695487976,
244
- "tpp_threshold_5_total_metric": 0.01075001060962677,
245
- "tpp_threshold_5_intended_diff_only": 0.008000016212463379,
246
- "tpp_threshold_5_unintended_diff_only": -0.002749994397163391,
247
- "tpp_threshold_10_total_metric": 0.002249985933303833,
248
- "tpp_threshold_10_intended_diff_only": 0.0059999823570251465,
249
- "tpp_threshold_10_unintended_diff_only": 0.0037499964237213135
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": 0.003250017762184143,
253
- "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
254
- "tpp_threshold_2_unintended_diff_only": 0.004749998450279236,
255
- "tpp_threshold_5_total_metric": 0.0102500319480896,
256
- "tpp_threshold_5_intended_diff_only": 0.017000019550323486,
257
  "tpp_threshold_5_unintended_diff_only": 0.006749987602233887,
258
- "tpp_threshold_10_total_metric": 0.02600005269050598,
259
- "tpp_threshold_10_intended_diff_only": 0.029000043869018555,
260
- "tpp_threshold_10_unintended_diff_only": 0.0029999911785125732
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.011250004172325134,
264
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
265
- "tpp_threshold_2_unintended_diff_only": -0.006250008940696716,
266
- "tpp_threshold_5_total_metric": 0.014500021934509277,
267
- "tpp_threshold_5_intended_diff_only": 0.013000011444091797,
268
- "tpp_threshold_5_unintended_diff_only": -0.0015000104904174805,
269
- "tpp_threshold_10_total_metric": 0.017250001430511475,
270
- "tpp_threshold_10_intended_diff_only": 0.018999993801116943,
271
- "tpp_threshold_10_unintended_diff_only": 0.0017499923706054688
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.003500044345855713,
275
- "tpp_threshold_2_intended_diff_only": 0.003999948501586914,
276
- "tpp_threshold_2_unintended_diff_only": 0.007499992847442627,
277
- "tpp_threshold_5_total_metric": 0.009999975562095642,
278
- "tpp_threshold_5_intended_diff_only": 0.011999964714050293,
279
- "tpp_threshold_5_unintended_diff_only": 0.001999989151954651,
280
- "tpp_threshold_10_total_metric": 0.015499979257583618,
281
- "tpp_threshold_10_intended_diff_only": 0.018999993801116943,
282
- "tpp_threshold_10_unintended_diff_only": 0.003500014543533325
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.015999987721443176,
286
- "tpp_threshold_2_intended_diff_only": 0.015999972820281982,
287
- "tpp_threshold_2_unintended_diff_only": -1.4901161193847656e-08,
288
- "tpp_threshold_5_total_metric": 0.02025000751018524,
289
- "tpp_threshold_5_intended_diff_only": 0.027000010013580322,
290
- "tpp_threshold_5_unintended_diff_only": 0.006750002503395081,
291
- "tpp_threshold_10_total_metric": 0.04375000298023224,
292
- "tpp_threshold_10_intended_diff_only": 0.046000003814697266,
293
- "tpp_threshold_10_unintended_diff_only": 0.002250000834465027
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "7b752ed9-5a56-4940-9c78-a1d00014088e",
73
+ "datetime_epoch_millis": 1777317425795,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.03327499777078629,
77
+ "tpp_threshold_2_intended_diff_only": 0.03860000371932983,
78
+ "tpp_threshold_2_unintended_diff_only": 0.005325005948543548,
79
+ "tpp_threshold_5_total_metric": 0.07347500324249268,
80
+ "tpp_threshold_5_intended_diff_only": 0.09190000891685485,
81
+ "tpp_threshold_5_unintended_diff_only": 0.018425005674362182,
82
+ "tpp_threshold_10_total_metric": 0.10349999815225601,
83
+ "tpp_threshold_10_intended_diff_only": 0.15780000686645507,
84
+ "tpp_threshold_10_unintended_diff_only": 0.054300008714199065,
85
+ "tpp_threshold_20_total_metric": 0.1278250113129616,
86
+ "tpp_threshold_20_intended_diff_only": 0.20930001735687256,
87
+ "tpp_threshold_20_unintended_diff_only": 0.08147500604391099,
88
+ "tpp_threshold_50_total_metric": 0.14660000950098037,
89
+ "tpp_threshold_50_intended_diff_only": 0.2668000221252442,
90
+ "tpp_threshold_50_unintended_diff_only": 0.12020001262426376,
91
+ "tpp_threshold_100_total_metric": 0.16400000900030137,
92
+ "tpp_threshold_100_intended_diff_only": 0.31930001974105837,
93
+ "tpp_threshold_100_unintended_diff_only": 0.15530001074075697,
94
+ "tpp_threshold_500_total_metric": 0.23262501657009124,
95
+ "tpp_threshold_500_intended_diff_only": 0.4371000289916992,
96
+ "tpp_threshold_500_unintended_diff_only": 0.20447501242160798
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.06065000891685486,
103
+ "tpp_threshold_2_intended_diff_only": 0.07060002088546753,
104
+ "tpp_threshold_2_unintended_diff_only": 0.009950011968612671,
105
+ "tpp_threshold_5_total_metric": 0.1334500193595886,
106
+ "tpp_threshold_5_intended_diff_only": 0.16980003118515014,
107
+ "tpp_threshold_5_unintended_diff_only": 0.03635001182556152,
108
+ "tpp_threshold_10_total_metric": 0.18390000164508818,
109
  "tpp_threshold_10_intended_diff_only": 0.28980002403259275,
110
+ "tpp_threshold_10_unintended_diff_only": 0.10590002238750458,
111
+ "tpp_threshold_20_total_metric": 0.209250009059906,
112
+ "tpp_threshold_20_intended_diff_only": 0.3684000253677368,
113
+ "tpp_threshold_20_unintended_diff_only": 0.15915001630783082,
114
+ "tpp_threshold_50_total_metric": 0.18035001158714295,
115
+ "tpp_threshold_50_intended_diff_only": 0.4146000385284424,
116
+ "tpp_threshold_50_unintended_diff_only": 0.23425002694129943,
117
+ "tpp_threshold_100_total_metric": 0.14240001142024994,
118
+ "tpp_threshold_100_intended_diff_only": 0.44000003337860105,
119
+ "tpp_threshold_100_unintended_diff_only": 0.2976000219583511,
120
+ "tpp_threshold_500_total_metric": 0.09025001227855682,
121
+ "tpp_threshold_500_intended_diff_only": 0.4674000382423401,
122
+ "tpp_threshold_500_unintended_diff_only": 0.3771500259637833
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.005899986624717713,
127
+ "tpp_threshold_2_intended_diff_only": 0.006599986553192138,
128
+ "tpp_threshold_2_unintended_diff_only": 0.0006999999284744262,
129
+ "tpp_threshold_5_total_metric": 0.01349998712539673,
130
+ "tpp_threshold_5_intended_diff_only": 0.01399998664855957,
131
+ "tpp_threshold_5_unintended_diff_only": 0.0004999995231628418,
132
+ "tpp_threshold_10_total_metric": 0.023099994659423827,
133
+ "tpp_threshold_10_intended_diff_only": 0.025799989700317383,
134
+ "tpp_threshold_10_unintended_diff_only": 0.0026999950408935548,
135
+ "tpp_threshold_20_total_metric": 0.046400013566017154,
136
+ "tpp_threshold_20_intended_diff_only": 0.0502000093460083,
137
+ "tpp_threshold_20_unintended_diff_only": 0.00379999577999115,
138
+ "tpp_threshold_50_total_metric": 0.11285000741481781,
139
+ "tpp_threshold_50_intended_diff_only": 0.1190000057220459,
140
+ "tpp_threshold_50_unintended_diff_only": 0.006149998307228089,
141
+ "tpp_threshold_100_total_metric": 0.18560000658035278,
142
+ "tpp_threshold_100_intended_diff_only": 0.19860000610351564,
143
+ "tpp_threshold_100_unintended_diff_only": 0.012999999523162841,
144
+ "tpp_threshold_500_total_metric": 0.37500002086162565,
145
+ "tpp_threshold_500_intended_diff_only": 0.40680001974105834,
146
+ "tpp_threshold_500_unintended_diff_only": 0.031799998879432675
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.012250006198883057,
194
  "tpp_threshold_10_total_metric": 0.1407499760389328,
195
  "tpp_threshold_10_intended_diff_only": 0.15700000524520874,
196
+ "tpp_threshold_10_unintended_diff_only": 0.01625002920627594,
197
+ "tpp_threshold_20_total_metric": 0.22074997425079346,
198
+ "tpp_threshold_20_intended_diff_only": 0.3610000014305115,
199
+ "tpp_threshold_20_unintended_diff_only": 0.14025002717971802,
200
+ "tpp_threshold_50_total_metric": 0.187749981880188,
201
+ "tpp_threshold_50_intended_diff_only": 0.41600000858306885,
202
+ "tpp_threshold_50_unintended_diff_only": 0.22825002670288086,
203
+ "tpp_threshold_100_total_metric": 0.13749994337558746,
204
+ "tpp_threshold_100_intended_diff_only": 0.43699997663497925,
205
+ "tpp_threshold_100_unintended_diff_only": 0.2995000332593918,
206
+ "tpp_threshold_500_total_metric": 0.08750000596046448,
207
+ "tpp_threshold_500_intended_diff_only": 0.45600003004074097,
208
+ "tpp_threshold_500_unintended_diff_only": 0.3685000240802765
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.049250006675720215,
212
  "tpp_threshold_2_intended_diff_only": 0.058000028133392334,
213
+ "tpp_threshold_2_unintended_diff_only": 0.00875002145767212,
214
+ "tpp_threshold_5_total_metric": 0.1860000193119049,
215
  "tpp_threshold_5_intended_diff_only": 0.27000004053115845,
216
+ "tpp_threshold_5_unintended_diff_only": 0.08400002121925354,
217
+ "tpp_threshold_10_total_metric": 0.17925001680850983,
218
  "tpp_threshold_10_intended_diff_only": 0.31700003147125244,
219
+ "tpp_threshold_10_unintended_diff_only": 0.13775001466274261,
220
+ "tpp_threshold_20_total_metric": 0.19500000774860382,
221
+ "tpp_threshold_20_intended_diff_only": 0.3500000238418579,
222
+ "tpp_threshold_20_unintended_diff_only": 0.1550000160932541,
223
+ "tpp_threshold_50_total_metric": 0.18150001764297485,
224
+ "tpp_threshold_50_intended_diff_only": 0.3850000500679016,
225
+ "tpp_threshold_50_unintended_diff_only": 0.20350003242492676,
226
+ "tpp_threshold_100_total_metric": 0.15974999964237213,
227
+ "tpp_threshold_100_intended_diff_only": 0.4240000247955322,
228
+ "tpp_threshold_100_unintended_diff_only": 0.2642500251531601,
229
+ "tpp_threshold_500_total_metric": 0.08050002157688141,
230
+ "tpp_threshold_500_intended_diff_only": 0.4590000510215759,
231
+ "tpp_threshold_500_unintended_diff_only": 0.3785000294446945
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.15175004303455353,
235
  "tpp_threshold_2_intended_diff_only": 0.18500006198883057,
236
+ "tpp_threshold_2_unintended_diff_only": 0.03325001895427704,
237
+ "tpp_threshold_5_total_metric": 0.19575004279613495,
238
+ "tpp_threshold_5_intended_diff_only": 0.24100005626678467,
239
+ "tpp_threshold_5_unintended_diff_only": 0.04525001347064972,
240
+ "tpp_threshold_10_total_metric": 0.14900000393390656,
241
+ "tpp_threshold_10_intended_diff_only": 0.3450000286102295,
242
+ "tpp_threshold_10_unintended_diff_only": 0.19600002467632294,
243
+ "tpp_threshold_20_total_metric": 0.1262499988079071,
244
+ "tpp_threshold_20_intended_diff_only": 0.36000001430511475,
245
+ "tpp_threshold_20_unintended_diff_only": 0.23375001549720764,
246
+ "tpp_threshold_50_total_metric": 0.08650003373622894,
247
+ "tpp_threshold_50_intended_diff_only": 0.3890000581741333,
248
+ "tpp_threshold_50_unintended_diff_only": 0.30250002443790436,
249
+ "tpp_threshold_100_total_metric": 0.06325003504753113,
250
+ "tpp_threshold_100_intended_diff_only": 0.40800005197525024,
251
+ "tpp_threshold_100_unintended_diff_only": 0.3447500169277191,
252
+ "tpp_threshold_500_total_metric": 0.04874999821186066,
253
+ "tpp_threshold_500_intended_diff_only": 0.4520000219345093,
254
+ "tpp_threshold_500_unintended_diff_only": 0.4032500237226486
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.009749993681907654,
258
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0007499903440475464,
260
+ "tpp_threshold_5_total_metric": 0.16225002706050873,
261
+ "tpp_threshold_5_intended_diff_only": 0.1820000410079956,
262
+ "tpp_threshold_5_unintended_diff_only": 0.019750013947486877,
263
+ "tpp_threshold_10_total_metric": 0.27025000751018524,
264
  "tpp_threshold_10_intended_diff_only": 0.3680000305175781,
265
+ "tpp_threshold_10_unintended_diff_only": 0.09775002300739288,
266
+ "tpp_threshold_20_total_metric": 0.29975004494190216,
267
+ "tpp_threshold_20_intended_diff_only": 0.4450000524520874,
268
+ "tpp_threshold_20_unintended_diff_only": 0.14525000751018524,
269
+ "tpp_threshold_50_total_metric": 0.25325003266334534,
270
+ "tpp_threshold_50_intended_diff_only": 0.47700005769729614,
271
+ "tpp_threshold_50_unintended_diff_only": 0.2237500250339508,
272
+ "tpp_threshold_100_total_metric": 0.18200001120567322,
273
+ "tpp_threshold_100_intended_diff_only": 0.4830000400543213,
274
+ "tpp_threshold_100_unintended_diff_only": 0.30100002884864807,
275
+ "tpp_threshold_500_total_metric": 0.12299998104572296,
276
+ "tpp_threshold_500_intended_diff_only": 0.49400001764297485,
277
+ "tpp_threshold_500_unintended_diff_only": 0.3710000365972519
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.008250012993812561,
281
+ "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464,
283
+ "tpp_threshold_5_total_metric": 0.025499999523162842,
284
+ "tpp_threshold_5_intended_diff_only": 0.046000003814697266,
285
+ "tpp_threshold_5_unintended_diff_only": 0.020500004291534424,
286
+ "tpp_threshold_10_total_metric": 0.18025000393390656,
287
+ "tpp_threshold_10_intended_diff_only": 0.26200002431869507,
288
+ "tpp_threshold_10_unintended_diff_only": 0.08175002038478851,
289
+ "tpp_threshold_20_total_metric": 0.2045000195503235,
290
+ "tpp_threshold_20_intended_diff_only": 0.32600003480911255,
291
+ "tpp_threshold_20_unintended_diff_only": 0.12150001525878906,
292
+ "tpp_threshold_50_total_metric": 0.1927499920129776,
293
+ "tpp_threshold_50_intended_diff_only": 0.406000018119812,
294
+ "tpp_threshold_50_unintended_diff_only": 0.2132500261068344,
295
+ "tpp_threshold_100_total_metric": 0.16950006783008575,
296
+ "tpp_threshold_100_intended_diff_only": 0.44800007343292236,
297
+ "tpp_threshold_100_unintended_diff_only": 0.2785000056028366,
298
+ "tpp_threshold_500_total_metric": 0.11150005459785461,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.3645000159740448
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.004249975085258484,
306
+ "tpp_threshold_2_intended_diff_only": 0.006999969482421875,
307
+ "tpp_threshold_2_unintended_diff_only": 0.002749994397163391,
308
+ "tpp_threshold_5_total_metric": 0.0037499964237213135,
309
+ "tpp_threshold_5_intended_diff_only": 0.004999995231628418,
310
+ "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045,
311
+ "tpp_threshold_10_total_metric": 0.0037499964237213135,
312
+ "tpp_threshold_10_intended_diff_only": 0.004999995231628418,
313
+ "tpp_threshold_10_unintended_diff_only": 0.0012499988079071045,
314
+ "tpp_threshold_20_total_metric": 0.018000036478042603,
315
+ "tpp_threshold_20_intended_diff_only": 0.021000027656555176,
316
+ "tpp_threshold_20_unintended_diff_only": 0.0029999911785125732,
317
+ "tpp_threshold_50_total_metric": 0.06699997186660767,
318
+ "tpp_threshold_50_intended_diff_only": 0.06699997186660767,
319
+ "tpp_threshold_50_unintended_diff_only": 0.0,
320
+ "tpp_threshold_100_total_metric": 0.12275004386901855,
321
+ "tpp_threshold_100_intended_diff_only": 0.13200002908706665,
322
+ "tpp_threshold_100_unintended_diff_only": 0.009249985218048096,
323
+ "tpp_threshold_500_total_metric": 0.4110000282526016,
324
+ "tpp_threshold_500_intended_diff_only": 0.4280000329017639,
325
+ "tpp_threshold_500_unintended_diff_only": 0.017000004649162292
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.011500000953674316,
329
+ "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
330
+ "tpp_threshold_2_unintended_diff_only": -0.002499997615814209,
331
+ "tpp_threshold_5_total_metric": 0.007250010967254639,
332
+ "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
333
  "tpp_threshold_5_unintended_diff_only": 0.006749987602233887,
334
+ "tpp_threshold_10_total_metric": 0.02625003457069397,
335
+ "tpp_threshold_10_intended_diff_only": 0.03200000524520874,
336
+ "tpp_threshold_10_unintended_diff_only": 0.0057499706745147705,
337
+ "tpp_threshold_20_total_metric": 0.07225003838539124,
338
+ "tpp_threshold_20_intended_diff_only": 0.07700002193450928,
339
+ "tpp_threshold_20_unintended_diff_only": 0.004749983549118042,
340
+ "tpp_threshold_50_total_metric": 0.12525007128715515,
341
+ "tpp_threshold_50_intended_diff_only": 0.1390000581741333,
342
+ "tpp_threshold_50_unintended_diff_only": 0.01374998688697815,
343
+ "tpp_threshold_100_total_metric": 0.20775000751018524,
344
+ "tpp_threshold_100_intended_diff_only": 0.22600001096725464,
345
+ "tpp_threshold_100_unintended_diff_only": 0.018250003457069397,
346
+ "tpp_threshold_500_total_metric": 0.4060000777244568,
347
+ "tpp_threshold_500_intended_diff_only": 0.43500006198883057,
348
+ "tpp_threshold_500_unintended_diff_only": 0.02899998426437378
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.0027499794960021973,
352
+ "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
353
+ "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402,
354
+ "tpp_threshold_5_total_metric": 0.011750012636184692,
355
+ "tpp_threshold_5_intended_diff_only": 0.00700002908706665,
356
+ "tpp_threshold_5_unintended_diff_only": -0.004749983549118042,
357
+ "tpp_threshold_10_total_metric": 0.016749992966651917,
358
+ "tpp_threshold_10_intended_diff_only": 0.014999985694885254,
359
+ "tpp_threshold_10_unintended_diff_only": -0.0017500072717666626,
360
+ "tpp_threshold_20_total_metric": 0.010500028729438782,
361
+ "tpp_threshold_20_intended_diff_only": 0.012000024318695068,
362
+ "tpp_threshold_20_unintended_diff_only": 0.0014999955892562866,
363
+ "tpp_threshold_50_total_metric": 0.047249987721443176,
364
+ "tpp_threshold_50_intended_diff_only": 0.046999990940093994,
365
+ "tpp_threshold_50_unintended_diff_only": -0.00024999678134918213,
366
+ "tpp_threshold_100_total_metric": 0.1042499989271164,
367
+ "tpp_threshold_100_intended_diff_only": 0.11599999666213989,
368
+ "tpp_threshold_100_unintended_diff_only": 0.011749997735023499,
369
+ "tpp_threshold_500_total_metric": 0.34849999845027924,
370
+ "tpp_threshold_500_intended_diff_only": 0.3799999952316284,
371
+ "tpp_threshold_500_unintended_diff_only": 0.03149999678134918
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.004999980330467224,
375
+ "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
376
+ "tpp_threshold_2_unintended_diff_only": 1.4901161193847656e-08,
377
+ "tpp_threshold_5_total_metric": 0.018749937415122986,
378
+ "tpp_threshold_5_intended_diff_only": 0.01799994707107544,
379
+ "tpp_threshold_5_unintended_diff_only": -0.0007499903440475464,
380
+ "tpp_threshold_10_total_metric": 0.011249959468841553,
381
+ "tpp_threshold_10_intended_diff_only": 0.01699995994567871,
382
+ "tpp_threshold_10_unintended_diff_only": 0.005750000476837158,
383
+ "tpp_threshold_20_total_metric": 0.03449995815753937,
384
+ "tpp_threshold_20_intended_diff_only": 0.034999966621398926,
385
+ "tpp_threshold_20_unintended_diff_only": 0.0005000084638595581,
386
+ "tpp_threshold_50_total_metric": 0.1274999976158142,
387
+ "tpp_threshold_50_intended_diff_only": 0.13899999856948853,
388
+ "tpp_threshold_50_unintended_diff_only": 0.011500000953674316,
389
+ "tpp_threshold_100_total_metric": 0.23149998486042023,
390
+ "tpp_threshold_100_intended_diff_only": 0.2459999918937683,
391
+ "tpp_threshold_100_unintended_diff_only": 0.014500007033348083,
392
+ "tpp_threshold_500_total_metric": 0.3727499693632126,
393
+ "tpp_threshold_500_intended_diff_only": 0.4179999828338623,
394
+ "tpp_threshold_500_unintended_diff_only": 0.04525001347064972
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.00599999725818634,
398
+ "tpp_threshold_2_intended_diff_only": 0.009999990463256836,
399
+ "tpp_threshold_2_unintended_diff_only": 0.003999993205070496,
400
+ "tpp_threshold_5_total_metric": 0.025999978184700012,
401
+ "tpp_threshold_5_intended_diff_only": 0.02599996328353882,
402
+ "tpp_threshold_5_unintended_diff_only": -1.4901161193847656e-08,
403
+ "tpp_threshold_10_total_metric": 0.05749998986721039,
404
+ "tpp_threshold_10_intended_diff_only": 0.06000000238418579,
405
+ "tpp_threshold_10_unintended_diff_only": 0.002500012516975403,
406
+ "tpp_threshold_20_total_metric": 0.09675000607967377,
407
+ "tpp_threshold_20_intended_diff_only": 0.10600000619888306,
408
+ "tpp_threshold_20_unintended_diff_only": 0.00925000011920929,
409
+ "tpp_threshold_50_total_metric": 0.19725000858306885,
410
+ "tpp_threshold_50_intended_diff_only": 0.203000009059906,
411
+ "tpp_threshold_50_unintended_diff_only": 0.005750000476837158,
412
+ "tpp_threshold_100_total_metric": 0.2617499977350235,
413
+ "tpp_threshold_100_intended_diff_only": 0.27300000190734863,
414
+ "tpp_threshold_100_unintended_diff_only": 0.011250004172325134,
415
+ "tpp_threshold_500_total_metric": 0.3367500305175781,
416
+ "tpp_threshold_500_intended_diff_only": 0.37300002574920654,
417
+ "tpp_threshold_500_unintended_diff_only": 0.03624999523162842
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-317382-tokens-1300000000/scr/saebench_scr_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,8 +69,8 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "f10c176b-9b1c-4898-b453-e83918863c29",
69
- "datetime_epoch_millis": 1776885910506,
70
  "eval_result_metrics": {
71
  "scr_metrics": {
72
  "scr_dir1_threshold_2": 0.2885073924088952,
@@ -78,18 +82,18 @@
78
  "scr_dir1_threshold_10": 0.2709502606944876,
79
  "scr_metric_threshold_10": 0.20186184242570115,
80
  "scr_dir2_threshold_10": 0.19500875267788764,
81
- "scr_dir1_threshold_20": null,
82
- "scr_metric_threshold_20": null,
83
- "scr_dir2_threshold_20": null,
84
- "scr_dir1_threshold_50": null,
85
- "scr_metric_threshold_50": null,
86
- "scr_dir2_threshold_50": null,
87
- "scr_dir1_threshold_100": null,
88
- "scr_metric_threshold_100": null,
89
- "scr_dir2_threshold_100": null,
90
- "scr_dir1_threshold_500": null,
91
- "scr_metric_threshold_500": null,
92
- "scr_dir2_threshold_500": null
93
  }
94
  },
95
  "eval_result_details": [
@@ -104,18 +108,18 @@
104
  "scr_dir1_threshold_10": 0.2307692895509249,
105
  "scr_metric_threshold_10": 0.005181279943224265,
106
  "scr_dir2_threshold_10": 0.005181279943224265,
107
- "scr_dir1_threshold_20": null,
108
- "scr_metric_threshold_20": null,
109
- "scr_dir2_threshold_20": null,
110
- "scr_dir1_threshold_50": null,
111
- "scr_metric_threshold_50": null,
112
- "scr_dir2_threshold_50": null,
113
- "scr_dir1_threshold_100": null,
114
- "scr_metric_threshold_100": null,
115
- "scr_dir2_threshold_100": null,
116
- "scr_dir1_threshold_500": null,
117
- "scr_metric_threshold_500": null,
118
- "scr_dir2_threshold_500": null
119
  },
120
  {
121
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
@@ -128,18 +132,18 @@
128
  "scr_dir1_threshold_10": 0.3560604589646946,
129
  "scr_metric_threshold_10": 0.2721711947358348,
130
  "scr_dir2_threshold_10": 0.2721711947358348,
131
- "scr_dir1_threshold_20": null,
132
- "scr_metric_threshold_20": null,
133
- "scr_dir2_threshold_20": null,
134
- "scr_dir1_threshold_50": null,
135
- "scr_metric_threshold_50": null,
136
- "scr_dir2_threshold_50": null,
137
- "scr_dir1_threshold_100": null,
138
- "scr_metric_threshold_100": null,
139
- "scr_dir2_threshold_100": null,
140
- "scr_dir1_threshold_500": null,
141
- "scr_metric_threshold_500": null,
142
- "scr_dir2_threshold_500": null
143
  },
144
  {
145
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
@@ -152,18 +156,18 @@
152
  "scr_dir1_threshold_10": 0.5156247962732202,
153
  "scr_metric_threshold_10": 0.10804029093651613,
154
  "scr_dir2_threshold_10": 0.10804029093651613,
155
- "scr_dir1_threshold_20": null,
156
- "scr_metric_threshold_20": null,
157
- "scr_dir2_threshold_20": null,
158
- "scr_dir1_threshold_50": null,
159
- "scr_metric_threshold_50": null,
160
- "scr_dir2_threshold_50": null,
161
- "scr_dir1_threshold_100": null,
162
- "scr_metric_threshold_100": null,
163
- "scr_dir2_threshold_100": null,
164
- "scr_dir1_threshold_500": null,
165
- "scr_metric_threshold_500": null,
166
- "scr_dir2_threshold_500": null
167
  },
168
  {
169
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
@@ -176,18 +180,18 @@
176
  "scr_dir1_threshold_10": 0.0067113234832316924,
177
  "scr_metric_threshold_10": 0.044585981215805634,
178
  "scr_dir2_threshold_10": 0.044585981215805634,
179
- "scr_dir1_threshold_20": null,
180
- "scr_metric_threshold_20": null,
181
- "scr_dir2_threshold_20": null,
182
- "scr_dir1_threshold_50": null,
183
- "scr_metric_threshold_50": null,
184
- "scr_dir2_threshold_50": null,
185
- "scr_dir1_threshold_100": null,
186
- "scr_metric_threshold_100": null,
187
- "scr_dir2_threshold_100": null,
188
- "scr_dir1_threshold_500": null,
189
- "scr_metric_threshold_500": null,
190
- "scr_dir2_threshold_500": null
191
  },
192
  {
193
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
@@ -200,18 +204,18 @@
200
  "scr_dir1_threshold_10": 0.07894729412163301,
201
  "scr_metric_threshold_10": 0.4999998817368318,
202
  "scr_dir2_threshold_10": 0.4999998817368318,
203
- "scr_dir1_threshold_20": null,
204
- "scr_metric_threshold_20": null,
205
- "scr_dir2_threshold_20": null,
206
- "scr_dir1_threshold_50": null,
207
- "scr_metric_threshold_50": null,
208
- "scr_dir2_threshold_50": null,
209
- "scr_dir1_threshold_100": null,
210
- "scr_metric_threshold_100": null,
211
- "scr_dir2_threshold_100": null,
212
- "scr_dir1_threshold_500": null,
213
- "scr_metric_threshold_500": null,
214
- "scr_dir2_threshold_500": null
215
  },
216
  {
217
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
@@ -224,18 +228,18 @@
224
  "scr_dir1_threshold_10": 0.3934425108216648,
225
  "scr_metric_threshold_10": 0.12977090206161526,
226
  "scr_dir2_threshold_10": 0.12977090206161526,
227
- "scr_dir1_threshold_20": null,
228
- "scr_metric_threshold_20": null,
229
- "scr_dir2_threshold_20": null,
230
- "scr_dir1_threshold_50": null,
231
- "scr_metric_threshold_50": null,
232
- "scr_dir2_threshold_50": null,
233
- "scr_dir1_threshold_100": null,
234
- "scr_metric_threshold_100": null,
235
- "scr_dir2_threshold_100": null,
236
- "scr_dir1_threshold_500": null,
237
- "scr_metric_threshold_500": null,
238
- "scr_dir2_threshold_500": null
239
  },
240
  {
241
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
@@ -248,18 +252,18 @@
248
  "scr_dir1_threshold_10": 0.40000005544617256,
249
  "scr_metric_threshold_10": 0.36909885188142205,
250
  "scr_dir2_threshold_10": 0.36909885188142205,
251
- "scr_dir1_threshold_20": null,
252
- "scr_metric_threshold_20": null,
253
- "scr_dir2_threshold_20": null,
254
- "scr_dir1_threshold_50": null,
255
- "scr_metric_threshold_50": null,
256
- "scr_dir2_threshold_50": null,
257
- "scr_dir1_threshold_100": null,
258
- "scr_metric_threshold_100": null,
259
- "scr_dir2_threshold_100": null,
260
- "scr_dir1_threshold_500": null,
261
- "scr_metric_threshold_500": null,
262
- "scr_dir2_threshold_500": null
263
  },
264
  {
265
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
@@ -272,18 +276,18 @@
272
  "scr_dir1_threshold_10": 0.18604635689435922,
273
  "scr_metric_threshold_10": 0.18604635689435922,
274
  "scr_dir2_threshold_10": 0.1312216389118512,
275
- "scr_dir1_threshold_20": null,
276
- "scr_metric_threshold_20": null,
277
- "scr_dir2_threshold_20": null,
278
- "scr_dir1_threshold_50": null,
279
- "scr_metric_threshold_50": null,
280
- "scr_dir2_threshold_50": null,
281
- "scr_dir1_threshold_100": null,
282
- "scr_metric_threshold_100": null,
283
- "scr_dir2_threshold_100": null,
284
- "scr_dir1_threshold_500": null,
285
- "scr_metric_threshold_500": null,
286
- "scr_dir2_threshold_500": null
287
  }
288
  ],
289
  "sae_bench_commit_hash": "Unknown",
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "7e261c04-6bb0-4026-8d96-74269cddf950",
73
+ "datetime_epoch_millis": 1777317397195,
74
  "eval_result_metrics": {
75
  "scr_metrics": {
76
  "scr_dir1_threshold_2": 0.2885073924088952,
 
82
  "scr_dir1_threshold_10": 0.2709502606944876,
83
  "scr_metric_threshold_10": 0.20186184242570115,
84
  "scr_dir2_threshold_10": 0.19500875267788764,
85
+ "scr_dir1_threshold_20": 0.27035680954058827,
86
+ "scr_metric_threshold_20": 0.2166289325602405,
87
+ "scr_dir2_threshold_20": 0.20953907403371513,
88
+ "scr_dir1_threshold_50": 0.11729968087135588,
89
+ "scr_metric_threshold_50": 0.26626300502796274,
90
+ "scr_dir2_threshold_50": 0.25901529970812726,
91
+ "scr_dir1_threshold_100": 0.0291723500274349,
92
+ "scr_metric_threshold_100": 0.2648691843249761,
93
+ "scr_dir2_threshold_100": 0.25028435093025925,
94
+ "scr_dir1_threshold_500": -1.1906623470141946,
95
+ "scr_metric_threshold_500": 0.1666632837766538,
96
+ "scr_dir2_threshold_500": 0.15979176734762407
97
  }
98
  },
99
  "eval_result_details": [
 
108
  "scr_dir1_threshold_10": 0.2307692895509249,
109
  "scr_metric_threshold_10": 0.005181279943224265,
110
  "scr_dir2_threshold_10": 0.005181279943224265,
111
+ "scr_dir1_threshold_20": 0.20512859700616604,
112
+ "scr_metric_threshold_20": 0.09326427430734313,
113
+ "scr_dir2_threshold_20": 0.09326427430734313,
114
+ "scr_dir1_threshold_50": 0.5128207283533913,
115
+ "scr_metric_threshold_50": 0.25129531998580606,
116
+ "scr_dir2_threshold_50": 0.25129531998580606,
117
+ "scr_dir1_threshold_100": 0.5897435701496917,
118
+ "scr_metric_threshold_100": 0.2694299542032543,
119
+ "scr_dir2_threshold_100": 0.2694299542032543,
120
+ "scr_dir1_threshold_500": -2.602563916422071,
121
+ "scr_metric_threshold_500": 0.10103619422217953,
122
+ "scr_dir2_threshold_500": 0.10103619422217953
123
  },
124
  {
125
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
 
132
  "scr_dir1_threshold_10": 0.3560604589646946,
133
  "scr_metric_threshold_10": 0.2721711947358348,
134
  "scr_dir2_threshold_10": 0.2721711947358348,
135
+ "scr_dir1_threshold_20": 0.5454546275545891,
136
+ "scr_metric_threshold_20": -0.006116128797061692,
137
+ "scr_dir2_threshold_20": -0.006116128797061692,
138
+ "scr_dir1_threshold_50": 0.6060607976273744,
139
+ "scr_metric_threshold_50": 0.15290522497562042,
140
+ "scr_dir2_threshold_50": 0.15290522497562042,
141
+ "scr_dir1_threshold_100": 0.34090914222161817,
142
+ "scr_metric_threshold_100": 0.35779827383502094,
143
+ "scr_dir2_threshold_100": 0.35779827383502094,
144
+ "scr_dir1_threshold_500": -1.939393378376975,
145
+ "scr_metric_threshold_500": 0.03363925521540607,
146
+ "scr_dir2_threshold_500": 0.03363925521540607
147
  },
148
  {
149
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
 
156
  "scr_dir1_threshold_10": 0.5156247962732202,
157
  "scr_metric_threshold_10": 0.10804029093651613,
158
  "scr_dir2_threshold_10": 0.10804029093651613,
159
+ "scr_dir1_threshold_20": 0.32812537834973393,
160
+ "scr_metric_threshold_20": 0.16834176159853043,
161
+ "scr_dir2_threshold_20": 0.16834176159853043,
162
+ "scr_dir1_threshold_50": -1.4218750873114772,
163
+ "scr_metric_threshold_50": 0.15577896145062706,
164
+ "scr_dir2_threshold_50": 0.15577896145062706,
165
+ "scr_dir1_threshold_100": -0.921875087311477,
166
+ "scr_metric_threshold_100": -0.005025060155005,
167
+ "scr_dir2_threshold_100": -0.005025060155005,
168
+ "scr_dir1_threshold_500": -4.3437492433005325,
169
+ "scr_metric_threshold_500": -0.1005024011832269,
170
+ "scr_dir2_threshold_500": -0.1005024011832269
171
  },
172
  {
173
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
 
180
  "scr_dir1_threshold_10": 0.0067113234832316924,
181
  "scr_metric_threshold_10": 0.044585981215805634,
182
  "scr_dir2_threshold_10": 0.044585981215805634,
183
+ "scr_dir1_threshold_20": -0.2214768751962653,
184
+ "scr_metric_threshold_20": -0.025477757787232685,
185
+ "scr_dir2_threshold_20": -0.025477757787232685,
186
+ "scr_dir1_threshold_50": 0.13422806978944357,
187
+ "scr_metric_threshold_50": -0.08280242807295153,
188
+ "scr_dir2_threshold_50": -0.08280242807295153,
189
+ "scr_dir1_threshold_100": -1.100671452373285,
190
+ "scr_metric_threshold_100": -0.2611465427598772,
191
+ "scr_dir2_threshold_100": -0.2611465427598772,
192
+ "scr_dir1_threshold_500": -1.7852352483493712,
193
+ "scr_metric_threshold_500": -0.5445859812158056,
194
+ "scr_dir2_threshold_500": -0.5445859812158056
195
  },
196
  {
197
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
 
204
  "scr_dir1_threshold_10": 0.07894729412163301,
205
  "scr_metric_threshold_10": 0.4999998817368318,
206
  "scr_dir2_threshold_10": 0.4999998817368318,
207
+ "scr_dir1_threshold_20": 0.11578955293692479,
208
+ "scr_metric_threshold_20": 0.595238157185469,
209
+ "scr_dir2_threshold_20": 0.595238157185469,
210
+ "scr_dir1_threshold_50": -0.17894726275076694,
211
+ "scr_metric_threshold_50": 0.6904761961077699,
212
+ "scr_dir2_threshold_50": 0.6904761961077699,
213
+ "scr_dir1_threshold_100": -0.1421053176441358,
214
+ "scr_metric_threshold_100": 0.7380950973057522,
215
+ "scr_dir2_threshold_100": 0.7380950973057522,
216
+ "scr_dir1_threshold_500": -0.17894726275076694,
217
+ "scr_metric_threshold_500": 0.801587280938177,
218
+ "scr_dir2_threshold_500": 0.801587280938177
219
  },
220
  {
221
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
 
228
  "scr_dir1_threshold_10": 0.3934425108216648,
229
  "scr_metric_threshold_10": 0.12977090206161526,
230
  "scr_dir2_threshold_10": 0.12977090206161526,
231
+ "scr_dir1_threshold_20": 0.4316940424658739,
232
+ "scr_metric_threshold_20": 0.17557251734733875,
233
+ "scr_dir2_threshold_20": 0.17557251734733875,
234
+ "scr_dir1_threshold_50": 0.4535520140184937,
235
+ "scr_metric_threshold_50": 0.28244272551540384,
236
+ "scr_dir2_threshold_50": 0.28244272551540384,
237
+ "scr_dir1_threshold_100": 0.5464479859815062,
238
+ "scr_metric_threshold_100": 0.3015266750097995,
239
+ "scr_dir2_threshold_100": 0.3015266750097995,
240
+ "scr_dir1_threshold_500": 0.5245903401373838,
241
+ "scr_metric_threshold_500": 0.3320610852002818,
242
+ "scr_dir2_threshold_500": 0.3320610852002818
243
  },
244
  {
245
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
 
252
  "scr_dir1_threshold_10": 0.40000005544617256,
253
  "scr_metric_threshold_10": 0.36909885188142205,
254
  "scr_dir2_threshold_10": 0.36909885188142205,
255
+ "scr_dir1_threshold_20": 0.5023254124779394,
256
+ "scr_metric_threshold_20": 0.4763948958977931,
257
+ "scr_dir2_threshold_20": 0.4763948958977931,
258
+ "scr_dir1_threshold_50": 0.5302325800601148,
259
+ "scr_metric_threshold_50": 0.3776824330771527,
260
+ "scr_dir2_threshold_50": 0.3776824330771527,
261
+ "scr_dir1_threshold_100": 0.6232557327183983,
262
+ "scr_metric_threshold_100": 0.4206008506837011,
263
+ "scr_dir2_threshold_100": 0.4206008506837011,
264
+ "scr_dir1_threshold_500": 0.44186052958857275,
265
+ "scr_metric_threshold_500": 0.35193143367601315,
266
+ "scr_dir2_threshold_500": 0.35193143367601315
267
  },
268
  {
269
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
 
276
  "scr_dir1_threshold_10": 0.18604635689435922,
277
  "scr_metric_threshold_10": 0.18604635689435922,
278
  "scr_dir2_threshold_10": 0.1312216389118512,
279
+ "scr_dir1_threshold_20": 0.2558137407297439,
280
+ "scr_metric_threshold_20": 0.2558137407297439,
281
+ "scr_dir2_threshold_20": 0.19909487251754088,
282
+ "scr_dir1_threshold_50": 0.3023256071842735,
283
+ "scr_metric_threshold_50": 0.3023256071842735,
284
+ "scr_dir2_threshold_50": 0.2443439646255898,
285
+ "scr_dir1_threshold_100": 0.2976742264771627,
286
+ "scr_metric_threshold_100": 0.2976742264771627,
287
+ "scr_dir2_threshold_100": 0.1809955593194283,
288
+ "scr_dir1_threshold_500": 0.35813940336020533,
289
+ "scr_metric_threshold_500": 0.35813940336020533,
290
+ "scr_dir2_threshold_500": 0.3031672719279674
291
  }
292
  ],
293
  "sae_bench_commit_hash": "Unknown",
gemma-2-2b-btk/k-100/seed-0/snapshots/step-317382-tokens-1300000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "741babd0-f44e-43bc-9a53-6d3219b8d652",
69
- "datetime_epoch_millis": 1776886063440,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.02855000048875809,
73
- "tpp_threshold_2_intended_diff_only": 0.032999998331069945,
74
- "tpp_threshold_2_unintended_diff_only": 0.004449997842311859,
75
- "tpp_threshold_5_total_metric": 0.07450000792741775,
76
- "tpp_threshold_5_intended_diff_only": 0.0993000090122223,
77
- "tpp_threshold_5_unintended_diff_only": 0.024800001084804537,
78
- "tpp_threshold_10_total_metric": 0.09805000424385071,
79
- "tpp_threshold_10_intended_diff_only": 0.15370001196861266,
80
- "tpp_threshold_10_unintended_diff_only": 0.05565000772476196,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.04950000643730164,
99
- "tpp_threshold_2_intended_diff_only": 0.05840001106262207,
100
- "tpp_threshold_2_unintended_diff_only": 0.008900004625320434,
101
- "tpp_threshold_5_total_metric": 0.1350500226020813,
102
- "tpp_threshold_5_intended_diff_only": 0.18260003328323365,
103
- "tpp_threshold_5_unintended_diff_only": 0.047550010681152347,
104
- "tpp_threshold_10_total_metric": 0.1737999975681305,
105
- "tpp_threshold_10_intended_diff_only": 0.28180001974105834,
106
- "tpp_threshold_10_unintended_diff_only": 0.10800002217292785,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.007599994540214539,
123
- "tpp_threshold_2_intended_diff_only": 0.007599985599517823,
124
- "tpp_threshold_2_unintended_diff_only": -8.940696716308593e-09,
125
- "tpp_threshold_5_total_metric": 0.01394999325275421,
126
- "tpp_threshold_5_intended_diff_only": 0.015999984741210938,
127
- "tpp_threshold_5_unintended_diff_only": 0.002049991488456726,
128
- "tpp_threshold_10_total_metric": 0.02230001091957092,
129
- "tpp_threshold_10_intended_diff_only": 0.02560000419616699,
130
- "tpp_threshold_10_unintended_diff_only": 0.003299993276596069,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.0017500221729278564,
190
  "tpp_threshold_10_total_metric": 0.05774995684623718,
191
  "tpp_threshold_10_intended_diff_only": 0.06599998474121094,
192
- "tpp_threshold_10_unintended_diff_only": 0.008250027894973755
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": 0.06900003552436829,
196
- "tpp_threshold_2_intended_diff_only": 0.08500003814697266,
197
- "tpp_threshold_2_unintended_diff_only": 0.01600000262260437,
198
- "tpp_threshold_5_total_metric": 0.18700000643730164,
199
- "tpp_threshold_5_intended_diff_only": 0.281000018119812,
200
- "tpp_threshold_5_unintended_diff_only": 0.09400001168251038,
201
- "tpp_threshold_10_total_metric": 0.1927499920129776,
202
  "tpp_threshold_10_intended_diff_only": 0.3370000123977661,
203
- "tpp_threshold_10_unintended_diff_only": 0.1442500203847885
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.153500035405159,
207
- "tpp_threshold_2_intended_diff_only": 0.1820000410079956,
208
- "tpp_threshold_2_unintended_diff_only": 0.02850000560283661,
209
- "tpp_threshold_5_total_metric": 0.21050004661083221,
210
  "tpp_threshold_5_intended_diff_only": 0.28700006008148193,
211
- "tpp_threshold_5_unintended_diff_only": 0.07650001347064972,
212
- "tpp_threshold_10_total_metric": 0.15950000286102295,
213
- "tpp_threshold_10_intended_diff_only": 0.3410000205039978,
214
- "tpp_threshold_10_unintended_diff_only": 0.18150001764297485
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.011750012636184692,
218
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
219
- "tpp_threshold_2_unintended_diff_only": -0.002750009298324585,
220
- "tpp_threshold_5_total_metric": 0.21775005757808685,
221
- "tpp_threshold_5_intended_diff_only": 0.2640000581741333,
222
- "tpp_threshold_5_unintended_diff_only": 0.04625000059604645,
223
- "tpp_threshold_10_total_metric": 0.2625000476837158,
224
  "tpp_threshold_10_intended_diff_only": 0.3840000629425049,
225
- "tpp_threshold_10_unintended_diff_only": 0.12150001525878906
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.003999993205070496,
229
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
230
- "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224,
231
- "tpp_threshold_5_total_metric": 0.04375001788139343,
232
- "tpp_threshold_5_intended_diff_only": 0.06300002336502075,
233
- "tpp_threshold_5_unintended_diff_only": 0.01925000548362732,
234
- "tpp_threshold_10_total_metric": 0.1964999884366989,
235
- "tpp_threshold_10_intended_diff_only": 0.281000018119812,
236
- "tpp_threshold_10_unintended_diff_only": 0.0845000296831131
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.010749980807304382,
242
- "tpp_threshold_2_intended_diff_only": 0.0059999823570251465,
243
- "tpp_threshold_2_unintended_diff_only": -0.004749998450279236,
244
- "tpp_threshold_5_total_metric": 0.011000022292137146,
245
- "tpp_threshold_5_intended_diff_only": 0.008000016212463379,
246
- "tpp_threshold_5_unintended_diff_only": -0.003000006079673767,
247
- "tpp_threshold_10_total_metric": 0.0014999955892562866,
248
- "tpp_threshold_10_intended_diff_only": 0.0059999823570251465,
249
- "tpp_threshold_10_unintended_diff_only": 0.00449998676776886
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": 0.0045000165700912476,
253
- "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
254
- "tpp_threshold_2_unintended_diff_only": 0.00449998676776886,
255
- "tpp_threshold_5_total_metric": 0.011500030755996704,
256
- "tpp_threshold_5_intended_diff_only": 0.018000006675720215,
257
- "tpp_threshold_5_unintended_diff_only": 0.006499975919723511,
258
- "tpp_threshold_10_total_metric": 0.028500065207481384,
259
- "tpp_threshold_10_intended_diff_only": 0.033000051975250244,
260
- "tpp_threshold_10_unintended_diff_only": 0.00449998676776886
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.009499996900558472,
264
  "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
265
- "tpp_threshold_2_unintended_diff_only": -0.007500022649765015,
266
- "tpp_threshold_5_total_metric": 0.01600000262260437,
267
- "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
268
- "tpp_threshold_5_unintended_diff_only": -0.0020000040531158447,
269
- "tpp_threshold_10_total_metric": 0.01799999177455902,
270
- "tpp_threshold_10_intended_diff_only": 0.019999980926513672,
271
- "tpp_threshold_10_unintended_diff_only": 0.001999989151954651
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.00850003957748413,
275
- "tpp_threshold_2_intended_diff_only": -0.001000046730041504,
276
- "tpp_threshold_2_unintended_diff_only": 0.007499992847442627,
277
- "tpp_threshold_5_total_metric": 0.007249951362609863,
278
- "tpp_threshold_5_intended_diff_only": 0.008999943733215332,
279
- "tpp_threshold_5_unintended_diff_only": 0.0017499923706054688,
280
- "tpp_threshold_10_total_metric": 0.011000007390975952,
281
- "tpp_threshold_10_intended_diff_only": 0.013999998569488525,
282
- "tpp_threshold_10_unintended_diff_only": 0.0029999911785125732
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.021750018000602722,
286
- "tpp_threshold_2_intended_diff_only": 0.022000014781951904,
287
- "tpp_threshold_2_unintended_diff_only": 0.00024999678134918213,
288
- "tpp_threshold_5_total_metric": 0.023999959230422974,
289
- "tpp_threshold_5_intended_diff_only": 0.030999958515167236,
290
- "tpp_threshold_5_unintended_diff_only": 0.006999999284744263,
291
- "tpp_threshold_10_total_metric": 0.05249999463558197,
292
- "tpp_threshold_10_intended_diff_only": 0.05500000715255737,
293
- "tpp_threshold_10_unintended_diff_only": 0.002500012516975403
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "47881ec7-cf88-4cf8-8d53-ae708a7cbe4b",
73
+ "datetime_epoch_millis": 1777317677846,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.028174997866153715,
77
+ "tpp_threshold_2_intended_diff_only": 0.03320000171661377,
78
+ "tpp_threshold_2_unintended_diff_only": 0.005025003850460053,
79
+ "tpp_threshold_5_total_metric": 0.07472501248121262,
80
+ "tpp_threshold_5_intended_diff_only": 0.09870001673698425,
81
+ "tpp_threshold_5_unintended_diff_only": 0.023975004255771638,
82
+ "tpp_threshold_10_total_metric": 0.09890000522136688,
83
+ "tpp_threshold_10_intended_diff_only": 0.15510001182556152,
84
+ "tpp_threshold_10_unintended_diff_only": 0.05620000660419464,
85
+ "tpp_threshold_20_total_metric": 0.1227250024676323,
86
+ "tpp_threshold_20_intended_diff_only": 0.21230001449584962,
87
+ "tpp_threshold_20_unintended_diff_only": 0.08957501202821731,
88
+ "tpp_threshold_50_total_metric": 0.1471000075340271,
89
+ "tpp_threshold_50_intended_diff_only": 0.2677000164985657,
90
+ "tpp_threshold_50_unintended_diff_only": 0.12060000896453858,
91
+ "tpp_threshold_100_total_metric": 0.17207499444484708,
92
+ "tpp_threshold_100_intended_diff_only": 0.32460000514984133,
93
+ "tpp_threshold_100_unintended_diff_only": 0.1525250107049942,
94
+ "tpp_threshold_500_total_metric": 0.23282500952482224,
95
+ "tpp_threshold_500_intended_diff_only": 0.43890002369880676,
96
+ "tpp_threshold_500_unintended_diff_only": 0.20607501417398452
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.0494499921798706,
103
+ "tpp_threshold_2_intended_diff_only": 0.05880000591278076,
104
+ "tpp_threshold_2_unintended_diff_only": 0.009350013732910157,
105
+ "tpp_threshold_5_total_metric": 0.13520001471042634,
106
+ "tpp_threshold_5_intended_diff_only": 0.182800030708313,
107
+ "tpp_threshold_5_unintended_diff_only": 0.04760001599788666,
108
+ "tpp_threshold_10_total_metric": 0.17215000987052917,
109
+ "tpp_threshold_10_intended_diff_only": 0.2818000316619873,
110
+ "tpp_threshold_10_unintended_diff_only": 0.10965002179145814,
111
+ "tpp_threshold_20_total_metric": 0.1992499977350235,
112
+ "tpp_threshold_20_intended_diff_only": 0.3744000196456909,
113
+ "tpp_threshold_20_unintended_diff_only": 0.17515002191066742,
114
+ "tpp_threshold_50_total_metric": 0.1827000081539154,
115
+ "tpp_threshold_50_intended_diff_only": 0.41860003471374513,
116
+ "tpp_threshold_50_unintended_diff_only": 0.23590002655982972,
117
+ "tpp_threshold_100_total_metric": 0.14964999556541442,
118
+ "tpp_threshold_100_intended_diff_only": 0.4428000211715698,
119
+ "tpp_threshold_100_unintended_diff_only": 0.2931500256061554,
120
+ "tpp_threshold_500_total_metric": 0.08850001096725464,
121
+ "tpp_threshold_500_intended_diff_only": 0.4674000382423401,
122
+ "tpp_threshold_500_unintended_diff_only": 0.3789000272750854
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.006900003552436829,
127
+ "tpp_threshold_2_intended_diff_only": 0.0075999975204467775,
128
+ "tpp_threshold_2_unintended_diff_only": 0.0006999939680099488,
129
+ "tpp_threshold_5_total_metric": 0.014250010251998901,
130
+ "tpp_threshold_5_intended_diff_only": 0.014600002765655517,
131
+ "tpp_threshold_5_unintended_diff_only": 0.0003499925136566162,
132
+ "tpp_threshold_10_total_metric": 0.02565000057220459,
133
+ "tpp_threshold_10_intended_diff_only": 0.02839999198913574,
134
+ "tpp_threshold_10_unintended_diff_only": 0.002749991416931152,
135
+ "tpp_threshold_20_total_metric": 0.04620000720024109,
136
+ "tpp_threshold_20_intended_diff_only": 0.0502000093460083,
137
+ "tpp_threshold_20_unintended_diff_only": 0.004000002145767212,
138
+ "tpp_threshold_50_total_metric": 0.11150000691413879,
139
+ "tpp_threshold_50_intended_diff_only": 0.11679999828338623,
140
+ "tpp_threshold_50_unintended_diff_only": 0.005299991369247437,
141
+ "tpp_threshold_100_total_metric": 0.19449999332427978,
142
+ "tpp_threshold_100_intended_diff_only": 0.2063999891281128,
143
+ "tpp_threshold_100_unintended_diff_only": 0.011899995803833007,
144
+ "tpp_threshold_500_total_metric": 0.37715000808238985,
145
+ "tpp_threshold_500_intended_diff_only": 0.4104000091552734,
146
+ "tpp_threshold_500_unintended_diff_only": 0.0332500010728836
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.0017500221729278564,
194
  "tpp_threshold_10_total_metric": 0.05774995684623718,
195
  "tpp_threshold_10_intended_diff_only": 0.06599998474121094,
196
+ "tpp_threshold_10_unintended_diff_only": 0.008250027894973755,
197
+ "tpp_threshold_20_total_metric": 0.21074995398521423,
198
+ "tpp_threshold_20_intended_diff_only": 0.3579999804496765,
199
+ "tpp_threshold_20_unintended_diff_only": 0.14725002646446228,
200
+ "tpp_threshold_50_total_metric": 0.18774999678134918,
201
+ "tpp_threshold_50_intended_diff_only": 0.4150000214576721,
202
+ "tpp_threshold_50_unintended_diff_only": 0.22725002467632294,
203
+ "tpp_threshold_100_total_metric": 0.16274996101856232,
204
+ "tpp_threshold_100_intended_diff_only": 0.4359999895095825,
205
+ "tpp_threshold_100_unintended_diff_only": 0.2732500284910202,
206
+ "tpp_threshold_500_total_metric": 0.08550000190734863,
207
+ "tpp_threshold_500_intended_diff_only": 0.45600003004074097,
208
+ "tpp_threshold_500_unintended_diff_only": 0.37050002813339233
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.07274998724460602,
212
+ "tpp_threshold_2_intended_diff_only": 0.08300000429153442,
213
+ "tpp_threshold_2_unintended_diff_only": 0.010250017046928406,
214
+ "tpp_threshold_5_total_metric": 0.18324998021125793,
215
+ "tpp_threshold_5_intended_diff_only": 0.28200000524520874,
216
+ "tpp_threshold_5_unintended_diff_only": 0.0987500250339508,
217
+ "tpp_threshold_10_total_metric": 0.1834999918937683,
218
  "tpp_threshold_10_intended_diff_only": 0.3370000123977661,
219
+ "tpp_threshold_10_unintended_diff_only": 0.1535000205039978,
220
+ "tpp_threshold_20_total_metric": 0.18474997580051422,
221
+ "tpp_threshold_20_intended_diff_only": 0.35600000619888306,
222
+ "tpp_threshold_20_unintended_diff_only": 0.17125003039836884,
223
+ "tpp_threshold_50_total_metric": 0.1900000125169754,
224
+ "tpp_threshold_50_intended_diff_only": 0.40400004386901855,
225
+ "tpp_threshold_50_unintended_diff_only": 0.21400003135204315,
226
+ "tpp_threshold_100_total_metric": 0.16724997758865356,
227
+ "tpp_threshold_100_intended_diff_only": 0.4350000023841858,
228
+ "tpp_threshold_100_unintended_diff_only": 0.2677500247955322,
229
+ "tpp_threshold_500_total_metric": 0.07975003123283386,
230
+ "tpp_threshold_500_intended_diff_only": 0.4590000510215759,
231
+ "tpp_threshold_500_unintended_diff_only": 0.37925001978874207
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.1485000103712082,
235
+ "tpp_threshold_2_intended_diff_only": 0.18300002813339233,
236
+ "tpp_threshold_2_unintended_diff_only": 0.03450001776218414,
237
+ "tpp_threshold_5_total_metric": 0.21525004506111145,
238
  "tpp_threshold_5_intended_diff_only": 0.28700006008148193,
239
+ "tpp_threshold_5_unintended_diff_only": 0.07175001502037048,
240
+ "tpp_threshold_10_total_metric": 0.1587500423192978,
241
+ "tpp_threshold_10_intended_diff_only": 0.3420000672340393,
242
+ "tpp_threshold_10_unintended_diff_only": 0.18325002491474152,
243
+ "tpp_threshold_20_total_metric": 0.109250009059906,
244
+ "tpp_threshold_20_intended_diff_only": 0.359000027179718,
245
+ "tpp_threshold_20_unintended_diff_only": 0.249750018119812,
246
+ "tpp_threshold_50_total_metric": 0.09399999678134918,
247
+ "tpp_threshold_50_intended_diff_only": 0.38200002908706665,
248
+ "tpp_threshold_50_unintended_diff_only": 0.28800003230571747,
249
+ "tpp_threshold_100_total_metric": 0.0585000216960907,
250
+ "tpp_threshold_100_intended_diff_only": 0.40800005197525024,
251
+ "tpp_threshold_100_unintended_diff_only": 0.34950003027915955,
252
+ "tpp_threshold_500_total_metric": 0.04674999415874481,
253
+ "tpp_threshold_500_intended_diff_only": 0.4520000219345093,
254
+ "tpp_threshold_500_unintended_diff_only": 0.40525002777576447
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.009499981999397278,
258
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704,
260
+ "tpp_threshold_5_total_metric": 0.22300003468990326,
261
+ "tpp_threshold_5_intended_diff_only": 0.26500004529953003,
262
+ "tpp_threshold_5_unintended_diff_only": 0.04200001060962677,
263
+ "tpp_threshold_10_total_metric": 0.2657500356435776,
264
  "tpp_threshold_10_intended_diff_only": 0.3840000629425049,
265
+ "tpp_threshold_10_unintended_diff_only": 0.11825002729892731,
266
+ "tpp_threshold_20_total_metric": 0.27650001645088196,
267
+ "tpp_threshold_20_intended_diff_only": 0.45100003480911255,
268
+ "tpp_threshold_20_unintended_diff_only": 0.1745000183582306,
269
+ "tpp_threshold_50_total_metric": 0.2227499932050705,
270
+ "tpp_threshold_50_intended_diff_only": 0.48000001907348633,
271
+ "tpp_threshold_50_unintended_diff_only": 0.25725002586841583,
272
+ "tpp_threshold_100_total_metric": 0.1785000115633011,
273
+ "tpp_threshold_100_intended_diff_only": 0.4830000400543213,
274
+ "tpp_threshold_100_unintended_diff_only": 0.3045000284910202,
275
+ "tpp_threshold_500_total_metric": 0.12024997174739838,
276
+ "tpp_threshold_500_intended_diff_only": 0.49400001764297485,
277
+ "tpp_threshold_500_unintended_diff_only": 0.3737500458955765
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.0072500258684158325,
281
+ "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464,
283
+ "tpp_threshold_5_total_metric": 0.03825002908706665,
284
+ "tpp_threshold_5_intended_diff_only": 0.06200003623962402,
285
+ "tpp_threshold_5_unintended_diff_only": 0.023750007152557373,
286
+ "tpp_threshold_10_total_metric": 0.19500002264976501,
287
+ "tpp_threshold_10_intended_diff_only": 0.2800000309944153,
288
+ "tpp_threshold_10_unintended_diff_only": 0.08500000834465027,
289
+ "tpp_threshold_20_total_metric": 0.21500003337860107,
290
+ "tpp_threshold_20_intended_diff_only": 0.34800004959106445,
291
+ "tpp_threshold_20_unintended_diff_only": 0.13300001621246338,
292
+ "tpp_threshold_50_total_metric": 0.21900004148483276,
293
+ "tpp_threshold_50_intended_diff_only": 0.41200006008148193,
294
+ "tpp_threshold_50_unintended_diff_only": 0.19300001859664917,
295
+ "tpp_threshold_100_total_metric": 0.18125000596046448,
296
+ "tpp_threshold_100_intended_diff_only": 0.4520000219345093,
297
+ "tpp_threshold_100_unintended_diff_only": 0.2707500159740448,
298
+ "tpp_threshold_500_total_metric": 0.11025005578994751,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.3657500147819519
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.005000025033950806,
306
+ "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
307
+ "tpp_threshold_2_unintended_diff_only": 0.0029999911785125732,
308
+ "tpp_threshold_5_total_metric": 0.005499988794326782,
309
+ "tpp_threshold_5_intended_diff_only": 0.0059999823570251465,
310
+ "tpp_threshold_5_unintended_diff_only": 0.0004999935626983643,
311
+ "tpp_threshold_10_total_metric": 0.0032500028610229492,
312
+ "tpp_threshold_10_intended_diff_only": 0.004999995231628418,
313
+ "tpp_threshold_10_unintended_diff_only": 0.0017499923706054688,
314
+ "tpp_threshold_20_total_metric": 0.01825001835823059,
315
+ "tpp_threshold_20_intended_diff_only": 0.022000014781951904,
316
+ "tpp_threshold_20_unintended_diff_only": 0.0037499964237213135,
317
+ "tpp_threshold_50_total_metric": 0.06300000846385956,
318
+ "tpp_threshold_50_intended_diff_only": 0.06099998950958252,
319
+ "tpp_threshold_50_unintended_diff_only": -0.0020000189542770386,
320
+ "tpp_threshold_100_total_metric": 0.14749999344348907,
321
+ "tpp_threshold_100_intended_diff_only": 0.15700000524520874,
322
+ "tpp_threshold_100_unintended_diff_only": 0.009500011801719666,
323
+ "tpp_threshold_500_total_metric": 0.41724999248981476,
324
+ "tpp_threshold_500_intended_diff_only": 0.4359999895095825,
325
+ "tpp_threshold_500_unintended_diff_only": 0.01874999701976776
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.013750046491622925,
329
+ "tpp_threshold_2_intended_diff_only": 0.01100003719329834,
330
+ "tpp_threshold_2_unintended_diff_only": -0.002750009298324585,
331
+ "tpp_threshold_5_total_metric": 0.007750064134597778,
332
+ "tpp_threshold_5_intended_diff_only": 0.01500004529953003,
333
+ "tpp_threshold_5_unintended_diff_only": 0.007249981164932251,
334
+ "tpp_threshold_10_total_metric": 0.030500024557113647,
335
+ "tpp_threshold_10_intended_diff_only": 0.03600001335144043,
336
+ "tpp_threshold_10_unintended_diff_only": 0.005499988794326782,
337
+ "tpp_threshold_20_total_metric": 0.07225003838539124,
338
+ "tpp_threshold_20_intended_diff_only": 0.07600003480911255,
339
+ "tpp_threshold_20_unintended_diff_only": 0.0037499964237213135,
340
+ "tpp_threshold_50_total_metric": 0.15050005912780762,
341
+ "tpp_threshold_50_intended_diff_only": 0.16300004720687866,
342
+ "tpp_threshold_50_unintended_diff_only": 0.012499988079071045,
343
+ "tpp_threshold_100_total_metric": 0.24500001966953278,
344
+ "tpp_threshold_100_intended_diff_only": 0.2630000114440918,
345
+ "tpp_threshold_100_unintended_diff_only": 0.01799999177455902,
346
+ "tpp_threshold_500_total_metric": 0.40800003707408905,
347
+ "tpp_threshold_500_intended_diff_only": 0.43800002336502075,
348
+ "tpp_threshold_500_unintended_diff_only": 0.0299999862909317
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.0034999847412109375,
352
  "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
353
+ "tpp_threshold_2_unintended_diff_only": -0.0015000104904174805,
354
+ "tpp_threshold_5_total_metric": 0.012750029563903809,
355
+ "tpp_threshold_5_intended_diff_only": 0.00700002908706665,
356
+ "tpp_threshold_5_unintended_diff_only": -0.005750000476837158,
357
+ "tpp_threshold_10_total_metric": 0.020500019192695618,
358
+ "tpp_threshold_10_intended_diff_only": 0.018000006675720215,
359
+ "tpp_threshold_10_unintended_diff_only": -0.002500012516975403,
360
+ "tpp_threshold_20_total_metric": 0.007250010967254639,
361
+ "tpp_threshold_20_intended_diff_only": 0.009000003337860107,
362
+ "tpp_threshold_20_unintended_diff_only": 0.0017499923706054688,
363
+ "tpp_threshold_50_total_metric": 0.04975003004074097,
364
+ "tpp_threshold_50_intended_diff_only": 0.050000011920928955,
365
+ "tpp_threshold_50_unintended_diff_only": 0.0002499818801879883,
366
+ "tpp_threshold_100_total_metric": 0.09274999797344208,
367
+ "tpp_threshold_100_intended_diff_only": 0.09799998998641968,
368
+ "tpp_threshold_100_unintended_diff_only": 0.0052499920129776,
369
+ "tpp_threshold_500_total_metric": 0.35349999368190765,
370
+ "tpp_threshold_500_intended_diff_only": 0.3889999985694885,
371
+ "tpp_threshold_500_unintended_diff_only": 0.03550000488758087
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.0,
375
+ "tpp_threshold_2_intended_diff_only": 0.0,
376
+ "tpp_threshold_2_unintended_diff_only": 0.0,
377
+ "tpp_threshold_5_total_metric": 0.01649998128414154,
378
+ "tpp_threshold_5_intended_diff_only": 0.015999972820281982,
379
+ "tpp_threshold_5_unintended_diff_only": -0.0005000084638595581,
380
+ "tpp_threshold_10_total_metric": 0.007249966263771057,
381
+ "tpp_threshold_10_intended_diff_only": 0.012999951839447021,
382
+ "tpp_threshold_10_unintended_diff_only": 0.005749985575675964,
383
+ "tpp_threshold_20_total_metric": 0.0274999737739563,
384
+ "tpp_threshold_20_intended_diff_only": 0.02899998426437378,
385
+ "tpp_threshold_20_unintended_diff_only": 0.0015000104904174805,
386
+ "tpp_threshold_50_total_metric": 0.1149999350309372,
387
+ "tpp_threshold_50_intended_diff_only": 0.1239999532699585,
388
+ "tpp_threshold_50_unintended_diff_only": 0.009000018239021301,
389
+ "tpp_threshold_100_total_metric": 0.2109999656677246,
390
+ "tpp_threshold_100_intended_diff_only": 0.22499996423721313,
391
+ "tpp_threshold_100_unintended_diff_only": 0.013999998569488525,
392
+ "tpp_threshold_500_total_metric": 0.3739999979734421,
393
+ "tpp_threshold_500_intended_diff_only": 0.41600000858306885,
394
+ "tpp_threshold_500_unintended_diff_only": 0.04200001060962677
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.012249961495399475,
398
+ "tpp_threshold_2_intended_diff_only": 0.01699995994567871,
399
+ "tpp_threshold_2_unintended_diff_only": 0.004749998450279236,
400
+ "tpp_threshold_5_total_metric": 0.028749987483024597,
401
+ "tpp_threshold_5_intended_diff_only": 0.02899998426437378,
402
+ "tpp_threshold_5_unintended_diff_only": 0.00024999678134918213,
403
+ "tpp_threshold_10_total_metric": 0.06674998998641968,
404
+ "tpp_threshold_10_intended_diff_only": 0.06999999284744263,
405
+ "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492,
406
+ "tpp_threshold_20_total_metric": 0.10574999451637268,
407
+ "tpp_threshold_20_intended_diff_only": 0.11500000953674316,
408
+ "tpp_threshold_20_unintended_diff_only": 0.009250015020370483,
409
+ "tpp_threshold_50_total_metric": 0.17925000190734863,
410
+ "tpp_threshold_50_intended_diff_only": 0.18599998950958252,
411
+ "tpp_threshold_50_unintended_diff_only": 0.006749987602233887,
412
+ "tpp_threshold_100_total_metric": 0.2762499898672104,
413
+ "tpp_threshold_100_intended_diff_only": 0.2889999747276306,
414
+ "tpp_threshold_100_unintended_diff_only": 0.012749984860420227,
415
+ "tpp_threshold_500_total_metric": 0.3330000191926956,
416
+ "tpp_threshold_500_intended_diff_only": 0.37300002574920654,
417
+ "tpp_threshold_500_unintended_diff_only": 0.040000006556510925
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-341796-tokens-1400000000/scr/saebench_scr_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,8 +69,8 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "ee0d2024-0f8a-447a-8cf2-2d691b0736f7",
69
- "datetime_epoch_millis": 1776886027710,
70
  "eval_result_metrics": {
71
  "scr_metrics": {
72
  "scr_dir1_threshold_2": 0.29573145348122126,
@@ -78,18 +82,18 @@
78
  "scr_dir1_threshold_10": 0.318147144192202,
79
  "scr_metric_threshold_10": 0.2178056251175732,
80
  "scr_dir2_threshold_10": 0.2061909045784877,
81
- "scr_dir1_threshold_20": null,
82
- "scr_metric_threshold_20": null,
83
- "scr_dir2_threshold_20": null,
84
- "scr_dir1_threshold_50": null,
85
- "scr_metric_threshold_50": null,
86
- "scr_dir2_threshold_50": null,
87
- "scr_dir1_threshold_100": null,
88
- "scr_metric_threshold_100": null,
89
- "scr_dir2_threshold_100": null,
90
- "scr_dir1_threshold_500": null,
91
- "scr_metric_threshold_500": null,
92
- "scr_dir2_threshold_500": null
93
  }
94
  },
95
  "eval_result_details": [
@@ -104,18 +108,18 @@
104
  "scr_dir1_threshold_10": 0.5641028776049328,
105
  "scr_metric_threshold_10": 0.036269422851059746,
106
  "scr_dir2_threshold_10": 0.036269422851059746,
107
- "scr_dir1_threshold_20": null,
108
- "scr_metric_threshold_20": null,
109
- "scr_dir2_threshold_20": null,
110
- "scr_dir1_threshold_50": null,
111
- "scr_metric_threshold_50": null,
112
- "scr_dir2_threshold_50": null,
113
- "scr_dir1_threshold_100": null,
114
- "scr_metric_threshold_100": null,
115
- "scr_dir2_threshold_100": null,
116
- "scr_dir1_threshold_500": null,
117
- "scr_metric_threshold_500": null,
118
- "scr_dir2_threshold_500": null
119
  },
120
  {
121
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
@@ -128,18 +132,18 @@
128
  "scr_dir1_threshold_10": 0.3636365688864726,
129
  "scr_metric_threshold_10": 0.34250758728798886,
130
  "scr_dir2_threshold_10": 0.34250758728798886,
131
- "scr_dir1_threshold_20": null,
132
- "scr_metric_threshold_20": null,
133
- "scr_dir2_threshold_20": null,
134
- "scr_dir1_threshold_50": null,
135
- "scr_metric_threshold_50": null,
136
- "scr_dir2_threshold_50": null,
137
- "scr_dir1_threshold_100": null,
138
- "scr_metric_threshold_100": null,
139
- "scr_dir2_threshold_100": null,
140
- "scr_dir1_threshold_500": null,
141
- "scr_metric_threshold_500": null,
142
- "scr_dir2_threshold_500": null
143
  },
144
  {
145
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
@@ -152,18 +156,18 @@
152
  "scr_dir1_threshold_10": 0.4687504074535596,
153
  "scr_metric_threshold_10": 0.10301508102112025,
154
  "scr_dir2_threshold_10": 0.10301508102112025,
155
- "scr_dir1_threshold_20": null,
156
- "scr_metric_threshold_20": null,
157
- "scr_dir2_threshold_20": null,
158
- "scr_dir1_threshold_50": null,
159
- "scr_metric_threshold_50": null,
160
- "scr_dir2_threshold_50": null,
161
- "scr_dir1_threshold_100": null,
162
- "scr_metric_threshold_100": null,
163
- "scr_dir2_threshold_100": null,
164
- "scr_dir1_threshold_500": null,
165
- "scr_metric_threshold_500": null,
166
- "scr_dir2_threshold_500": null
167
  },
168
  {
169
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
@@ -176,18 +180,18 @@
176
  "scr_dir1_threshold_10": -0.03355741747856333,
177
  "scr_metric_threshold_10": 0.08917196243161127,
178
  "scr_dir2_threshold_10": 0.08917196243161127,
179
- "scr_dir1_threshold_20": null,
180
- "scr_metric_threshold_20": null,
181
- "scr_dir2_threshold_20": null,
182
- "scr_dir1_threshold_50": null,
183
- "scr_metric_threshold_50": null,
184
- "scr_dir2_threshold_50": null,
185
- "scr_dir1_threshold_100": null,
186
- "scr_metric_threshold_100": null,
187
- "scr_dir2_threshold_100": null,
188
- "scr_dir1_threshold_500": null,
189
- "scr_metric_threshold_500": null,
190
- "scr_dir2_threshold_500": null
191
  },
192
  {
193
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
@@ -200,18 +204,18 @@
200
  "scr_dir1_threshold_10": 0.09473687842942387,
201
  "scr_metric_threshold_10": 0.4325397322853009,
202
  "scr_dir2_threshold_10": 0.4325397322853009,
203
- "scr_dir1_threshold_20": null,
204
- "scr_metric_threshold_20": null,
205
- "scr_dir2_threshold_20": null,
206
- "scr_dir1_threshold_50": null,
207
- "scr_metric_threshold_50": null,
208
- "scr_dir2_threshold_50": null,
209
- "scr_dir1_threshold_100": null,
210
- "scr_metric_threshold_100": null,
211
- "scr_dir2_threshold_100": null,
212
- "scr_dir1_threshold_500": null,
213
- "scr_metric_threshold_500": null,
214
- "scr_dir2_threshold_500": null
215
  },
216
  {
217
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
@@ -224,18 +228,18 @@
224
  "scr_dir1_threshold_10": 0.4316940424658739,
225
  "scr_metric_threshold_10": 0.1183206688641517,
226
  "scr_dir2_threshold_10": 0.1183206688641517,
227
- "scr_dir1_threshold_20": null,
228
- "scr_metric_threshold_20": null,
229
- "scr_dir2_threshold_20": null,
230
- "scr_dir1_threshold_50": null,
231
- "scr_metric_threshold_50": null,
232
- "scr_dir2_threshold_50": null,
233
- "scr_dir1_threshold_100": null,
234
- "scr_metric_threshold_100": null,
235
- "scr_dir2_threshold_100": null,
236
- "scr_dir1_threshold_500": null,
237
- "scr_metric_threshold_500": null,
238
- "scr_dir2_threshold_500": null
239
  },
240
  {
241
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
@@ -248,18 +252,18 @@
248
  "scr_dir1_threshold_10": 0.40000005544617256,
249
  "scr_metric_threshold_10": 0.3648068054696092,
250
  "scr_dir2_threshold_10": 0.3648068054696092,
251
- "scr_dir1_threshold_20": null,
252
- "scr_metric_threshold_20": null,
253
- "scr_dir2_threshold_20": null,
254
- "scr_dir1_threshold_50": null,
255
- "scr_metric_threshold_50": null,
256
- "scr_dir2_threshold_50": null,
257
- "scr_dir1_threshold_100": null,
258
- "scr_metric_threshold_100": null,
259
- "scr_dir2_threshold_100": null,
260
- "scr_dir1_threshold_500": null,
261
- "scr_metric_threshold_500": null,
262
- "scr_dir2_threshold_500": null
263
  },
264
  {
265
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
@@ -272,18 +276,18 @@
272
  "scr_dir1_threshold_10": 0.2558137407297439,
273
  "scr_metric_threshold_10": 0.2558137407297439,
274
  "scr_dir2_threshold_10": 0.16289597641705988,
275
- "scr_dir1_threshold_20": null,
276
- "scr_metric_threshold_20": null,
277
- "scr_dir2_threshold_20": null,
278
- "scr_dir1_threshold_50": null,
279
- "scr_metric_threshold_50": null,
280
- "scr_dir2_threshold_50": null,
281
- "scr_dir1_threshold_100": null,
282
- "scr_metric_threshold_100": null,
283
- "scr_dir2_threshold_100": null,
284
- "scr_dir1_threshold_500": null,
285
- "scr_metric_threshold_500": null,
286
- "scr_dir2_threshold_500": null
287
  }
288
  ],
289
  "sae_bench_commit_hash": "Unknown",
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "4ae4fbf3-e7ee-418e-b465-45cd831027dc",
73
+ "datetime_epoch_millis": 1777317523062,
74
  "eval_result_metrics": {
75
  "scr_metrics": {
76
  "scr_dir1_threshold_2": 0.29573145348122126,
 
82
  "scr_dir1_threshold_10": 0.318147144192202,
83
  "scr_metric_threshold_10": 0.2178056251175732,
84
  "scr_dir2_threshold_10": 0.2061909045784877,
85
+ "scr_dir1_threshold_20": 0.2715260894565377,
86
+ "scr_metric_threshold_20": 0.2595251976969898,
87
+ "scr_dir2_threshold_20": 0.24778416638655848,
88
+ "scr_dir1_threshold_50": 0.3220415073303419,
89
+ "scr_metric_threshold_50": 0.3325713457900344,
90
+ "scr_dir2_threshold_50": 0.3218352782092032,
91
+ "scr_dir1_threshold_100": 0.07808705309726434,
92
+ "scr_metric_threshold_100": 0.3342134066696645,
93
+ "scr_dir2_threshold_100": 0.3154009425730885,
94
+ "scr_dir1_threshold_500": -1.1534098777555588,
95
+ "scr_metric_threshold_500": 0.15969367766919482,
96
+ "scr_dir2_threshold_500": 0.13723497080528102
97
  }
98
  },
99
  "eval_result_details": [
 
108
  "scr_dir1_threshold_10": 0.5641028776049328,
109
  "scr_metric_threshold_10": 0.036269422851059746,
110
  "scr_dir2_threshold_10": 0.036269422851059746,
111
+ "scr_dir1_threshold_20": 0.2820514388024664,
112
+ "scr_metric_threshold_20": 0.14248705143262683,
113
+ "scr_dir2_threshold_20": 0.14248705143262683,
114
+ "scr_dir1_threshold_50": 0.5128207283533913,
115
+ "scr_metric_threshold_50": 0.2616580342884179,
116
+ "scr_dir2_threshold_50": 0.2616580342884179,
117
+ "scr_dir1_threshold_100": 0.5256410746257707,
118
+ "scr_metric_threshold_100": 0.3082901714420895,
119
+ "scr_dir2_threshold_100": 0.3082901714420895,
120
+ "scr_dir1_threshold_500": -2.564102113442909,
121
+ "scr_metric_threshold_500": 0.025906708548447916,
122
+ "scr_dir2_threshold_500": 0.025906708548447916
123
  },
124
  {
125
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
 
132
  "scr_dir1_threshold_10": 0.3636365688864726,
133
  "scr_metric_threshold_10": 0.34250758728798886,
134
  "scr_dir2_threshold_10": 0.34250758728798886,
135
+ "scr_dir1_threshold_20": 0.5454546275545891,
136
+ "scr_metric_threshold_20": 0.22629968192630534,
137
+ "scr_dir2_threshold_20": 0.22629968192630534,
138
+ "scr_dir1_threshold_50": 0.37878788562954907,
139
+ "scr_metric_threshold_50": 0.44954129945407983,
140
+ "scr_dir2_threshold_50": 0.44954129945407983,
141
+ "scr_dir1_threshold_100": 0.128787998517109,
142
+ "scr_metric_threshold_100": 0.553516764944451,
143
+ "scr_dir2_threshold_100": 0.553516764944451,
144
+ "scr_dir1_threshold_500": -2.0454539502292293,
145
+ "scr_metric_threshold_500": -0.24770631499321016,
146
+ "scr_dir2_threshold_500": -0.24770631499321016
147
  },
148
  {
149
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
 
156
  "scr_dir1_threshold_10": 0.4687504074535596,
157
  "scr_metric_threshold_10": 0.10301508102112025,
158
  "scr_dir2_threshold_10": 0.10301508102112025,
159
+ "scr_dir1_threshold_20": 0.2656252619344312,
160
+ "scr_metric_threshold_20": 0.1809045617464338,
161
+ "scr_dir2_threshold_20": 0.1809045617464338,
162
+ "scr_dir1_threshold_50": 0.32812537834973393,
163
+ "scr_metric_threshold_50": 0.19346736189433716,
164
+ "scr_dir2_threshold_50": 0.19346736189433716,
165
+ "scr_dir1_threshold_100": -0.15624982537704588,
166
+ "scr_metric_threshold_100": 0.037688550204100946,
167
+ "scr_dir2_threshold_100": 0.037688550204100946,
168
+ "scr_dir1_threshold_500": -4.249999534338789,
169
+ "scr_metric_threshold_500": -0.052763730669115945,
170
+ "scr_dir2_threshold_500": -0.052763730669115945
171
  },
172
  {
173
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
 
180
  "scr_dir1_threshold_10": -0.03355741747856333,
181
  "scr_metric_threshold_10": 0.08917196243161127,
182
  "scr_dir2_threshold_10": 0.08917196243161127,
183
+ "scr_dir1_threshold_20": -0.2617452161268579,
184
+ "scr_metric_threshold_20": 0.00955420662613804,
185
+ "scr_dir2_threshold_20": 0.00955420662613804,
186
+ "scr_dir1_threshold_50": 0.14765071675590696,
187
+ "scr_metric_threshold_50": -0.07324841127051662,
188
+ "scr_dir2_threshold_50": -0.07324841127051662,
189
+ "scr_dir1_threshold_100": -1.167785487268007,
190
+ "scr_metric_threshold_100": -0.2611465427598772,
191
+ "scr_dir2_threshold_100": -0.2611465427598772,
192
+ "scr_dir1_threshold_500": -1.6241614845957986,
193
+ "scr_metric_threshold_500": -0.5414013089483273,
194
+ "scr_dir2_threshold_500": -0.5414013089483273
195
  },
196
  {
197
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
 
204
  "scr_dir1_threshold_10": 0.09473687842942387,
205
  "scr_metric_threshold_10": 0.4325397322853009,
206
  "scr_dir2_threshold_10": 0.4325397322853009,
207
+ "scr_dir1_threshold_20": 0.11052646273721473,
208
+ "scr_metric_threshold_20": 0.5357141758984867,
209
+ "scr_dir2_threshold_20": 0.5357141758984867,
210
+ "scr_dir1_threshold_50": -0.13157882353605502,
211
+ "scr_metric_threshold_50": 0.6785713525451061,
212
+ "scr_dir2_threshold_50": 0.6785713525451061,
213
+ "scr_dir1_threshold_100": -0.11052614902855408,
214
+ "scr_metric_threshold_100": 0.7380950973057522,
215
+ "scr_dir2_threshold_100": 0.7380950973057522,
216
+ "scr_dir1_threshold_500": -0.16315799215163673,
217
+ "scr_metric_threshold_500": 0.773809391467407,
218
+ "scr_dir2_threshold_500": 0.773809391467407
219
  },
220
  {
221
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
 
228
  "scr_dir1_threshold_10": 0.4316940424658739,
229
  "scr_metric_threshold_10": 0.1183206688641517,
230
  "scr_dir2_threshold_10": 0.1183206688641517,
231
+ "scr_dir1_threshold_20": 0.4535520140184937,
232
+ "scr_metric_threshold_20": 0.19465646684173443,
233
+ "scr_dir2_threshold_20": 0.19465646684173443,
234
+ "scr_dir1_threshold_50": 0.4754099855711136,
235
+ "scr_metric_threshold_50": 0.2709922648193172,
236
+ "scr_dir2_threshold_50": 0.2709922648193172,
237
+ "scr_dir1_threshold_100": 0.4699452484015855,
238
+ "scr_metric_threshold_100": 0.3282443408011273,
239
+ "scr_dir2_threshold_100": 0.3282443408011273,
240
+ "scr_dir1_threshold_500": 0.4754099855711136,
241
+ "scr_metric_threshold_500": 0.3320610852002818,
242
+ "scr_dir2_threshold_500": 0.3320610852002818
243
  },
244
  {
245
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
 
252
  "scr_dir1_threshold_10": 0.40000005544617256,
253
  "scr_metric_threshold_10": 0.3648068054696092,
254
  "scr_dir2_threshold_10": 0.3648068054696092,
255
+ "scr_dir1_threshold_20": 0.48372100373097293,
256
+ "scr_metric_threshold_20": 0.493562314103202,
257
+ "scr_dir2_threshold_20": 0.493562314103202,
258
+ "scr_dir1_threshold_50": 0.5348836822468563,
259
+ "scr_metric_threshold_50": 0.549356359317294,
260
+ "scr_dir2_threshold_50": 0.549356359317294,
261
+ "scr_dir1_threshold_100": 0.5581394704114272,
262
+ "scr_metric_threshold_100": 0.5922747769238424,
263
+ "scr_dir2_threshold_100": 0.5922747769238424,
264
+ "scr_dir1_threshold_500": 0.49302320810445616,
265
+ "scr_metric_threshold_500": 0.5364807317097504,
266
+ "scr_dir2_threshold_500": 0.5364807317097504
267
  },
268
  {
269
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
 
276
  "scr_dir1_threshold_10": 0.2558137407297439,
277
  "scr_metric_threshold_10": 0.2558137407297439,
278
  "scr_dir2_threshold_10": 0.16289597641705988,
279
+ "scr_dir1_threshold_20": 0.29302312300099165,
280
+ "scr_metric_threshold_20": 0.29302312300099165,
281
+ "scr_dir2_threshold_20": 0.19909487251754088,
282
+ "scr_dir1_threshold_50": 0.3302325052722394,
283
+ "scr_metric_threshold_50": 0.3302325052722394,
284
+ "scr_dir2_threshold_50": 0.2443439646255898,
285
+ "scr_dir1_threshold_100": 0.3767440944958292,
286
+ "scr_metric_threshold_100": 0.3767440944958292,
287
+ "scr_dir2_threshold_100": 0.2262443817232214,
288
+ "scr_dir1_threshold_500": 0.4511628590383247,
289
+ "scr_metric_threshold_500": 0.4511628590383247,
290
+ "scr_dir2_threshold_500": 0.2714932041270145
291
  }
292
  ],
293
  "sae_bench_commit_hash": "Unknown",
gemma-2-2b-btk/k-100/seed-0/snapshots/step-341796-tokens-1400000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "1046eb3f-8c1b-4ba5-ba3c-594ea58905af",
69
- "datetime_epoch_millis": 1776886177672,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.02099999934434891,
73
- "tpp_threshold_2_intended_diff_only": 0.024000000953674317,
74
- "tpp_threshold_2_unintended_diff_only": 0.003000001609325409,
75
- "tpp_threshold_5_total_metric": 0.07117501050233842,
76
- "tpp_threshold_5_intended_diff_only": 0.09110001325607299,
77
- "tpp_threshold_5_unintended_diff_only": 0.019925002753734586,
78
- "tpp_threshold_10_total_metric": 0.09664999395608902,
79
- "tpp_threshold_10_intended_diff_only": 0.14990000128746034,
80
- "tpp_threshold_10_unintended_diff_only": 0.05325000733137131,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.03469999432563782,
99
- "tpp_threshold_2_intended_diff_only": 0.040400004386901854,
100
- "tpp_threshold_2_unintended_diff_only": 0.005700010061264038,
101
- "tpp_threshold_5_total_metric": 0.12625001072883607,
102
- "tpp_threshold_5_intended_diff_only": 0.16380002498626708,
103
- "tpp_threshold_5_unintended_diff_only": 0.03755001425743103,
104
- "tpp_threshold_10_total_metric": 0.16529999673366547,
105
  "tpp_threshold_10_intended_diff_only": 0.26860001087188723,
106
- "tpp_threshold_10_unintended_diff_only": 0.10330001413822174,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.0073000043630599976,
123
- "tpp_threshold_2_intended_diff_only": 0.0075999975204467775,
124
- "tpp_threshold_2_unintended_diff_only": 0.0002999931573867798,
125
- "tpp_threshold_5_total_metric": 0.016100010275840758,
126
- "tpp_threshold_5_intended_diff_only": 0.018400001525878906,
127
- "tpp_threshold_5_unintended_diff_only": 0.002299991250038147,
128
- "tpp_threshold_10_total_metric": 0.027999991178512575,
129
- "tpp_threshold_10_intended_diff_only": 0.031199991703033447,
130
- "tpp_threshold_10_unintended_diff_only": 0.003200000524520874,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.0020000189542770386,
190
  "tpp_threshold_10_total_metric": 0.011749953031539917,
191
  "tpp_threshold_10_intended_diff_only": 0.014999985694885254,
192
- "tpp_threshold_10_unintended_diff_only": 0.003250032663345337
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": -2.9802322387695312e-08,
196
- "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
197
- "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162,
198
- "tpp_threshold_5_total_metric": 0.20000001788139343,
199
  "tpp_threshold_5_intended_diff_only": 0.25700002908706665,
200
- "tpp_threshold_5_unintended_diff_only": 0.05700001120567322,
201
- "tpp_threshold_10_total_metric": 0.20800000429153442,
202
  "tpp_threshold_10_intended_diff_only": 0.3140000104904175,
203
- "tpp_threshold_10_unintended_diff_only": 0.10600000619888306
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.14650000631809235,
207
- "tpp_threshold_2_intended_diff_only": 0.17400002479553223,
208
- "tpp_threshold_2_unintended_diff_only": 0.02750001847743988,
209
- "tpp_threshold_5_total_metric": 0.20575004816055298,
210
  "tpp_threshold_5_intended_diff_only": 0.2820000648498535,
211
- "tpp_threshold_5_unintended_diff_only": 0.07625001668930054,
212
- "tpp_threshold_10_total_metric": 0.1392500102519989,
213
- "tpp_threshold_10_intended_diff_only": 0.3400000333786011,
214
- "tpp_threshold_10_unintended_diff_only": 0.20075002312660217
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.011750012636184692,
218
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
219
- "tpp_threshold_2_unintended_diff_only": -0.002750009298324585,
220
- "tpp_threshold_5_total_metric": 0.1757500171661377,
221
- "tpp_threshold_5_intended_diff_only": 0.2070000171661377,
222
- "tpp_threshold_5_unintended_diff_only": 0.03125,
223
- "tpp_threshold_10_total_metric": 0.2652500122785568,
224
  "tpp_threshold_10_intended_diff_only": 0.3830000162124634,
225
- "tpp_threshold_10_unintended_diff_only": 0.11775000393390656
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.003999993205070496,
229
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
230
- "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224,
231
- "tpp_threshold_5_total_metric": 0.03475002944469452,
232
  "tpp_threshold_5_intended_diff_only": 0.05600005388259888,
233
- "tpp_threshold_5_unintended_diff_only": 0.021250024437904358,
234
- "tpp_threshold_10_total_metric": 0.20225000381469727,
235
- "tpp_threshold_10_intended_diff_only": 0.29100000858306885,
236
- "tpp_threshold_10_unintended_diff_only": 0.08875000476837158
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.008999988436698914,
242
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
243
- "tpp_threshold_2_unintended_diff_only": -0.003999993205070496,
244
- "tpp_threshold_5_total_metric": 0.01150001585483551,
245
- "tpp_threshold_5_intended_diff_only": 0.009000003337860107,
246
- "tpp_threshold_5_unintended_diff_only": -0.002500012516975403,
247
- "tpp_threshold_10_total_metric": 0.0009999722242355347,
248
- "tpp_threshold_10_intended_diff_only": 0.0059999823570251465,
249
- "tpp_threshold_10_unintended_diff_only": 0.005000010132789612
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": 0.0075000375509262085,
253
- "tpp_threshold_2_intended_diff_only": 0.012000024318695068,
254
- "tpp_threshold_2_unintended_diff_only": 0.00449998676776886,
255
- "tpp_threshold_5_total_metric": 0.023750051856040955,
256
- "tpp_threshold_5_intended_diff_only": 0.030000030994415283,
257
- "tpp_threshold_5_unintended_diff_only": 0.006249979138374329,
258
- "tpp_threshold_10_total_metric": 0.05175001919269562,
259
  "tpp_threshold_10_intended_diff_only": 0.054000020027160645,
260
- "tpp_threshold_10_unintended_diff_only": 0.002250000834465027
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.012000009417533875,
264
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
265
- "tpp_threshold_2_unintended_diff_only": -0.0070000141859054565,
266
- "tpp_threshold_5_total_metric": 0.016499996185302734,
267
- "tpp_threshold_5_intended_diff_only": 0.014999985694885254,
268
- "tpp_threshold_5_unintended_diff_only": -0.0015000104904174805,
269
- "tpp_threshold_10_total_metric": 0.01675000786781311,
270
- "tpp_threshold_10_intended_diff_only": 0.018000006675720215,
271
- "tpp_threshold_10_unintended_diff_only": 0.0012499988079071045
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.005500033497810364,
275
  "tpp_threshold_2_intended_diff_only": 0.0029999613761901855,
276
- "tpp_threshold_2_unintended_diff_only": 0.00849999487400055,
277
- "tpp_threshold_5_total_metric": 0.01299998164176941,
278
- "tpp_threshold_5_intended_diff_only": 0.014999985694885254,
279
- "tpp_threshold_5_unintended_diff_only": 0.0020000040531158447,
280
- "tpp_threshold_10_total_metric": 0.0172499418258667,
281
- "tpp_threshold_10_intended_diff_only": 0.02199995517730713,
282
- "tpp_threshold_10_unintended_diff_only": 0.00475001335144043
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.013500019907951355,
286
- "tpp_threshold_2_intended_diff_only": 0.013000011444091797,
287
- "tpp_threshold_2_unintended_diff_only": -0.0005000084638595581,
288
- "tpp_threshold_5_total_metric": 0.015750005841255188,
289
- "tpp_threshold_5_intended_diff_only": 0.023000001907348633,
290
- "tpp_threshold_5_unintended_diff_only": 0.007249996066093445,
291
- "tpp_threshold_10_total_metric": 0.053250014781951904,
292
- "tpp_threshold_10_intended_diff_only": 0.0559999942779541,
293
- "tpp_threshold_10_unintended_diff_only": 0.0027499794960021973
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "9eacd63d-b991-4aa3-80f2-c5cd6cb7f767",
73
+ "datetime_epoch_millis": 1777317798761,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.020099997520446777,
77
+ "tpp_threshold_2_intended_diff_only": 0.024300003051757814,
78
+ "tpp_threshold_2_unintended_diff_only": 0.004200005531311035,
79
+ "tpp_threshold_5_total_metric": 0.0722500130534172,
80
+ "tpp_threshold_5_intended_diff_only": 0.09150001406669617,
81
+ "tpp_threshold_5_unintended_diff_only": 0.019250001013278964,
82
+ "tpp_threshold_10_total_metric": 0.09729999452829362,
83
+ "tpp_threshold_10_intended_diff_only": 0.15040000081062319,
84
+ "tpp_threshold_10_unintended_diff_only": 0.05310000628232956,
85
+ "tpp_threshold_20_total_metric": 0.1315750077366829,
86
+ "tpp_threshold_20_intended_diff_only": 0.20780001282691954,
87
+ "tpp_threshold_20_unintended_diff_only": 0.07622500509023666,
88
+ "tpp_threshold_50_total_metric": 0.15980000793933868,
89
+ "tpp_threshold_50_intended_diff_only": 0.26720001697540285,
90
+ "tpp_threshold_50_unintended_diff_only": 0.10740000903606414,
91
+ "tpp_threshold_100_total_metric": 0.1829250082373619,
92
+ "tpp_threshold_100_intended_diff_only": 0.3314000189304352,
93
+ "tpp_threshold_100_unintended_diff_only": 0.14847501069307328,
94
+ "tpp_threshold_500_total_metric": 0.2333500251173973,
95
+ "tpp_threshold_500_intended_diff_only": 0.4380000352859497,
96
+ "tpp_threshold_500_unintended_diff_only": 0.2046500101685524
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.034550005197525026,
103
+ "tpp_threshold_2_intended_diff_only": 0.04200001955032349,
104
+ "tpp_threshold_2_unintended_diff_only": 0.007450014352798462,
105
+ "tpp_threshold_5_total_metric": 0.12605001032352448,
106
+ "tpp_threshold_5_intended_diff_only": 0.16400002241134642,
107
+ "tpp_threshold_5_unintended_diff_only": 0.03795001208782196,
108
+ "tpp_threshold_10_total_metric": 0.16494999527931214,
109
  "tpp_threshold_10_intended_diff_only": 0.26860001087188723,
110
+ "tpp_threshold_10_unintended_diff_only": 0.10365001559257507,
111
+ "tpp_threshold_20_total_metric": 0.21355001330375672,
112
+ "tpp_threshold_20_intended_diff_only": 0.3624000310897827,
113
+ "tpp_threshold_20_unintended_diff_only": 0.148850017786026,
114
+ "tpp_threshold_50_total_metric": 0.20535000562667846,
115
+ "tpp_threshold_50_intended_diff_only": 0.41320003271102906,
116
+ "tpp_threshold_50_unintended_diff_only": 0.20785002708435057,
117
+ "tpp_threshold_100_total_metric": 0.15640001595020295,
118
+ "tpp_threshold_100_intended_diff_only": 0.43960003852844237,
119
+ "tpp_threshold_100_unintended_diff_only": 0.28320002257823945,
120
+ "tpp_threshold_500_total_metric": 0.09075000882148743,
121
+ "tpp_threshold_500_intended_diff_only": 0.46720004081726074,
122
+ "tpp_threshold_500_unintended_diff_only": 0.3764500319957733
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.005649989843368531,
127
+ "tpp_threshold_2_intended_diff_only": 0.006599986553192138,
128
+ "tpp_threshold_2_unintended_diff_only": 0.0009499967098236084,
129
+ "tpp_threshold_5_total_metric": 0.018450015783309938,
130
+ "tpp_threshold_5_intended_diff_only": 0.0190000057220459,
131
+ "tpp_threshold_5_unintended_diff_only": 0.0005499899387359619,
132
+ "tpp_threshold_10_total_metric": 0.029649993777275084,
133
+ "tpp_threshold_10_intended_diff_only": 0.03219999074935913,
134
+ "tpp_threshold_10_unintended_diff_only": 0.0025499969720840453,
135
+ "tpp_threshold_20_total_metric": 0.04960000216960907,
136
+ "tpp_threshold_20_intended_diff_only": 0.0531999945640564,
137
+ "tpp_threshold_20_unintended_diff_only": 0.0035999923944473266,
138
+ "tpp_threshold_50_total_metric": 0.1142500102519989,
139
+ "tpp_threshold_50_intended_diff_only": 0.12120000123977662,
140
+ "tpp_threshold_50_unintended_diff_only": 0.00694999098777771,
141
+ "tpp_threshold_100_total_metric": 0.20945000052452087,
142
+ "tpp_threshold_100_intended_diff_only": 0.22319999933242798,
143
+ "tpp_threshold_100_unintended_diff_only": 0.013749998807907105,
144
+ "tpp_threshold_500_total_metric": 0.3759500414133072,
145
+ "tpp_threshold_500_intended_diff_only": 0.4088000297546387,
146
+ "tpp_threshold_500_unintended_diff_only": 0.03284998834133148
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.0020000189542770386,
194
  "tpp_threshold_10_total_metric": 0.011749953031539917,
195
  "tpp_threshold_10_intended_diff_only": 0.014999985694885254,
196
+ "tpp_threshold_10_unintended_diff_only": 0.003250032663345337,
197
+ "tpp_threshold_20_total_metric": 0.23874998092651367,
198
+ "tpp_threshold_20_intended_diff_only": 0.3190000057220459,
199
+ "tpp_threshold_20_unintended_diff_only": 0.08025002479553223,
200
+ "tpp_threshold_50_total_metric": 0.2254999428987503,
201
+ "tpp_threshold_50_intended_diff_only": 0.4089999794960022,
202
+ "tpp_threshold_50_unintended_diff_only": 0.1835000365972519,
203
+ "tpp_threshold_100_total_metric": 0.18924999237060547,
204
+ "tpp_threshold_100_intended_diff_only": 0.44200003147125244,
205
+ "tpp_threshold_100_unintended_diff_only": 0.252750039100647,
206
+ "tpp_threshold_500_total_metric": 0.09324999153614044,
207
+ "tpp_threshold_500_intended_diff_only": 0.45600003004074097,
208
+ "tpp_threshold_500_unintended_diff_only": 0.3627500385046005
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.004499971866607666,
212
+ "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
213
+ "tpp_threshold_2_unintended_diff_only": 0.000500023365020752,
214
+ "tpp_threshold_5_total_metric": 0.20025001466274261,
215
  "tpp_threshold_5_intended_diff_only": 0.25700002908706665,
216
+ "tpp_threshold_5_unintended_diff_only": 0.056750014424324036,
217
+ "tpp_threshold_10_total_metric": 0.20299997925758362,
218
  "tpp_threshold_10_intended_diff_only": 0.3140000104904175,
219
+ "tpp_threshold_10_unintended_diff_only": 0.11100003123283386,
220
+ "tpp_threshold_20_total_metric": 0.2227499932050705,
221
+ "tpp_threshold_20_intended_diff_only": 0.34200000762939453,
222
+ "tpp_threshold_20_unintended_diff_only": 0.11925001442432404,
223
+ "tpp_threshold_50_total_metric": 0.22325001657009125,
224
+ "tpp_threshold_50_intended_diff_only": 0.39000004529953003,
225
+ "tpp_threshold_50_unintended_diff_only": 0.16675002872943878,
226
+ "tpp_threshold_100_total_metric": 0.17649997770786285,
227
+ "tpp_threshold_100_intended_diff_only": 0.4259999990463257,
228
+ "tpp_threshold_100_unintended_diff_only": 0.24950002133846283,
229
+ "tpp_threshold_500_total_metric": 0.08125001192092896,
230
+ "tpp_threshold_500_intended_diff_only": 0.4590000510215759,
231
+ "tpp_threshold_500_unintended_diff_only": 0.377750039100647
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.14125004410743713,
235
+ "tpp_threshold_2_intended_diff_only": 0.17600005865097046,
236
+ "tpp_threshold_2_unintended_diff_only": 0.034750014543533325,
237
+ "tpp_threshold_5_total_metric": 0.20950005948543549,
238
  "tpp_threshold_5_intended_diff_only": 0.2820000648498535,
239
+ "tpp_threshold_5_unintended_diff_only": 0.07250000536441803,
240
+ "tpp_threshold_10_total_metric": 0.14225001633167267,
241
+ "tpp_threshold_10_intended_diff_only": 0.3410000205039978,
242
+ "tpp_threshold_10_unintended_diff_only": 0.19875000417232513,
243
+ "tpp_threshold_20_total_metric": 0.1052500307559967,
244
+ "tpp_threshold_20_intended_diff_only": 0.35700005292892456,
245
+ "tpp_threshold_20_unintended_diff_only": 0.25175002217292786,
246
+ "tpp_threshold_50_total_metric": 0.08875003457069397,
247
+ "tpp_threshold_50_intended_diff_only": 0.3760000467300415,
248
+ "tpp_threshold_50_unintended_diff_only": 0.28725001215934753,
249
+ "tpp_threshold_100_total_metric": 0.05075004696846008,
250
+ "tpp_threshold_100_intended_diff_only": 0.393000066280365,
251
+ "tpp_threshold_100_unintended_diff_only": 0.3422500193119049,
252
+ "tpp_threshold_500_total_metric": 0.0482499897480011,
253
+ "tpp_threshold_500_intended_diff_only": 0.4520000219345093,
254
+ "tpp_threshold_500_unintended_diff_only": 0.4037500321865082
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.009499981999397278,
258
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704,
260
+ "tpp_threshold_5_total_metric": 0.1757500022649765,
261
+ "tpp_threshold_5_intended_diff_only": 0.20800000429153442,
262
+ "tpp_threshold_5_unintended_diff_only": 0.03225000202655792,
263
+ "tpp_threshold_10_total_metric": 0.26874999701976776,
264
  "tpp_threshold_10_intended_diff_only": 0.3830000162124634,
265
+ "tpp_threshold_10_unintended_diff_only": 0.11425001919269562,
266
+ "tpp_threshold_20_total_metric": 0.280750036239624,
267
+ "tpp_threshold_20_intended_diff_only": 0.4540000557899475,
268
+ "tpp_threshold_20_unintended_diff_only": 0.1732500195503235,
269
+ "tpp_threshold_50_total_metric": 0.24549999833106995,
270
+ "tpp_threshold_50_intended_diff_only": 0.4790000319480896,
271
+ "tpp_threshold_50_unintended_diff_only": 0.23350003361701965,
272
+ "tpp_threshold_100_total_metric": 0.1745000183582306,
273
+ "tpp_threshold_100_intended_diff_only": 0.4830000400543213,
274
+ "tpp_threshold_100_unintended_diff_only": 0.3085000216960907,
275
+ "tpp_threshold_500_total_metric": 0.11750000715255737,
276
+ "tpp_threshold_500_intended_diff_only": 0.4930000305175781,
277
+ "tpp_threshold_500_unintended_diff_only": 0.37550002336502075
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.006250038743019104,
281
+ "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464,
283
+ "tpp_threshold_5_total_metric": 0.0297500342130661,
284
  "tpp_threshold_5_intended_diff_only": 0.05600005388259888,
285
+ "tpp_threshold_5_unintended_diff_only": 0.026250019669532776,
286
+ "tpp_threshold_10_total_metric": 0.1990000307559967,
287
+ "tpp_threshold_10_intended_diff_only": 0.2900000214576721,
288
+ "tpp_threshold_10_unintended_diff_only": 0.09099999070167542,
289
+ "tpp_threshold_20_total_metric": 0.22025002539157867,
290
+ "tpp_threshold_20_intended_diff_only": 0.3400000333786011,
291
+ "tpp_threshold_20_unintended_diff_only": 0.1197500079870224,
292
+ "tpp_threshold_50_total_metric": 0.24375003576278687,
293
+ "tpp_threshold_50_intended_diff_only": 0.41200006008148193,
294
+ "tpp_threshold_50_unintended_diff_only": 0.16825002431869507,
295
+ "tpp_threshold_100_total_metric": 0.1910000443458557,
296
+ "tpp_threshold_100_intended_diff_only": 0.4540000557899475,
297
+ "tpp_threshold_100_unintended_diff_only": 0.2630000114440918,
298
+ "tpp_threshold_500_total_metric": 0.11350004374980927,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.36250002682209015
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.005250006914138794,
306
+ "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
307
+ "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135,
308
+ "tpp_threshold_5_total_metric": 0.00700002908706665,
309
+ "tpp_threshold_5_intended_diff_only": 0.008000016212463379,
310
+ "tpp_threshold_5_unintended_diff_only": 0.0009999871253967285,
311
+ "tpp_threshold_10_total_metric": 0.003000006079673767,
312
+ "tpp_threshold_10_intended_diff_only": 0.004999995231628418,
313
+ "tpp_threshold_10_unintended_diff_only": 0.001999989151954651,
314
+ "tpp_threshold_20_total_metric": 0.016249999403953552,
315
+ "tpp_threshold_20_intended_diff_only": 0.019999980926513672,
316
+ "tpp_threshold_20_unintended_diff_only": 0.0037499815225601196,
317
+ "tpp_threshold_50_total_metric": 0.06299999356269836,
318
+ "tpp_threshold_50_intended_diff_only": 0.06199997663497925,
319
+ "tpp_threshold_50_unintended_diff_only": -0.0010000169277191162,
320
+ "tpp_threshold_100_total_metric": 0.143249973654747,
321
+ "tpp_threshold_100_intended_diff_only": 0.1499999761581421,
322
+ "tpp_threshold_100_unintended_diff_only": 0.006750002503395081,
323
+ "tpp_threshold_500_total_metric": 0.41075004637241364,
324
+ "tpp_threshold_500_intended_diff_only": 0.4240000247955322,
325
+ "tpp_threshold_500_unintended_diff_only": 0.013249978423118591
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.016500011086463928,
329
+ "tpp_threshold_2_intended_diff_only": 0.013999998569488525,
330
+ "tpp_threshold_2_unintended_diff_only": -0.002500012516975403,
331
+ "tpp_threshold_5_total_metric": 0.019000038504600525,
332
+ "tpp_threshold_5_intended_diff_only": 0.026000022888183594,
333
+ "tpp_threshold_5_unintended_diff_only": 0.006999984383583069,
334
+ "tpp_threshold_10_total_metric": 0.04975001513957977,
335
  "tpp_threshold_10_intended_diff_only": 0.054000020027160645,
336
+ "tpp_threshold_10_unintended_diff_only": 0.004250004887580872,
337
+ "tpp_threshold_20_total_metric": 0.07975003123283386,
338
+ "tpp_threshold_20_intended_diff_only": 0.08100003004074097,
339
+ "tpp_threshold_20_unintended_diff_only": 0.0012499988079071045,
340
+ "tpp_threshold_50_total_metric": 0.15800003707408905,
341
+ "tpp_threshold_50_intended_diff_only": 0.1690000295639038,
342
+ "tpp_threshold_50_unintended_diff_only": 0.010999992489814758,
343
+ "tpp_threshold_100_total_metric": 0.2757500857114792,
344
+ "tpp_threshold_100_intended_diff_only": 0.29600006341934204,
345
+ "tpp_threshold_100_unintended_diff_only": 0.020249977707862854,
346
+ "tpp_threshold_500_total_metric": 0.40750008821487427,
347
+ "tpp_threshold_500_intended_diff_only": 0.43900007009506226,
348
+ "tpp_threshold_500_unintended_diff_only": 0.03149998188018799
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.0007500052452087402,
352
+ "tpp_threshold_2_intended_diff_only": 0.0,
353
+ "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402,
354
+ "tpp_threshold_5_total_metric": 0.009749993681907654,
355
+ "tpp_threshold_5_intended_diff_only": 0.004999995231628418,
356
+ "tpp_threshold_5_unintended_diff_only": -0.004749998450279236,
357
+ "tpp_threshold_10_total_metric": 0.01600000262260437,
358
+ "tpp_threshold_10_intended_diff_only": 0.013999998569488525,
359
+ "tpp_threshold_10_unintended_diff_only": -0.0020000040531158447,
360
+ "tpp_threshold_20_total_metric": 0.007999986410140991,
361
+ "tpp_threshold_20_intended_diff_only": 0.010999977588653564,
362
+ "tpp_threshold_20_unintended_diff_only": 0.0029999911785125732,
363
+ "tpp_threshold_50_total_metric": 0.04050002992153168,
364
+ "tpp_threshold_50_intended_diff_only": 0.04000002145767212,
365
+ "tpp_threshold_50_unintended_diff_only": -0.0005000084638595581,
366
+ "tpp_threshold_100_total_metric": 0.08500000834465027,
367
+ "tpp_threshold_100_intended_diff_only": 0.09200000762939453,
368
+ "tpp_threshold_100_unintended_diff_only": 0.006999999284744263,
369
+ "tpp_threshold_500_total_metric": 0.35725001990795135,
370
+ "tpp_threshold_500_intended_diff_only": 0.3880000114440918,
371
+ "tpp_threshold_500_unintended_diff_only": 0.030749991536140442
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.0029999613761901855,
375
  "tpp_threshold_2_intended_diff_only": 0.0029999613761901855,
376
+ "tpp_threshold_2_unintended_diff_only": 0.0,
377
+ "tpp_threshold_5_total_metric": 0.02049998939037323,
378
+ "tpp_threshold_5_intended_diff_only": 0.019999980926513672,
379
+ "tpp_threshold_5_unintended_diff_only": -0.0005000084638595581,
380
+ "tpp_threshold_10_total_metric": 0.012249961495399475,
381
+ "tpp_threshold_10_intended_diff_only": 0.01799994707107544,
382
+ "tpp_threshold_10_unintended_diff_only": 0.005749985575675964,
383
+ "tpp_threshold_20_total_metric": 0.04599998891353607,
384
+ "tpp_threshold_20_intended_diff_only": 0.046999990940093994,
385
+ "tpp_threshold_20_unintended_diff_only": 0.0010000020265579224,
386
+ "tpp_threshold_50_total_metric": 0.13274995982646942,
387
+ "tpp_threshold_50_intended_diff_only": 0.15099996328353882,
388
+ "tpp_threshold_50_unintended_diff_only": 0.018250003457069397,
389
+ "tpp_threshold_100_total_metric": 0.26899996399879456,
390
+ "tpp_threshold_100_intended_diff_only": 0.28999996185302734,
391
+ "tpp_threshold_100_unintended_diff_only": 0.020999997854232788,
392
+ "tpp_threshold_500_total_metric": 0.37400001287460327,
393
+ "tpp_threshold_500_intended_diff_only": 0.42000001668930054,
394
+ "tpp_threshold_500_unintended_diff_only": 0.046000003814697266
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.0027499645948410034,
398
+ "tpp_threshold_2_intended_diff_only": 0.006999969482421875,
399
+ "tpp_threshold_2_unintended_diff_only": 0.004250004887580872,
400
+ "tpp_threshold_5_total_metric": 0.036000028252601624,
401
+ "tpp_threshold_5_intended_diff_only": 0.03600001335144043,
402
+ "tpp_threshold_5_unintended_diff_only": -1.4901161193847656e-08,
403
+ "tpp_threshold_10_total_metric": 0.06724998354911804,
404
+ "tpp_threshold_10_intended_diff_only": 0.06999999284744263,
405
+ "tpp_threshold_10_unintended_diff_only": 0.002750009298324585,
406
+ "tpp_threshold_20_total_metric": 0.09800000488758087,
407
+ "tpp_threshold_20_intended_diff_only": 0.10699999332427979,
408
+ "tpp_threshold_20_unintended_diff_only": 0.008999988436698914,
409
+ "tpp_threshold_50_total_metric": 0.177000030875206,
410
+ "tpp_threshold_50_intended_diff_only": 0.18400001525878906,
411
+ "tpp_threshold_50_unintended_diff_only": 0.006999984383583069,
412
+ "tpp_threshold_100_total_metric": 0.27424997091293335,
413
+ "tpp_threshold_100_intended_diff_only": 0.2879999876022339,
414
+ "tpp_threshold_100_unintended_diff_only": 0.013750016689300537,
415
+ "tpp_threshold_500_total_metric": 0.3302500396966934,
416
+ "tpp_threshold_500_intended_diff_only": 0.37300002574920654,
417
+ "tpp_threshold_500_unintended_diff_only": 0.04274998605251312
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-36621-tokens-150000000/scr/saebench_scr_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,8 +69,8 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "cca7884e-2c23-40d8-9267-172dcce0d980",
69
- "datetime_epoch_millis": 1776886349225,
70
  "eval_result_metrics": {
71
  "scr_metrics": {
72
  "scr_dir1_threshold_2": 0.21228453030290387,
@@ -78,18 +82,18 @@
78
  "scr_dir1_threshold_10": 0.31016100751738523,
79
  "scr_metric_threshold_10": 0.23549713630750585,
80
  "scr_dir2_threshold_10": 0.24712242598313489,
81
- "scr_dir1_threshold_20": null,
82
- "scr_metric_threshold_20": null,
83
- "scr_dir2_threshold_20": null,
84
- "scr_dir1_threshold_50": null,
85
- "scr_metric_threshold_50": null,
86
- "scr_dir2_threshold_50": null,
87
- "scr_dir1_threshold_100": null,
88
- "scr_metric_threshold_100": null,
89
- "scr_dir2_threshold_100": null,
90
- "scr_dir1_threshold_500": null,
91
- "scr_metric_threshold_500": null,
92
- "scr_dir2_threshold_500": null
93
  }
94
  },
95
  "eval_result_details": [
@@ -104,18 +108,18 @@
104
  "scr_dir1_threshold_10": 0.5256410746257707,
105
  "scr_metric_threshold_10": 0.090673634335731,
106
  "scr_dir2_threshold_10": 0.090673634335731,
107
- "scr_dir1_threshold_20": null,
108
- "scr_metric_threshold_20": null,
109
- "scr_dir2_threshold_20": null,
110
- "scr_dir1_threshold_50": null,
111
- "scr_metric_threshold_50": null,
112
- "scr_dir2_threshold_50": null,
113
- "scr_dir1_threshold_100": null,
114
- "scr_metric_threshold_100": null,
115
- "scr_dir2_threshold_100": null,
116
- "scr_dir1_threshold_500": null,
117
- "scr_metric_threshold_500": null,
118
- "scr_dir2_threshold_500": null
119
  },
120
  {
121
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
@@ -128,18 +132,18 @@
128
  "scr_dir1_threshold_10": 0.4393940557023345,
129
  "scr_metric_threshold_10": 0.3516819627607703,
130
  "scr_dir2_threshold_10": 0.3516819627607703,
131
- "scr_dir1_threshold_20": null,
132
- "scr_metric_threshold_20": null,
133
- "scr_dir2_threshold_20": null,
134
- "scr_dir1_threshold_50": null,
135
- "scr_metric_threshold_50": null,
136
- "scr_dir2_threshold_50": null,
137
- "scr_dir1_threshold_100": null,
138
- "scr_metric_threshold_100": null,
139
- "scr_dir2_threshold_100": null,
140
- "scr_dir1_threshold_500": null,
141
- "scr_metric_threshold_500": null,
142
- "scr_dir2_threshold_500": null
143
  },
144
  {
145
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
@@ -152,18 +156,18 @@
152
  "scr_dir1_threshold_10": 0.5312495925464404,
153
  "scr_metric_threshold_10": 0.13316589123232284,
154
  "scr_dir2_threshold_10": 0.13316589123232284,
155
- "scr_dir1_threshold_20": null,
156
- "scr_metric_threshold_20": null,
157
- "scr_dir2_threshold_20": null,
158
- "scr_dir1_threshold_50": null,
159
- "scr_metric_threshold_50": null,
160
- "scr_dir2_threshold_50": null,
161
- "scr_dir1_threshold_100": null,
162
- "scr_metric_threshold_100": null,
163
- "scr_dir2_threshold_100": null,
164
- "scr_dir1_threshold_500": null,
165
- "scr_metric_threshold_500": null,
166
- "scr_dir2_threshold_500": null
167
  },
168
  {
169
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
@@ -176,18 +180,18 @@
176
  "scr_dir1_threshold_10": 0.523489632191311,
177
  "scr_metric_threshold_10": 0.2388534572401228,
178
  "scr_dir2_threshold_10": 0.2388534572401228,
179
- "scr_dir1_threshold_20": null,
180
- "scr_metric_threshold_20": null,
181
- "scr_dir2_threshold_20": null,
182
- "scr_dir1_threshold_50": null,
183
- "scr_metric_threshold_50": null,
184
- "scr_dir2_threshold_50": null,
185
- "scr_dir1_threshold_100": null,
186
- "scr_metric_threshold_100": null,
187
- "scr_dir2_threshold_100": null,
188
- "scr_dir1_threshold_500": null,
189
- "scr_metric_threshold_500": null,
190
- "scr_dir2_threshold_500": null
191
  },
192
  {
193
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
@@ -200,18 +204,18 @@
200
  "scr_dir1_threshold_10": 0.08421069803000372,
201
  "scr_metric_threshold_10": 0.515872927644938,
202
  "scr_dir2_threshold_10": 0.515872927644938,
203
- "scr_dir1_threshold_20": null,
204
- "scr_metric_threshold_20": null,
205
- "scr_dir2_threshold_20": null,
206
- "scr_dir1_threshold_50": null,
207
- "scr_metric_threshold_50": null,
208
- "scr_dir2_threshold_50": null,
209
- "scr_dir1_threshold_100": null,
210
- "scr_metric_threshold_100": null,
211
- "scr_dir2_threshold_100": null,
212
- "scr_dir1_threshold_500": null,
213
- "scr_metric_threshold_500": null,
214
- "scr_dir2_threshold_500": null
215
  },
216
  {
217
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
@@ -224,18 +228,18 @@
224
  "scr_dir1_threshold_10": 0.1912566810955531,
225
  "scr_metric_threshold_10": 0.11450392446499717,
226
  "scr_dir2_threshold_10": 0.11450392446499717,
227
- "scr_dir1_threshold_20": null,
228
- "scr_metric_threshold_20": null,
229
- "scr_dir2_threshold_20": null,
230
- "scr_dir1_threshold_50": null,
231
- "scr_metric_threshold_50": null,
232
- "scr_dir2_threshold_50": null,
233
- "scr_dir1_threshold_100": null,
234
- "scr_metric_threshold_100": null,
235
- "scr_dir2_threshold_100": null,
236
- "scr_dir1_threshold_500": null,
237
- "scr_metric_threshold_500": null,
238
- "scr_dir2_threshold_500": null
239
  },
240
  {
241
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
@@ -248,18 +252,18 @@
248
  "scr_dir1_threshold_10": 0.11162783863611286,
249
  "scr_metric_threshold_10": 0.3648068054696092,
250
  "scr_dir2_threshold_10": 0.3648068054696092,
251
- "scr_dir1_threshold_20": null,
252
- "scr_metric_threshold_20": null,
253
- "scr_dir2_threshold_20": null,
254
- "scr_dir1_threshold_50": null,
255
- "scr_metric_threshold_50": null,
256
- "scr_dir2_threshold_50": null,
257
- "scr_dir1_threshold_100": null,
258
- "scr_metric_threshold_100": null,
259
- "scr_dir2_threshold_100": null,
260
- "scr_dir1_threshold_500": null,
261
- "scr_metric_threshold_500": null,
262
- "scr_dir2_threshold_500": null
263
  },
264
  {
265
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
@@ -272,18 +276,18 @@
272
  "scr_dir1_threshold_10": 0.07441848731155573,
273
  "scr_metric_threshold_10": 0.07441848731155573,
274
  "scr_dir2_threshold_10": 0.16742080471658802,
275
- "scr_dir1_threshold_20": null,
276
- "scr_metric_threshold_20": null,
277
- "scr_dir2_threshold_20": null,
278
- "scr_dir1_threshold_50": null,
279
- "scr_metric_threshold_50": null,
280
- "scr_dir2_threshold_50": null,
281
- "scr_dir1_threshold_100": null,
282
- "scr_metric_threshold_100": null,
283
- "scr_dir2_threshold_100": null,
284
- "scr_dir1_threshold_500": null,
285
- "scr_metric_threshold_500": null,
286
- "scr_dir2_threshold_500": null
287
  }
288
  ],
289
  "sae_bench_commit_hash": "Unknown",
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "24bb8c69-0395-42bf-9bbf-393aaebfb200",
73
+ "datetime_epoch_millis": 1777317528036,
74
  "eval_result_metrics": {
75
  "scr_metrics": {
76
  "scr_dir1_threshold_2": 0.21228453030290387,
 
82
  "scr_dir1_threshold_10": 0.31016100751738523,
83
  "scr_metric_threshold_10": 0.23549713630750585,
84
  "scr_dir2_threshold_10": 0.24712242598313489,
85
+ "scr_dir1_threshold_20": 0.31543387835359465,
86
+ "scr_metric_threshold_20": 0.3069632724374114,
87
+ "scr_dir2_threshold_20": 0.3149739227937308,
88
+ "scr_dir1_threshold_50": 0.1988876140030742,
89
+ "scr_metric_threshold_50": 0.37701390886542524,
90
+ "scr_dir2_threshold_50": 0.3821018004981899,
91
+ "scr_dir1_threshold_100": 0.25416206607371694,
92
+ "scr_metric_threshold_100": 0.43762714271230724,
93
+ "scr_dir2_threshold_100": 0.44252561875760116,
94
+ "scr_dir1_threshold_500": -0.2234237339772708,
95
+ "scr_metric_threshold_500": 0.22606711540544355,
96
+ "scr_dir2_threshold_500": 0.22959758188411156
97
  }
98
  },
99
  "eval_result_details": [
 
108
  "scr_dir1_threshold_10": 0.5256410746257707,
109
  "scr_metric_threshold_10": 0.090673634335731,
110
  "scr_dir2_threshold_10": 0.090673634335731,
111
+ "scr_dir1_threshold_20": 0.5641028776049328,
112
+ "scr_metric_threshold_20": 0.23316053135219456,
113
+ "scr_dir2_threshold_20": 0.23316053135219456,
114
+ "scr_dir1_threshold_50": 0.3589742805987668,
115
+ "scr_metric_threshold_50": 0.2927460227800901,
116
+ "scr_dir2_threshold_50": 0.2927460227800901,
117
+ "scr_dir1_threshold_100": 0.4487182328294704,
118
+ "scr_metric_threshold_100": 0.31606209135692587,
119
+ "scr_dir2_threshold_100": 0.31606209135692587,
120
+ "scr_dir1_threshold_500": -1.4743589253742293,
121
+ "scr_metric_threshold_500": 0.12953369715840288,
122
+ "scr_dir2_threshold_500": 0.12953369715840288
123
  },
124
  {
125
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
 
132
  "scr_dir1_threshold_10": 0.4393940557023345,
133
  "scr_metric_threshold_10": 0.3516819627607703,
134
  "scr_dir2_threshold_10": 0.3516819627607703,
135
+ "scr_dir1_threshold_20": 0.4469697140738727,
136
+ "scr_metric_threshold_20": 0.40061172224601954,
137
+ "scr_dir2_threshold_20": 0.40061172224601954,
138
+ "scr_dir1_threshold_50": -0.05303006015100743,
139
+ "scr_metric_threshold_50": 0.5168196276077031,
140
+ "scr_dir2_threshold_50": 0.5168196276077031,
141
+ "scr_dir1_threshold_100": 0.24242422874090183,
142
+ "scr_metric_threshold_100": 0.5810397090856065,
143
+ "scr_dir2_threshold_100": 0.5810397090856065,
144
+ "scr_dir1_threshold_500": -0.5151510909679566,
145
+ "scr_metric_threshold_500": 0.28440363460714707,
146
+ "scr_dir2_threshold_500": 0.28440363460714707
147
  },
148
  {
149
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
 
156
  "scr_dir1_threshold_10": 0.5312495925464404,
157
  "scr_metric_threshold_10": 0.13316589123232284,
158
  "scr_dir2_threshold_10": 0.13316589123232284,
159
+ "scr_dir1_threshold_20": 0.43749988358469727,
160
+ "scr_metric_threshold_20": 0.16080402160563206,
161
+ "scr_dir2_threshold_20": 0.16080402160563206,
162
+ "scr_dir1_threshold_50": 0.3593749708961743,
163
+ "scr_metric_threshold_50": 0.2386935023309456,
164
+ "scr_dir2_threshold_50": 0.2386935023309456,
165
+ "scr_dir1_threshold_100": 0.2656252619344312,
166
+ "scr_metric_threshold_100": 0.2688441627817573,
167
+ "scr_dir2_threshold_100": 0.2688441627817573,
168
+ "scr_dir1_threshold_500": -0.4843752037267798,
169
+ "scr_metric_threshold_500": -0.08793960103532353,
170
+ "scr_dir2_threshold_500": -0.08793960103532353
171
  },
172
  {
173
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
 
180
  "scr_dir1_threshold_10": 0.523489632191311,
181
  "scr_metric_threshold_10": 0.2388534572401228,
182
  "scr_dir2_threshold_10": 0.2388534572401228,
183
+ "scr_dir1_threshold_20": 0.42952990333245994,
184
+ "scr_metric_threshold_20": 0.3089171962431611,
185
+ "scr_dir2_threshold_20": 0.3089171962431611,
186
+ "scr_dir1_threshold_50": 0.41610725636599655,
187
+ "scr_metric_threshold_50": 0.3407642985653504,
188
+ "scr_dir2_threshold_50": 0.3407642985653504,
189
+ "scr_dir1_threshold_100": 0.4630873208110233,
190
+ "scr_metric_threshold_100": 0.5063695343586597,
191
+ "scr_dir2_threshold_100": 0.5063695343586597,
192
+ "scr_dir1_threshold_500": 0.013422646966463385,
193
+ "scr_metric_threshold_500": -0.3439489708328287,
194
+ "scr_dir2_threshold_500": -0.3439489708328287
195
  },
196
  {
197
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
 
204
  "scr_dir1_threshold_10": 0.08421069803000372,
205
  "scr_metric_threshold_10": 0.515872927644938,
206
  "scr_dir2_threshold_10": 0.515872927644938,
207
+ "scr_dir1_threshold_20": 0.06842111372221286,
208
+ "scr_metric_threshold_20": 0.6150794054390177,
209
+ "scr_dir2_threshold_20": 0.6150794054390177,
210
+ "scr_dir1_threshold_50": -0.18947375685884774,
211
+ "scr_metric_threshold_50": 0.6865079937623274,
212
+ "scr_dir2_threshold_50": 0.6865079937623274,
213
+ "scr_dir1_threshold_100": -0.1421053176441358,
214
+ "scr_metric_threshold_100": 0.6944443984532124,
215
+ "scr_dir2_threshold_100": 0.6944443984532124,
216
+ "scr_dir1_threshold_500": -0.26315796078077064,
217
+ "scr_metric_threshold_500": 0.595238157185469,
218
+ "scr_dir2_threshold_500": 0.595238157185469
219
  },
220
  {
221
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
 
228
  "scr_dir1_threshold_10": 0.1912566810955531,
229
  "scr_metric_threshold_10": 0.11450392446499717,
230
  "scr_dir2_threshold_10": 0.11450392446499717,
231
+ "scr_dir1_threshold_20": 0.251366184292382,
232
+ "scr_metric_threshold_20": 0.16412205665125212,
233
+ "scr_dir2_threshold_20": 0.16412205665125212,
234
+ "scr_dir1_threshold_50": 0.3224041847027747,
235
+ "scr_metric_threshold_50": 0.2709922648193172,
236
+ "scr_dir2_threshold_50": 0.2709922648193172,
237
+ "scr_dir1_threshold_100": 0.2950818016891243,
238
+ "scr_metric_threshold_100": 0.3702289841890732,
239
+ "scr_dir2_threshold_100": 0.3702289841890732,
240
+ "scr_dir1_threshold_500": 0.4153004823742847,
241
+ "scr_metric_threshold_500": 0.4618319872618971,
242
+ "scr_dir2_threshold_500": 0.4618319872618971
243
  },
244
  {
245
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
 
252
  "scr_dir1_threshold_10": 0.11162783863611286,
253
  "scr_metric_threshold_10": 0.3648068054696092,
254
  "scr_dir2_threshold_10": 0.3648068054696092,
255
+ "scr_dir1_threshold_20": 0.18604658254743003,
256
+ "scr_metric_threshold_20": 0.4334764782912447,
257
+ "scr_dir2_threshold_20": 0.4334764782912447,
258
+ "scr_dir1_threshold_50": 0.20930237071200095,
259
+ "scr_metric_threshold_50": 0.5021458952989327,
260
+ "scr_dir2_threshold_50": 0.5021458952989327,
261
+ "scr_dir1_threshold_100": 0.23720926106331344,
262
+ "scr_metric_threshold_100": 0.5407725223076157,
263
+ "scr_dir2_threshold_100": 0.5407725223076157,
264
+ "scr_dir1_threshold_500": 0.22790705668983022,
265
+ "scr_metric_threshold_500": 0.4763948958977931,
266
+ "scr_dir2_threshold_500": 0.4763948958977931
267
  },
268
  {
269
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
 
276
  "scr_dir1_threshold_10": 0.07441848731155573,
277
  "scr_metric_threshold_10": 0.07441848731155573,
278
  "scr_dir2_threshold_10": 0.16742080471658802,
279
+ "scr_dir1_threshold_20": 0.13953476767076942,
280
+ "scr_metric_threshold_20": 0.13953476767076942,
281
+ "scr_dir2_threshold_20": 0.20361997052132483,
282
+ "scr_dir1_threshold_50": 0.16744166575873534,
283
+ "scr_metric_threshold_50": 0.16744166575873534,
284
+ "scr_dir2_threshold_50": 0.208144798820853,
285
+ "scr_dir1_threshold_100": 0.22325573916560695,
286
+ "scr_metric_threshold_100": 0.22325573916560695,
287
+ "scr_dir2_threshold_100": 0.26244354752795823,
288
+ "scr_dir1_threshold_500": 0.29302312300099165,
289
+ "scr_metric_threshold_500": 0.29302312300099165,
290
+ "scr_dir2_threshold_500": 0.3212668548303358
291
  }
292
  ],
293
  "sae_bench_commit_hash": "Unknown",
gemma-2-2b-btk/k-100/seed-0/snapshots/step-36621-tokens-150000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "5bc96b3d-e5d5-4509-a36b-af52fc6011a3",
69
- "datetime_epoch_millis": 1776886504387,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.008874990046024323,
73
- "tpp_threshold_2_intended_diff_only": 0.008799993991851806,
74
- "tpp_threshold_2_unintended_diff_only": -7.499605417251589e-05,
75
- "tpp_threshold_5_total_metric": 0.0228750005364418,
76
- "tpp_threshold_5_intended_diff_only": 0.026200002431869505,
77
- "tpp_threshold_5_unintended_diff_only": 0.003325001895427704,
78
- "tpp_threshold_10_total_metric": 0.06649999469518661,
79
- "tpp_threshold_10_intended_diff_only": 0.07329999804496765,
80
- "tpp_threshold_10_unintended_diff_only": 0.006800003349781036,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.009499993920326234,
99
- "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
100
- "tpp_threshold_2_unintended_diff_only": -0.0004999905824661255,
101
- "tpp_threshold_5_total_metric": 0.031799998879432675,
102
- "tpp_threshold_5_intended_diff_only": 0.03560000658035278,
103
- "tpp_threshold_5_unintended_diff_only": 0.003800007700920105,
104
- "tpp_threshold_10_total_metric": 0.11019999086856842,
105
- "tpp_threshold_10_intended_diff_only": 0.12100000381469726,
106
- "tpp_threshold_10_unintended_diff_only": 0.010800012946128845,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.008249986171722411,
123
- "tpp_threshold_2_intended_diff_only": 0.008599984645843505,
124
- "tpp_threshold_2_unintended_diff_only": 0.00034999847412109375,
125
- "tpp_threshold_5_total_metric": 0.013950002193450928,
126
- "tpp_threshold_5_intended_diff_only": 0.01679999828338623,
127
- "tpp_threshold_5_unintended_diff_only": 0.0028499960899353027,
128
- "tpp_threshold_10_total_metric": 0.02279999852180481,
129
- "tpp_threshold_10_intended_diff_only": 0.025599992275238036,
130
- "tpp_threshold_10_unintended_diff_only": 0.0027999937534332275,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.0015000104904174805,
190
  "tpp_threshold_10_total_metric": 0.0342499315738678,
191
  "tpp_threshold_10_intended_diff_only": 0.039999961853027344,
192
- "tpp_threshold_10_unintended_diff_only": 0.005750030279159546
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": -0.0002500265836715698,
196
- "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
197
- "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983,
198
- "tpp_threshold_5_total_metric": 0.04749999940395355,
199
  "tpp_threshold_5_intended_diff_only": 0.05500000715255737,
200
- "tpp_threshold_5_unintended_diff_only": 0.007500007748603821,
201
- "tpp_threshold_10_total_metric": 0.13649998605251312,
202
- "tpp_threshold_10_intended_diff_only": 0.14800000190734863,
203
- "tpp_threshold_10_unintended_diff_only": 0.01150001585483551
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.021499991416931152,
207
- "tpp_threshold_2_intended_diff_only": 0.018000006675720215,
208
- "tpp_threshold_2_unintended_diff_only": -0.0034999847412109375,
209
- "tpp_threshold_5_total_metric": 0.07300002872943878,
210
- "tpp_threshold_5_intended_diff_only": 0.07600003480911255,
211
- "tpp_threshold_5_unintended_diff_only": 0.003000006079673767,
212
- "tpp_threshold_10_total_metric": 0.2199999988079071,
213
- "tpp_threshold_10_intended_diff_only": 0.23500001430511475,
214
- "tpp_threshold_10_unintended_diff_only": 0.015000015497207642
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.011750012636184692,
218
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
219
- "tpp_threshold_2_unintended_diff_only": -0.002750009298324585,
220
- "tpp_threshold_5_total_metric": 0.013000011444091797,
221
- "tpp_threshold_5_intended_diff_only": 0.018000006675720215,
222
- "tpp_threshold_5_unintended_diff_only": 0.004999995231628418,
223
- "tpp_threshold_10_total_metric": 0.07600003480911255,
224
- "tpp_threshold_10_intended_diff_only": 0.08500003814697266,
225
- "tpp_threshold_10_unintended_diff_only": 0.009000003337860107
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.0050000399351119995,
229
- "tpp_threshold_2_intended_diff_only": 0.006000041961669922,
230
- "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224,
231
- "tpp_threshold_5_total_metric": 0.011999979615211487,
232
- "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
233
- "tpp_threshold_5_unintended_diff_only": 0.0020000189542770386,
234
- "tpp_threshold_10_total_metric": 0.08425000309944153,
235
- "tpp_threshold_10_intended_diff_only": 0.09700000286102295,
236
- "tpp_threshold_10_unintended_diff_only": 0.012749999761581421
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.010749965906143188,
242
- "tpp_threshold_2_intended_diff_only": 0.006999969482421875,
243
- "tpp_threshold_2_unintended_diff_only": -0.0037499964237213135,
244
- "tpp_threshold_5_total_metric": 0.011749982833862305,
245
- "tpp_threshold_5_intended_diff_only": 0.009999990463256836,
246
- "tpp_threshold_5_unintended_diff_only": -0.0017499923706054688,
247
- "tpp_threshold_10_total_metric": 0.001749977469444275,
248
- "tpp_threshold_10_intended_diff_only": 0.006999969482421875,
249
- "tpp_threshold_10_unintended_diff_only": 0.0052499920129776
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": 0.0035000592470169067,
253
- "tpp_threshold_2_intended_diff_only": 0.010000050067901611,
254
- "tpp_threshold_2_unintended_diff_only": 0.006499990820884705,
255
- "tpp_threshold_5_total_metric": 0.01700003445148468,
256
- "tpp_threshold_5_intended_diff_only": 0.025000035762786865,
257
- "tpp_threshold_5_unintended_diff_only": 0.008000001311302185,
258
- "tpp_threshold_10_total_metric": 0.04400007426738739,
259
- "tpp_threshold_10_intended_diff_only": 0.04700005054473877,
260
- "tpp_threshold_10_unintended_diff_only": 0.0029999762773513794
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.012749999761581421,
264
- "tpp_threshold_2_intended_diff_only": 0.0059999823570251465,
265
- "tpp_threshold_2_unintended_diff_only": -0.006750017404556274,
266
- "tpp_threshold_5_total_metric": 0.016499996185302734,
267
- "tpp_threshold_5_intended_diff_only": 0.014999985694885254,
268
- "tpp_threshold_5_unintended_diff_only": -0.0015000104904174805,
269
- "tpp_threshold_10_total_metric": 0.01950003206729889,
270
- "tpp_threshold_10_intended_diff_only": 0.021000027656555176,
271
- "tpp_threshold_10_unintended_diff_only": 0.0014999955892562866
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.008250057697296143,
275
- "tpp_threshold_2_intended_diff_only": -0.001000046730041504,
276
- "tpp_threshold_2_unintended_diff_only": 0.007250010967254639,
277
- "tpp_threshold_5_total_metric": -0.001999989151954651,
278
- "tpp_threshold_5_intended_diff_only": 0.0,
279
- "tpp_threshold_5_unintended_diff_only": 0.001999989151954651,
280
- "tpp_threshold_10_total_metric": 0.005999952554702759,
281
- "tpp_threshold_10_intended_diff_only": 0.007999956607818604,
282
- "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.022499963641166687,
286
- "tpp_threshold_2_intended_diff_only": 0.0209999680519104,
287
- "tpp_threshold_2_unintended_diff_only": -0.0014999955892562866,
288
- "tpp_threshold_5_total_metric": 0.02649998664855957,
289
- "tpp_threshold_5_intended_diff_only": 0.0339999794960022,
290
- "tpp_threshold_5_unintended_diff_only": 0.007499992847442627,
291
- "tpp_threshold_10_total_metric": 0.042749956250190735,
292
- "tpp_threshold_10_intended_diff_only": 0.04499995708465576,
293
- "tpp_threshold_10_unintended_diff_only": 0.002250000834465027
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "3fb463c9-ba39-4caa-9e4c-89b22f6f4a41",
73
+ "datetime_epoch_millis": 1777317803319,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.007924996316432953,
77
+ "tpp_threshold_2_intended_diff_only": 0.009100002050399781,
78
+ "tpp_threshold_2_unintended_diff_only": 0.0011750057339668274,
79
+ "tpp_threshold_5_total_metric": 0.024074992537498473,
80
+ "tpp_threshold_5_intended_diff_only": 0.025599998235702512,
81
+ "tpp_threshold_5_unintended_diff_only": 0.0015250056982040404,
82
+ "tpp_threshold_10_total_metric": 0.06730000227689743,
83
+ "tpp_threshold_10_intended_diff_only": 0.0746000051498413,
84
+ "tpp_threshold_10_unintended_diff_only": 0.0073000028729438785,
85
+ "tpp_threshold_20_total_metric": 0.11055000871419907,
86
+ "tpp_threshold_20_intended_diff_only": 0.12020001411437987,
87
+ "tpp_threshold_20_unintended_diff_only": 0.009650005400180817,
88
+ "tpp_threshold_50_total_metric": 0.19522501081228255,
89
+ "tpp_threshold_50_intended_diff_only": 0.21430001854896544,
90
+ "tpp_threshold_50_unintended_diff_only": 0.019075007736682893,
91
+ "tpp_threshold_100_total_metric": 0.2717500224709511,
92
+ "tpp_threshold_100_intended_diff_only": 0.30330002307891846,
93
+ "tpp_threshold_100_unintended_diff_only": 0.03155000060796738,
94
+ "tpp_threshold_500_total_metric": 0.32325002253055574,
95
+ "tpp_threshold_500_intended_diff_only": 0.4257000327110291,
96
+ "tpp_threshold_500_unintended_diff_only": 0.10245001018047332
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.008499985933303833,
103
+ "tpp_threshold_2_intended_diff_only": 0.009399998188018798,
104
+ "tpp_threshold_2_unintended_diff_only": 0.0009000122547149659,
105
+ "tpp_threshold_5_total_metric": 0.03449999392032623,
106
+ "tpp_threshold_5_intended_diff_only": 0.03680000305175781,
107
+ "tpp_threshold_5_unintended_diff_only": 0.0023000091314315796,
108
+ "tpp_threshold_10_total_metric": 0.10905000567436218,
109
+ "tpp_threshold_10_intended_diff_only": 0.12080001831054688,
110
+ "tpp_threshold_10_unintended_diff_only": 0.011750012636184692,
111
+ "tpp_threshold_20_total_metric": 0.1711000084877014,
112
+ "tpp_threshold_20_intended_diff_only": 0.18520002365112304,
113
+ "tpp_threshold_20_unintended_diff_only": 0.01410001516342163,
114
+ "tpp_threshold_50_total_metric": 0.29200000166893003,
115
+ "tpp_threshold_50_intended_diff_only": 0.32460001707077024,
116
+ "tpp_threshold_50_unintended_diff_only": 0.03260001540184021,
117
+ "tpp_threshold_100_total_metric": 0.3614000231027603,
118
+ "tpp_threshold_100_intended_diff_only": 0.4128000378608704,
119
+ "tpp_threshold_100_unintended_diff_only": 0.05140001475811005,
120
+ "tpp_threshold_500_total_metric": 0.2880500257015228,
121
+ "tpp_threshold_500_intended_diff_only": 0.4648000478744507,
122
+ "tpp_threshold_500_unintended_diff_only": 0.17675002217292785
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.007350006699562072,
127
+ "tpp_threshold_2_intended_diff_only": 0.008800005912780762,
128
+ "tpp_threshold_2_unintended_diff_only": 0.001449999213218689,
129
+ "tpp_threshold_5_total_metric": 0.013649991154670716,
130
+ "tpp_threshold_5_intended_diff_only": 0.014399993419647216,
131
+ "tpp_threshold_5_unintended_diff_only": 0.0007500022649765015,
132
+ "tpp_threshold_10_total_metric": 0.025549998879432677,
133
+ "tpp_threshold_10_intended_diff_only": 0.02839999198913574,
134
+ "tpp_threshold_10_unintended_diff_only": 0.002849993109703064,
135
+ "tpp_threshold_20_total_metric": 0.050000008940696714,
136
+ "tpp_threshold_20_intended_diff_only": 0.05520000457763672,
137
+ "tpp_threshold_20_unintended_diff_only": 0.005199995636940002,
138
+ "tpp_threshold_50_total_metric": 0.09845001995563507,
139
+ "tpp_threshold_50_intended_diff_only": 0.10400002002716065,
140
+ "tpp_threshold_50_unintended_diff_only": 0.005550000071525574,
141
+ "tpp_threshold_100_total_metric": 0.18210002183914184,
142
+ "tpp_threshold_100_intended_diff_only": 0.19380000829696656,
143
+ "tpp_threshold_100_unintended_diff_only": 0.011699986457824708,
144
+ "tpp_threshold_500_total_metric": 0.35845001935958865,
145
+ "tpp_threshold_500_intended_diff_only": 0.3866000175476074,
146
+ "tpp_threshold_500_unintended_diff_only": 0.028149998188018797
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.0015000104904174805,
194
  "tpp_threshold_10_total_metric": 0.0342499315738678,
195
  "tpp_threshold_10_intended_diff_only": 0.039999961853027344,
196
+ "tpp_threshold_10_unintended_diff_only": 0.005750030279159546,
197
+ "tpp_threshold_20_total_metric": 0.13749995827674866,
198
+ "tpp_threshold_20_intended_diff_only": 0.1589999794960022,
199
+ "tpp_threshold_20_unintended_diff_only": 0.02150002121925354,
200
+ "tpp_threshold_50_total_metric": 0.24274995923042297,
201
+ "tpp_threshold_50_intended_diff_only": 0.265999972820282,
202
+ "tpp_threshold_50_unintended_diff_only": 0.02325001358985901,
203
+ "tpp_threshold_100_total_metric": 0.3435000032186508,
204
+ "tpp_threshold_100_intended_diff_only": 0.37300002574920654,
205
+ "tpp_threshold_100_unintended_diff_only": 0.029500022530555725,
206
+ "tpp_threshold_500_total_metric": 0.2985000014305115,
207
+ "tpp_threshold_500_intended_diff_only": 0.45600003004074097,
208
+ "tpp_threshold_500_unintended_diff_only": 0.1575000286102295
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.004499971866607666,
212
+ "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
213
+ "tpp_threshold_2_unintended_diff_only": 0.000500023365020752,
214
+ "tpp_threshold_5_total_metric": 0.050999999046325684,
215
  "tpp_threshold_5_intended_diff_only": 0.05500000715255737,
216
+ "tpp_threshold_5_unintended_diff_only": 0.0040000081062316895,
217
+ "tpp_threshold_10_total_metric": 0.13799996674060822,
218
+ "tpp_threshold_10_intended_diff_only": 0.14899998903274536,
219
+ "tpp_threshold_10_unintended_diff_only": 0.011000022292137146,
220
+ "tpp_threshold_20_total_metric": 0.15974996984004974,
221
+ "tpp_threshold_20_intended_diff_only": 0.171999990940094,
222
+ "tpp_threshold_20_unintended_diff_only": 0.01225002110004425,
223
+ "tpp_threshold_50_total_metric": 0.27275002002716064,
224
+ "tpp_threshold_50_intended_diff_only": 0.3030000329017639,
225
+ "tpp_threshold_50_unintended_diff_only": 0.03025001287460327,
226
+ "tpp_threshold_100_total_metric": 0.33875003457069397,
227
+ "tpp_threshold_100_intended_diff_only": 0.39900004863739014,
228
+ "tpp_threshold_100_unintended_diff_only": 0.06025001406669617,
229
+ "tpp_threshold_500_total_metric": 0.2370000183582306,
230
+ "tpp_threshold_500_intended_diff_only": 0.4590000510215759,
231
+ "tpp_threshold_500_unintended_diff_only": 0.22200003266334534
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.01075001060962677,
235
+ "tpp_threshold_2_intended_diff_only": 0.013000011444091797,
236
+ "tpp_threshold_2_unintended_diff_only": 0.002250000834465027,
237
+ "tpp_threshold_5_total_metric": 0.07375001907348633,
238
+ "tpp_threshold_5_intended_diff_only": 0.07700002193450928,
239
+ "tpp_threshold_5_unintended_diff_only": 0.0032500028610229492,
240
+ "tpp_threshold_10_total_metric": 0.2170000523328781,
241
+ "tpp_threshold_10_intended_diff_only": 0.23600006103515625,
242
+ "tpp_threshold_10_unintended_diff_only": 0.019000008702278137,
243
+ "tpp_threshold_20_total_metric": 0.2615000456571579,
244
+ "tpp_threshold_20_intended_diff_only": 0.2780000567436218,
245
+ "tpp_threshold_20_unintended_diff_only": 0.016500011086463928,
246
+ "tpp_threshold_50_total_metric": 0.30249999463558197,
247
+ "tpp_threshold_50_intended_diff_only": 0.3410000205039978,
248
+ "tpp_threshold_50_unintended_diff_only": 0.03850002586841583,
249
+ "tpp_threshold_100_total_metric": 0.3310000151395798,
250
+ "tpp_threshold_100_intended_diff_only": 0.38700002431869507,
251
+ "tpp_threshold_100_unintended_diff_only": 0.056000009179115295,
252
+ "tpp_threshold_500_total_metric": 0.22000005841255188,
253
+ "tpp_threshold_500_intended_diff_only": 0.4440000653266907,
254
+ "tpp_threshold_500_unintended_diff_only": 0.2240000069141388
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.009499981999397278,
258
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704,
260
+ "tpp_threshold_5_total_metric": 0.01924997568130493,
261
+ "tpp_threshold_5_intended_diff_only": 0.018999993801116943,
262
+ "tpp_threshold_5_unintended_diff_only": -0.0002499818801879883,
263
+ "tpp_threshold_10_total_metric": 0.07725001871585846,
264
+ "tpp_threshold_10_intended_diff_only": 0.08600002527236938,
265
+ "tpp_threshold_10_unintended_diff_only": 0.008750006556510925,
266
+ "tpp_threshold_20_total_metric": 0.1287500411272049,
267
+ "tpp_threshold_20_intended_diff_only": 0.1350000500679016,
268
+ "tpp_threshold_20_unintended_diff_only": 0.006250008940696716,
269
+ "tpp_threshold_50_total_metric": 0.3552500158548355,
270
+ "tpp_threshold_50_intended_diff_only": 0.4100000262260437,
271
+ "tpp_threshold_50_unintended_diff_only": 0.05475001037120819,
272
+ "tpp_threshold_100_total_metric": 0.38600002229213715,
273
+ "tpp_threshold_100_intended_diff_only": 0.46400004625320435,
274
+ "tpp_threshold_100_unintended_diff_only": 0.0780000239610672,
275
+ "tpp_threshold_500_total_metric": 0.3410000056028366,
276
+ "tpp_threshold_500_intended_diff_only": 0.48900002241134644,
277
+ "tpp_threshold_500_unintended_diff_only": 0.14800001680850983
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.008250012993812561,
281
+ "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464,
283
+ "tpp_threshold_5_total_metric": 0.015000000596046448,
284
+ "tpp_threshold_5_intended_diff_only": 0.018000006675720215,
285
+ "tpp_threshold_5_unintended_diff_only": 0.003000006079673767,
286
+ "tpp_threshold_10_total_metric": 0.07875005900859833,
287
+ "tpp_threshold_10_intended_diff_only": 0.09300005435943604,
288
+ "tpp_threshold_10_unintended_diff_only": 0.014249995350837708,
289
+ "tpp_threshold_20_total_metric": 0.1680000275373459,
290
+ "tpp_threshold_20_intended_diff_only": 0.1820000410079956,
291
+ "tpp_threshold_20_unintended_diff_only": 0.01400001347064972,
292
+ "tpp_threshold_50_total_metric": 0.28675001859664917,
293
+ "tpp_threshold_50_intended_diff_only": 0.3030000329017639,
294
+ "tpp_threshold_50_unintended_diff_only": 0.016250014305114746,
295
+ "tpp_threshold_100_total_metric": 0.40775004029273987,
296
+ "tpp_threshold_100_intended_diff_only": 0.4410000443458557,
297
+ "tpp_threshold_100_unintended_diff_only": 0.033250004053115845,
298
+ "tpp_threshold_500_total_metric": 0.3437500447034836,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.13225002586841583
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.008000031113624573,
306
+ "tpp_threshold_2_intended_diff_only": 0.012000024318695068,
307
+ "tpp_threshold_2_unintended_diff_only": 0.003999993205070496,
308
+ "tpp_threshold_5_total_metric": 0.006750002503395081,
309
+ "tpp_threshold_5_intended_diff_only": 0.008000016212463379,
310
+ "tpp_threshold_5_unintended_diff_only": 0.0012500137090682983,
311
+ "tpp_threshold_10_total_metric": 0.003999993205070496,
312
+ "tpp_threshold_10_intended_diff_only": 0.0059999823570251465,
313
+ "tpp_threshold_10_unintended_diff_only": 0.001999989151954651,
314
+ "tpp_threshold_20_total_metric": 0.025000005960464478,
315
+ "tpp_threshold_20_intended_diff_only": 0.027000010013580322,
316
+ "tpp_threshold_20_unintended_diff_only": 0.0020000040531158447,
317
+ "tpp_threshold_50_total_metric": 0.083250030875206,
318
+ "tpp_threshold_50_intended_diff_only": 0.08100003004074097,
319
+ "tpp_threshold_50_unintended_diff_only": -0.002250000834465027,
320
+ "tpp_threshold_100_total_metric": 0.1327500194311142,
321
+ "tpp_threshold_100_intended_diff_only": 0.1380000114440918,
322
+ "tpp_threshold_100_unintended_diff_only": 0.0052499920129776,
323
+ "tpp_threshold_500_total_metric": 0.38975001871585846,
324
+ "tpp_threshold_500_intended_diff_only": 0.3920000195503235,
325
+ "tpp_threshold_500_unintended_diff_only": 0.002250000834465027
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.010750040411949158,
329
+ "tpp_threshold_2_intended_diff_only": 0.01100003719329834,
330
+ "tpp_threshold_2_unintended_diff_only": 0.00024999678134918213,
331
+ "tpp_threshold_5_total_metric": 0.014250025153160095,
332
+ "tpp_threshold_5_intended_diff_only": 0.022000014781951904,
333
+ "tpp_threshold_5_unintended_diff_only": 0.007749989628791809,
334
+ "tpp_threshold_10_total_metric": 0.04425002634525299,
335
+ "tpp_threshold_10_intended_diff_only": 0.04900002479553223,
336
+ "tpp_threshold_10_unintended_diff_only": 0.004749998450279236,
337
+ "tpp_threshold_20_total_metric": 0.0777500569820404,
338
+ "tpp_threshold_20_intended_diff_only": 0.08400005102157593,
339
+ "tpp_threshold_20_unintended_diff_only": 0.0062499940395355225,
340
+ "tpp_threshold_50_total_metric": 0.12025007605552673,
341
+ "tpp_threshold_50_intended_diff_only": 0.1300000548362732,
342
+ "tpp_threshold_50_unintended_diff_only": 0.00974997878074646,
343
+ "tpp_threshold_100_total_metric": 0.27500008046627045,
344
+ "tpp_threshold_100_intended_diff_only": 0.30100005865097046,
345
+ "tpp_threshold_100_unintended_diff_only": 0.025999978184700012,
346
+ "tpp_threshold_500_total_metric": 0.4050000458955765,
347
+ "tpp_threshold_500_intended_diff_only": 0.437000036239624,
348
+ "tpp_threshold_500_unintended_diff_only": 0.031999990344047546
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.0029999762773513794,
352
+ "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
353
+ "tpp_threshold_2_unintended_diff_only": -0.0010000020265579224,
354
+ "tpp_threshold_5_total_metric": 0.011499971151351929,
355
+ "tpp_threshold_5_intended_diff_only": 0.0059999823570251465,
356
+ "tpp_threshold_5_unintended_diff_only": -0.005499988794326782,
357
+ "tpp_threshold_10_total_metric": 0.02000001072883606,
358
+ "tpp_threshold_10_intended_diff_only": 0.018000006675720215,
359
+ "tpp_threshold_10_unintended_diff_only": -0.0020000040531158447,
360
+ "tpp_threshold_20_total_metric": 0.008750036358833313,
361
+ "tpp_threshold_20_intended_diff_only": 0.012000024318695068,
362
+ "tpp_threshold_20_unintended_diff_only": 0.0032499879598617554,
363
+ "tpp_threshold_50_total_metric": 0.04725003242492676,
364
+ "tpp_threshold_50_intended_diff_only": 0.04400002956390381,
365
+ "tpp_threshold_50_unintended_diff_only": -0.0032500028610229492,
366
+ "tpp_threshold_100_total_metric": 0.10300001502037048,
367
+ "tpp_threshold_100_intended_diff_only": 0.10699999332427979,
368
+ "tpp_threshold_100_unintended_diff_only": 0.003999978303909302,
369
+ "tpp_threshold_500_total_metric": 0.33225004374980927,
370
+ "tpp_threshold_500_intended_diff_only": 0.3400000333786011,
371
+ "tpp_threshold_500_unintended_diff_only": 0.007749989628791809
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.0007499754428863525,
375
+ "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
376
+ "tpp_threshold_2_unintended_diff_only": 0.000250011682510376,
377
+ "tpp_threshold_5_total_metric": 0.006499975919723511,
378
+ "tpp_threshold_5_intended_diff_only": 0.0059999823570251465,
379
+ "tpp_threshold_5_unintended_diff_only": -0.0004999935626983643,
380
+ "tpp_threshold_10_total_metric": 0.001999974250793457,
381
+ "tpp_threshold_10_intended_diff_only": 0.006999969482421875,
382
+ "tpp_threshold_10_unintended_diff_only": 0.004999995231628418,
383
+ "tpp_threshold_20_total_metric": 0.034249961376190186,
384
+ "tpp_threshold_20_intended_diff_only": 0.034999966621398926,
385
+ "tpp_threshold_20_unintended_diff_only": 0.0007500052452087402,
386
+ "tpp_threshold_50_total_metric": 0.0627499669790268,
387
+ "tpp_threshold_50_intended_diff_only": 0.07099997997283936,
388
+ "tpp_threshold_50_unintended_diff_only": 0.008250012993812561,
389
+ "tpp_threshold_100_total_metric": 0.12849996984004974,
390
+ "tpp_threshold_100_intended_diff_only": 0.1369999647140503,
391
+ "tpp_threshold_100_unintended_diff_only": 0.00849999487400055,
392
+ "tpp_threshold_500_total_metric": 0.3200000077486038,
393
+ "tpp_threshold_500_intended_diff_only": 0.3920000195503235,
394
+ "tpp_threshold_500_unintended_diff_only": 0.07200001180171967
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.014250010251998901,
398
+ "tpp_threshold_2_intended_diff_only": 0.018000006675720215,
399
+ "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135,
400
+ "tpp_threshold_5_total_metric": 0.02924998104572296,
401
+ "tpp_threshold_5_intended_diff_only": 0.029999971389770508,
402
+ "tpp_threshold_5_unintended_diff_only": 0.0007499903440475464,
403
+ "tpp_threshold_10_total_metric": 0.05749998986721039,
404
+ "tpp_threshold_10_intended_diff_only": 0.06199997663497925,
405
+ "tpp_threshold_10_unintended_diff_only": 0.00449998676776886,
406
+ "tpp_threshold_20_total_metric": 0.1042499840259552,
407
+ "tpp_threshold_20_intended_diff_only": 0.11799997091293335,
408
+ "tpp_threshold_20_unintended_diff_only": 0.01374998688697815,
409
+ "tpp_threshold_50_total_metric": 0.17874999344348907,
410
+ "tpp_threshold_50_intended_diff_only": 0.1940000057220459,
411
+ "tpp_threshold_50_unintended_diff_only": 0.015250012278556824,
412
+ "tpp_threshold_100_total_metric": 0.27125002443790436,
413
+ "tpp_threshold_100_intended_diff_only": 0.28600001335144043,
414
+ "tpp_threshold_100_unintended_diff_only": 0.014749988913536072,
415
+ "tpp_threshold_500_total_metric": 0.3452499806880951,
416
+ "tpp_threshold_500_intended_diff_only": 0.37199997901916504,
417
+ "tpp_threshold_500_unintended_diff_only": 0.026749998331069946
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-366210-tokens-1500000000/scr/saebench_scr_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,8 +69,8 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "e81d4b4c-86dd-4b68-af3d-4dea2d44039b",
69
- "datetime_epoch_millis": 1776886866141,
70
  "eval_result_metrics": {
71
  "scr_metrics": {
72
  "scr_dir1_threshold_2": 0.3064341517762689,
@@ -78,18 +82,18 @@
78
  "scr_dir1_threshold_10": 0.2193571108810824,
79
  "scr_metric_threshold_10": 0.23299617539347295,
80
  "scr_dir2_threshold_10": 0.22370700659247295,
81
- "scr_dir1_threshold_20": null,
82
- "scr_metric_threshold_20": null,
83
- "scr_dir2_threshold_20": null,
84
- "scr_dir1_threshold_50": null,
85
- "scr_metric_threshold_50": null,
86
- "scr_dir2_threshold_50": null,
87
- "scr_dir1_threshold_100": null,
88
- "scr_metric_threshold_100": null,
89
- "scr_dir2_threshold_100": null,
90
- "scr_dir1_threshold_500": null,
91
- "scr_metric_threshold_500": null,
92
- "scr_dir2_threshold_500": null
93
  }
94
  },
95
  "eval_result_details": [
@@ -104,18 +108,18 @@
104
  "scr_dir1_threshold_10": 0.4487182328294704,
105
  "scr_metric_threshold_10": 0.046632137153671566,
106
  "scr_dir2_threshold_10": 0.046632137153671566,
107
- "scr_dir1_threshold_20": null,
108
- "scr_metric_threshold_20": null,
109
- "scr_dir2_threshold_20": null,
110
- "scr_dir1_threshold_50": null,
111
- "scr_metric_threshold_50": null,
112
- "scr_dir2_threshold_50": null,
113
- "scr_dir1_threshold_100": null,
114
- "scr_metric_threshold_100": null,
115
- "scr_dir2_threshold_100": null,
116
- "scr_dir1_threshold_500": null,
117
- "scr_metric_threshold_500": null,
118
- "scr_dir2_threshold_500": null
119
  },
120
  {
121
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
@@ -128,18 +132,18 @@
128
  "scr_dir1_threshold_10": 0.5075758841466581,
129
  "scr_metric_threshold_10": 0.3608563382335518,
130
  "scr_dir2_threshold_10": 0.3608563382335518,
131
- "scr_dir1_threshold_20": null,
132
- "scr_metric_threshold_20": null,
133
- "scr_dir2_threshold_20": null,
134
- "scr_dir1_threshold_50": null,
135
- "scr_metric_threshold_50": null,
136
- "scr_dir2_threshold_50": null,
137
- "scr_dir1_threshold_100": null,
138
- "scr_metric_threshold_100": null,
139
- "scr_dir2_threshold_100": null,
140
- "scr_dir1_threshold_500": null,
141
- "scr_metric_threshold_500": null,
142
- "scr_dir2_threshold_500": null
143
  },
144
  {
145
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
@@ -152,18 +156,18 @@
152
  "scr_dir1_threshold_10": 0.43749988358469727,
153
  "scr_metric_threshold_10": 0.13065336115482035,
154
  "scr_dir2_threshold_10": 0.13065336115482035,
155
- "scr_dir1_threshold_20": null,
156
- "scr_metric_threshold_20": null,
157
- "scr_dir2_threshold_20": null,
158
- "scr_dir1_threshold_50": null,
159
- "scr_metric_threshold_50": null,
160
- "scr_dir2_threshold_50": null,
161
- "scr_dir1_threshold_100": null,
162
- "scr_metric_threshold_100": null,
163
- "scr_dir2_threshold_100": null,
164
- "scr_dir1_threshold_500": null,
165
- "scr_metric_threshold_500": null,
166
- "scr_dir2_threshold_500": null
167
  },
168
  {
169
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
@@ -176,18 +180,18 @@
176
  "scr_dir1_threshold_10": -0.8120809423135005,
177
  "scr_metric_threshold_10": 0.11146504795136565,
178
  "scr_dir2_threshold_10": 0.11146504795136565,
179
- "scr_dir1_threshold_20": null,
180
- "scr_metric_threshold_20": null,
181
- "scr_dir2_threshold_20": null,
182
- "scr_dir1_threshold_50": null,
183
- "scr_metric_threshold_50": null,
184
- "scr_dir2_threshold_50": null,
185
- "scr_dir1_threshold_100": null,
186
- "scr_metric_threshold_100": null,
187
- "scr_dir2_threshold_100": null,
188
- "scr_dir1_threshold_500": null,
189
- "scr_metric_threshold_500": null,
190
- "scr_dir2_threshold_500": null
191
  },
192
  {
193
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
@@ -200,18 +204,18 @@
200
  "scr_dir1_threshold_10": 0.08947378822971379,
201
  "scr_metric_threshold_10": 0.45634918288429205,
202
  "scr_dir2_threshold_10": 0.45634918288429205,
203
- "scr_dir1_threshold_20": null,
204
- "scr_metric_threshold_20": null,
205
- "scr_dir2_threshold_20": null,
206
- "scr_dir1_threshold_50": null,
207
- "scr_metric_threshold_50": null,
208
- "scr_dir2_threshold_50": null,
209
- "scr_dir1_threshold_100": null,
210
- "scr_metric_threshold_100": null,
211
- "scr_dir2_threshold_100": null,
212
- "scr_dir1_threshold_500": null,
213
- "scr_metric_threshold_500": null,
214
- "scr_dir2_threshold_500": null
215
  },
216
  {
217
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
@@ -224,18 +228,18 @@
224
  "scr_dir1_threshold_10": 0.4371584539269045,
225
  "scr_metric_threshold_10": 0.12595415766246074,
226
  "scr_dir2_threshold_10": 0.12595415766246074,
227
- "scr_dir1_threshold_20": null,
228
- "scr_metric_threshold_20": null,
229
- "scr_dir2_threshold_20": null,
230
- "scr_dir1_threshold_50": null,
231
- "scr_metric_threshold_50": null,
232
- "scr_dir2_threshold_50": null,
233
- "scr_dir1_threshold_100": null,
234
- "scr_metric_threshold_100": null,
235
- "scr_dir2_threshold_100": null,
236
- "scr_dir1_threshold_500": null,
237
- "scr_metric_threshold_500": null,
238
- "scr_dir2_threshold_500": null
239
  },
240
  {
241
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
@@ -248,18 +252,18 @@
248
  "scr_dir1_threshold_10": 0.4093022598196558,
249
  "scr_metric_threshold_10": 0.3948498512825616,
250
  "scr_dir2_threshold_10": 0.3948498512825616,
251
- "scr_dir1_threshold_20": null,
252
- "scr_metric_threshold_20": null,
253
- "scr_dir2_threshold_20": null,
254
- "scr_dir1_threshold_50": null,
255
- "scr_metric_threshold_50": null,
256
- "scr_dir2_threshold_50": null,
257
- "scr_dir1_threshold_100": null,
258
- "scr_metric_threshold_100": null,
259
- "scr_dir2_threshold_100": null,
260
- "scr_dir1_threshold_500": null,
261
- "scr_metric_threshold_500": null,
262
- "scr_dir2_threshold_500": null
263
  },
264
  {
265
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
@@ -272,18 +276,18 @@
272
  "scr_dir1_threshold_10": 0.2372093268250598,
273
  "scr_metric_threshold_10": 0.2372093268250598,
274
  "scr_dir2_threshold_10": 0.16289597641705988,
275
- "scr_dir1_threshold_20": null,
276
- "scr_metric_threshold_20": null,
277
- "scr_dir2_threshold_20": null,
278
- "scr_dir1_threshold_50": null,
279
- "scr_metric_threshold_50": null,
280
- "scr_dir2_threshold_50": null,
281
- "scr_dir1_threshold_100": null,
282
- "scr_metric_threshold_100": null,
283
- "scr_dir2_threshold_100": null,
284
- "scr_dir1_threshold_500": null,
285
- "scr_metric_threshold_500": null,
286
- "scr_dir2_threshold_500": null
287
  }
288
  ],
289
  "sae_bench_commit_hash": "Unknown",
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "73319541-c05b-4ccc-8b94-f13b29a5f786",
73
+ "datetime_epoch_millis": 1777317574054,
74
  "eval_result_metrics": {
75
  "scr_metrics": {
76
  "scr_dir1_threshold_2": 0.3064341517762689,
 
82
  "scr_dir1_threshold_10": 0.2193571108810824,
83
  "scr_metric_threshold_10": 0.23299617539347295,
84
  "scr_dir2_threshold_10": 0.22370700659247295,
85
+ "scr_dir1_threshold_20": 0.2380992428185572,
86
+ "scr_metric_threshold_20": 0.2664283945461302,
87
+ "scr_dir2_threshold_20": 0.257562767827177,
88
+ "scr_dir1_threshold_50": 0.20460997694614932,
89
+ "scr_metric_threshold_50": 0.3379127570315681,
90
+ "scr_dir2_threshold_50": 0.32995735300586543,
91
+ "scr_dir1_threshold_100": 0.050597810757260865,
92
+ "scr_metric_threshold_100": 0.320738200291545,
93
+ "scr_dir2_threshold_100": 0.2985004778092627,
94
+ "scr_dir1_threshold_500": -1.1350601095370043,
95
+ "scr_metric_threshold_500": 0.1776629517746716,
96
+ "scr_dir2_threshold_500": 0.163122795574028
97
  }
98
  },
99
  "eval_result_details": [
 
108
  "scr_dir1_threshold_10": 0.4487182328294704,
109
  "scr_metric_threshold_10": 0.046632137153671566,
110
  "scr_dir2_threshold_10": 0.046632137153671566,
111
+ "scr_dir1_threshold_20": 0.14102610148224506,
112
+ "scr_metric_threshold_20": 0.13471497710162714,
113
+ "scr_dir2_threshold_20": 0.13471497710162714,
114
+ "scr_dir1_threshold_50": 0.4102564298503083,
115
+ "scr_metric_threshold_50": 0.25906739431680575,
116
+ "scr_dir2_threshold_50": 0.25906739431680575,
117
+ "scr_dir1_threshold_100": 0.5384614208981502,
118
+ "scr_metric_threshold_100": 0.3082901714420895,
119
+ "scr_dir2_threshold_100": 0.3082901714420895,
120
+ "scr_dir1_threshold_500": -2.5769224597152887,
121
+ "scr_metric_threshold_500": 0.08031092003311918,
122
+ "scr_dir2_threshold_500": 0.08031092003311918
123
  },
124
  {
125
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
 
132
  "scr_dir1_threshold_10": 0.5075758841466581,
133
  "scr_metric_threshold_10": 0.3608563382335518,
134
  "scr_dir2_threshold_10": 0.3608563382335518,
135
+ "scr_dir1_threshold_20": 0.5227272008897346,
136
+ "scr_metric_threshold_20": 0.2660550659387731,
137
+ "scr_dir2_threshold_20": 0.2660550659387731,
138
+ "scr_dir1_threshold_50": 0.38636354400108736,
139
+ "scr_metric_threshold_50": 0.49235474786507843,
140
+ "scr_dir2_threshold_50": 0.49235474786507843,
141
+ "scr_dir1_threshold_100": 0.5000002257751198,
142
+ "scr_metric_threshold_100": 0.5290520674790153,
143
+ "scr_dir2_threshold_100": 0.5290520674790153,
144
+ "scr_dir1_threshold_500": -1.8560602331895748,
145
+ "scr_metric_threshold_500": -0.321100954221084,
146
+ "scr_dir2_threshold_500": -0.321100954221084
147
  },
148
  {
149
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
 
156
  "scr_dir1_threshold_10": 0.43749988358469727,
157
  "scr_metric_threshold_10": 0.13065336115482035,
158
  "scr_dir2_threshold_10": 0.13065336115482035,
159
+ "scr_dir1_threshold_20": 0.09374970896174313,
160
+ "scr_metric_threshold_20": 0.17085429167603294,
161
+ "scr_dir2_threshold_20": 0.17085429167603294,
162
+ "scr_dir1_threshold_50": -0.7187499417923486,
163
+ "scr_metric_threshold_50": 0.16080402160563206,
164
+ "scr_dir2_threshold_50": 0.16080402160563206,
165
+ "scr_dir1_threshold_100": -0.921875087311477,
166
+ "scr_metric_threshold_100": 0.05025135035200431,
167
+ "scr_dir2_threshold_100": 0.05025135035200431,
168
+ "scr_dir1_threshold_500": -4.296874854480872,
169
+ "scr_metric_threshold_500": 0.0904522808732169,
170
+ "scr_dir2_threshold_500": 0.0904522808732169
171
  },
172
  {
173
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
 
180
  "scr_dir1_threshold_10": -0.8120809423135005,
181
  "scr_metric_threshold_10": 0.11146504795136565,
182
  "scr_dir2_threshold_10": 0.11146504795136565,
183
+ "scr_dir1_threshold_20": -0.1744968107512386,
184
+ "scr_metric_threshold_20": 0.022293085519754383,
185
+ "scr_dir2_threshold_20": 0.022293085519754383,
186
+ "scr_dir1_threshold_50": 0.32214752750714554,
187
+ "scr_metric_threshold_50": -0.12101906475380056,
188
+ "scr_dir2_threshold_50": -0.12101906475380056,
189
+ "scr_dir1_threshold_100": -1.0469800644450267,
190
+ "scr_metric_threshold_100": -0.2547770084012175,
191
+ "scr_dir2_threshold_100": -0.2547770084012175,
192
+ "scr_dir1_threshold_500": -1.6912755194905202,
193
+ "scr_metric_threshold_500": -0.5382164468571459,
194
+ "scr_dir2_threshold_500": -0.5382164468571459
195
  },
196
  {
197
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
 
204
  "scr_dir1_threshold_10": 0.08947378822971379,
205
  "scr_metric_threshold_10": 0.45634918288429205,
206
  "scr_dir2_threshold_10": 0.45634918288429205,
207
+ "scr_dir1_threshold_20": 0.11578955293692479,
208
+ "scr_metric_threshold_20": 0.5277777712076017,
209
+ "scr_dir2_threshold_20": 0.5277777712076017,
210
+ "scr_dir1_threshold_50": -0.11052614902855408,
211
+ "scr_metric_threshold_50": 0.6865079937623274,
212
+ "scr_dir2_threshold_50": 0.6865079937623274,
213
+ "scr_dir1_threshold_100": -0.09999996862913393,
214
+ "scr_metric_threshold_100": 0.7380950973057522,
215
+ "scr_dir2_threshold_100": 0.7380950973057522,
216
+ "scr_dir1_threshold_500": -0.13684191373576507,
217
+ "scr_metric_threshold_500": 0.801587280938177,
218
+ "scr_dir2_threshold_500": 0.801587280938177
219
  },
220
  {
221
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
 
228
  "scr_dir1_threshold_10": 0.4371584539269045,
229
  "scr_metric_threshold_10": 0.12595415766246074,
230
  "scr_dir2_threshold_10": 0.12595415766246074,
231
+ "scr_dir1_threshold_20": 0.4153004823742847,
232
+ "scr_metric_threshold_20": 0.19847321124088896,
233
+ "scr_dir2_threshold_20": 0.19847321124088896,
234
+ "scr_dir1_threshold_50": 0.4590164254795243,
235
+ "scr_metric_threshold_50": 0.297709930610645,
236
+ "scr_dir2_threshold_50": 0.297709930610645,
237
+ "scr_dir1_threshold_100": 0.4863388084931748,
238
+ "scr_metric_threshold_100": 0.36259549539076413,
239
+ "scr_dir2_threshold_100": 0.36259549539076413,
240
+ "scr_dir1_threshold_500": 0.5519127231510342,
241
+ "scr_metric_threshold_500": 0.35496177909383203,
242
+ "scr_dir2_threshold_500": 0.35496177909383203
243
  },
244
  {
245
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
 
252
  "scr_dir1_threshold_10": 0.4093022598196558,
253
  "scr_metric_threshold_10": 0.3948498512825616,
254
  "scr_dir2_threshold_10": 0.3948498512825616,
255
+ "scr_dir1_threshold_20": 0.5116278940822855,
256
+ "scr_metric_threshold_20": 0.5321889411118851,
257
+ "scr_dir2_threshold_20": 0.5321889411118851,
258
+ "scr_dir1_threshold_50": 0.5441858866203395,
259
+ "scr_metric_threshold_50": 0.5836909399141642,
260
+ "scr_dir2_threshold_50": 0.5836909399141642,
261
+ "scr_dir1_threshold_100": 0.5627905725981688,
262
+ "scr_metric_threshold_100": 0.4463518500848407,
263
+ "scr_dir2_threshold_100": 0.4463518500848407,
264
+ "scr_dir1_threshold_500": 0.4744185221266269,
265
+ "scr_metric_threshold_500": 0.5021458952989327,
266
+ "scr_dir2_threshold_500": 0.5021458952989327
267
  },
268
  {
269
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
 
276
  "scr_dir1_threshold_10": 0.2372093268250598,
277
  "scr_metric_threshold_10": 0.2372093268250598,
278
  "scr_dir2_threshold_10": 0.16289597641705988,
279
+ "scr_dir1_threshold_20": 0.27906981257247854,
280
+ "scr_metric_threshold_20": 0.27906981257247854,
281
+ "scr_dir2_threshold_20": 0.208144798820853,
282
+ "scr_dir1_threshold_50": 0.3441860929316922,
283
+ "scr_metric_threshold_50": 0.3441860929316922,
284
+ "scr_dir2_threshold_50": 0.2805428607260708,
285
+ "scr_dir1_threshold_100": 0.386046578679111,
286
+ "scr_metric_threshold_100": 0.386046578679111,
287
+ "scr_dir2_threshold_100": 0.208144798820853,
288
+ "scr_dir1_threshold_500": 0.4511628590383247,
289
+ "scr_metric_threshold_500": 0.4511628590383247,
290
+ "scr_dir2_threshold_500": 0.33484160943317604
291
  }
292
  ],
293
  "sae_bench_commit_hash": "Unknown",
gemma-2-2b-btk/k-100/seed-0/snapshots/step-366210-tokens-1500000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "3e660730-edb8-473e-9d5f-6f939a52b841",
69
- "datetime_epoch_millis": 1776887026749,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.02302500158548355,
73
- "tpp_threshold_2_intended_diff_only": 0.026000005006790162,
74
- "tpp_threshold_2_unintended_diff_only": 0.00297500342130661,
75
- "tpp_threshold_5_total_metric": 0.08602501600980758,
76
- "tpp_threshold_5_intended_diff_only": 0.11180001497268677,
77
- "tpp_threshold_5_unintended_diff_only": 0.02577499896287918,
78
- "tpp_threshold_10_total_metric": 0.10325000882148742,
79
- "tpp_threshold_10_intended_diff_only": 0.15720001459121705,
80
- "tpp_threshold_10_unintended_diff_only": 0.053950005769729616,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.037299996614456175,
99
- "tpp_threshold_2_intended_diff_only": 0.043000006675720216,
100
- "tpp_threshold_2_unintended_diff_only": 0.005700010061264038,
101
- "tpp_threshold_5_total_metric": 0.15745001435279846,
102
- "tpp_threshold_5_intended_diff_only": 0.20680001974105836,
103
- "tpp_threshold_5_unintended_diff_only": 0.049350005388259885,
104
- "tpp_threshold_10_total_metric": 0.1800000101327896,
105
  "tpp_threshold_10_intended_diff_only": 0.2850000262260437,
106
- "tpp_threshold_10_unintended_diff_only": 0.10500001609325409,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.008750006556510925,
123
- "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
124
- "tpp_threshold_2_unintended_diff_only": 0.00024999678134918213,
125
- "tpp_threshold_5_total_metric": 0.014600017666816711,
126
- "tpp_threshold_5_intended_diff_only": 0.016800010204315187,
127
- "tpp_threshold_5_unintended_diff_only": 0.002199992537498474,
128
- "tpp_threshold_10_total_metric": 0.02650000751018524,
129
- "tpp_threshold_10_intended_diff_only": 0.02940000295639038,
130
- "tpp_threshold_10_unintended_diff_only": 0.002899995446205139,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.0015000104904174805,
190
  "tpp_threshold_10_total_metric": 0.04349999129772186,
191
  "tpp_threshold_10_intended_diff_only": 0.050000011920928955,
192
- "tpp_threshold_10_unintended_diff_only": 0.006500020623207092
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": -2.9802322387695312e-08,
196
- "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
197
- "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162,
198
- "tpp_threshold_5_total_metric": 0.20250000059604645,
199
  "tpp_threshold_5_intended_diff_only": 0.2639999985694885,
200
- "tpp_threshold_5_unintended_diff_only": 0.06149999797344208,
201
- "tpp_threshold_10_total_metric": 0.2124999612569809,
202
  "tpp_threshold_10_intended_diff_only": 0.3199999928474426,
203
- "tpp_threshold_10_unintended_diff_only": 0.10750003159046173
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.1600000411272049,
207
- "tpp_threshold_2_intended_diff_only": 0.18700003623962402,
208
- "tpp_threshold_2_unintended_diff_only": 0.02699999511241913,
209
- "tpp_threshold_5_total_metric": 0.20900003612041473,
210
  "tpp_threshold_5_intended_diff_only": 0.28800004720687866,
211
- "tpp_threshold_5_unintended_diff_only": 0.07900001108646393,
212
- "tpp_threshold_10_total_metric": 0.14600004255771637,
213
- "tpp_threshold_10_intended_diff_only": 0.34800004959106445,
214
- "tpp_threshold_10_unintended_diff_only": 0.20200000703334808
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.012499988079071045,
218
  "tpp_threshold_2_intended_diff_only": 0.009999990463256836,
219
- "tpp_threshold_2_unintended_diff_only": -0.002499997615814209,
220
- "tpp_threshold_5_total_metric": 0.1927500218153,
221
- "tpp_threshold_5_intended_diff_only": 0.2200000286102295,
222
- "tpp_threshold_5_unintended_diff_only": 0.027250006794929504,
223
- "tpp_threshold_10_total_metric": 0.2982500344514847,
224
  "tpp_threshold_10_intended_diff_only": 0.4180000424385071,
225
- "tpp_threshold_10_unintended_diff_only": 0.1197500079870224
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.002749994397163391,
229
- "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
230
- "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983,
231
- "tpp_threshold_5_total_metric": 0.1665000170469284,
232
- "tpp_threshold_5_intended_diff_only": 0.24400001764297485,
233
- "tpp_threshold_5_unintended_diff_only": 0.07750000059604645,
234
- "tpp_threshold_10_total_metric": 0.19975002110004425,
235
- "tpp_threshold_10_intended_diff_only": 0.2890000343322754,
236
- "tpp_threshold_10_unintended_diff_only": 0.08925001323223114
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.009999975562095642,
242
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
243
- "tpp_threshold_2_unintended_diff_only": -0.004999980330467224,
244
- "tpp_threshold_5_total_metric": 0.012500017881393433,
245
- "tpp_threshold_5_intended_diff_only": 0.009000003337860107,
246
- "tpp_threshold_5_unintended_diff_only": -0.003500014543533325,
247
- "tpp_threshold_10_total_metric": 0.0007499903440475464,
248
- "tpp_threshold_10_intended_diff_only": 0.0059999823570251465,
249
- "tpp_threshold_10_unintended_diff_only": 0.0052499920129776
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": 0.006750032305717468,
253
- "tpp_threshold_2_intended_diff_only": 0.012000024318695068,
254
- "tpp_threshold_2_unintended_diff_only": 0.0052499920129776,
255
- "tpp_threshold_5_total_metric": 0.013500064611434937,
256
- "tpp_threshold_5_intended_diff_only": 0.020000040531158447,
257
- "tpp_threshold_5_unintended_diff_only": 0.006499975919723511,
258
- "tpp_threshold_10_total_metric": 0.03900006413459778,
259
  "tpp_threshold_10_intended_diff_only": 0.04200005531311035,
260
- "tpp_threshold_10_unintended_diff_only": 0.0029999911785125732
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.010750025510787964,
264
- "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
265
- "tpp_threshold_2_unintended_diff_only": -0.006750017404556274,
266
- "tpp_threshold_5_total_metric": 0.01875002682209015,
267
- "tpp_threshold_5_intended_diff_only": 0.017000019550323486,
268
- "tpp_threshold_5_unintended_diff_only": -0.0017500072717666626,
269
- "tpp_threshold_10_total_metric": 0.019250035285949707,
270
- "tpp_threshold_10_intended_diff_only": 0.021000027656555176,
271
- "tpp_threshold_10_unintended_diff_only": 0.0017499923706054688
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.0034999996423721313,
275
  "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
276
- "tpp_threshold_2_unintended_diff_only": 0.00849999487400055,
277
- "tpp_threshold_5_total_metric": 0.01124998927116394,
278
- "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
279
- "tpp_threshold_5_unintended_diff_only": 0.002750009298324585,
280
- "tpp_threshold_10_total_metric": 0.017999976873397827,
281
- "tpp_threshold_10_intended_diff_only": 0.019999980926513672,
282
- "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.019749999046325684,
286
- "tpp_threshold_2_intended_diff_only": 0.018999993801116943,
287
- "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402,
288
- "tpp_threshold_5_total_metric": 0.0169999897480011,
289
- "tpp_threshold_5_intended_diff_only": 0.02399998903274536,
290
- "tpp_threshold_5_unintended_diff_only": 0.006999999284744263,
291
- "tpp_threshold_10_total_metric": 0.05549997091293335,
292
- "tpp_threshold_10_intended_diff_only": 0.05799996852874756,
293
- "tpp_threshold_10_unintended_diff_only": 0.002499997615814209
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "62838e27-2c52-4c09-b695-fa8bebfe94e5",
73
+ "datetime_epoch_millis": 1777317847872,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.022224995493888854,
77
+ "tpp_threshold_2_intended_diff_only": 0.026500004529953006,
78
+ "tpp_threshold_2_unintended_diff_only": 0.004275009036064148,
79
+ "tpp_threshold_5_total_metric": 0.08510000854730607,
80
+ "tpp_threshold_5_intended_diff_only": 0.11040001511573792,
81
+ "tpp_threshold_5_unintended_diff_only": 0.025300006568431854,
82
+ "tpp_threshold_10_total_metric": 0.10447500497102738,
83
+ "tpp_threshold_10_intended_diff_only": 0.1586000144481659,
84
+ "tpp_threshold_10_unintended_diff_only": 0.05412500947713852,
85
+ "tpp_threshold_20_total_metric": 0.13792500346899034,
86
+ "tpp_threshold_20_intended_diff_only": 0.21370001435279845,
87
+ "tpp_threshold_20_unintended_diff_only": 0.07577501088380813,
88
+ "tpp_threshold_50_total_metric": 0.16550001502037048,
89
+ "tpp_threshold_50_intended_diff_only": 0.27290002107620237,
90
+ "tpp_threshold_50_unintended_diff_only": 0.1074000060558319,
91
+ "tpp_threshold_100_total_metric": 0.18742501735687256,
92
+ "tpp_threshold_100_intended_diff_only": 0.33280002474784853,
93
+ "tpp_threshold_100_unintended_diff_only": 0.14537500739097595,
94
+ "tpp_threshold_500_total_metric": 0.2368500128388405,
95
+ "tpp_threshold_500_intended_diff_only": 0.43860002756118777,
96
+ "tpp_threshold_500_unintended_diff_only": 0.20175001472234727
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.03739998936653137,
103
+ "tpp_threshold_2_intended_diff_only": 0.04480000734329224,
104
+ "tpp_threshold_2_unintended_diff_only": 0.007400017976760864,
105
+ "tpp_threshold_5_total_metric": 0.15745001137256623,
106
+ "tpp_threshold_5_intended_diff_only": 0.207200026512146,
107
+ "tpp_threshold_5_unintended_diff_only": 0.04975001513957977,
108
+ "tpp_threshold_10_total_metric": 0.17965000569820405,
109
  "tpp_threshold_10_intended_diff_only": 0.2850000262260437,
110
+ "tpp_threshold_10_unintended_diff_only": 0.10535002052783966,
111
+ "tpp_threshold_20_total_metric": 0.22324999868869783,
112
+ "tpp_threshold_20_intended_diff_only": 0.3706000208854675,
113
+ "tpp_threshold_20_unintended_diff_only": 0.1473500221967697,
114
+ "tpp_threshold_50_total_metric": 0.20945002734661103,
115
+ "tpp_threshold_50_intended_diff_only": 0.4186000466346741,
116
+ "tpp_threshold_50_unintended_diff_only": 0.20915001928806304,
117
+ "tpp_threshold_100_total_metric": 0.16580002307891845,
118
+ "tpp_threshold_100_intended_diff_only": 0.4430000424385071,
119
+ "tpp_threshold_100_unintended_diff_only": 0.2772000193595886,
120
+ "tpp_threshold_500_total_metric": 0.09770000874996185,
121
+ "tpp_threshold_500_intended_diff_only": 0.4674000382423401,
122
+ "tpp_threshold_500_unintended_diff_only": 0.36970002949237823
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.0070500016212463375,
127
+ "tpp_threshold_2_intended_diff_only": 0.00820000171661377,
128
+ "tpp_threshold_2_unintended_diff_only": 0.0011500000953674316,
129
+ "tpp_threshold_5_total_metric": 0.012750005722045899,
130
+ "tpp_threshold_5_intended_diff_only": 0.013600003719329835,
131
+ "tpp_threshold_5_unintended_diff_only": 0.0008499979972839356,
132
+ "tpp_threshold_10_total_metric": 0.029300004243850708,
133
+ "tpp_threshold_10_intended_diff_only": 0.03220000267028809,
134
+ "tpp_threshold_10_unintended_diff_only": 0.002899998426437378,
135
+ "tpp_threshold_20_total_metric": 0.052600008249282834,
136
+ "tpp_threshold_20_intended_diff_only": 0.056800007820129395,
137
+ "tpp_threshold_20_unintended_diff_only": 0.004199999570846558,
138
+ "tpp_threshold_50_total_metric": 0.12155000269412994,
139
+ "tpp_threshold_50_intended_diff_only": 0.1271999955177307,
140
+ "tpp_threshold_50_unintended_diff_only": 0.005649992823600769,
141
+ "tpp_threshold_100_total_metric": 0.20905001163482667,
142
+ "tpp_threshold_100_intended_diff_only": 0.22260000705718994,
143
+ "tpp_threshold_100_unintended_diff_only": 0.013549995422363282,
144
+ "tpp_threshold_500_total_metric": 0.3760000169277191,
145
+ "tpp_threshold_500_intended_diff_only": 0.4098000168800354,
146
+ "tpp_threshold_500_unintended_diff_only": 0.033799999952316286
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.0015000104904174805,
194
  "tpp_threshold_10_total_metric": 0.04349999129772186,
195
  "tpp_threshold_10_intended_diff_only": 0.050000011920928955,
196
+ "tpp_threshold_10_unintended_diff_only": 0.006500020623207092,
197
+ "tpp_threshold_20_total_metric": 0.2409999668598175,
198
+ "tpp_threshold_20_intended_diff_only": 0.35199999809265137,
199
+ "tpp_threshold_20_unintended_diff_only": 0.11100003123283386,
200
+ "tpp_threshold_50_total_metric": 0.226500004529953,
201
+ "tpp_threshold_50_intended_diff_only": 0.4100000262260437,
202
+ "tpp_threshold_50_unintended_diff_only": 0.1835000216960907,
203
+ "tpp_threshold_100_total_metric": 0.21800000965595245,
204
+ "tpp_threshold_100_intended_diff_only": 0.44200003147125244,
205
+ "tpp_threshold_100_unintended_diff_only": 0.2240000218153,
206
+ "tpp_threshold_500_total_metric": 0.11824999749660492,
207
+ "tpp_threshold_500_intended_diff_only": 0.45600003004074097,
208
+ "tpp_threshold_500_unintended_diff_only": 0.33775003254413605
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.004249975085258484,
212
+ "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
213
+ "tpp_threshold_2_unintended_diff_only": 0.0007500201463699341,
214
+ "tpp_threshold_5_total_metric": 0.2032499611377716,
215
  "tpp_threshold_5_intended_diff_only": 0.2639999985694885,
216
+ "tpp_threshold_5_unintended_diff_only": 0.06075003743171692,
217
+ "tpp_threshold_10_total_metric": 0.2072499841451645,
218
  "tpp_threshold_10_intended_diff_only": 0.3199999928474426,
219
+ "tpp_threshold_10_unintended_diff_only": 0.11275000870227814,
220
+ "tpp_threshold_20_total_metric": 0.22249998152256012,
221
+ "tpp_threshold_20_intended_diff_only": 0.34299999475479126,
222
+ "tpp_threshold_20_unintended_diff_only": 0.12050001323223114,
223
+ "tpp_threshold_50_total_metric": 0.22625002264976501,
224
+ "tpp_threshold_50_intended_diff_only": 0.39900004863739014,
225
+ "tpp_threshold_50_unintended_diff_only": 0.17275002598762512,
226
+ "tpp_threshold_100_total_metric": 0.18025001883506775,
227
+ "tpp_threshold_100_intended_diff_only": 0.4320000410079956,
228
+ "tpp_threshold_100_unintended_diff_only": 0.25175002217292786,
229
+ "tpp_threshold_500_total_metric": 0.0832500159740448,
230
+ "tpp_threshold_500_intended_diff_only": 0.4590000510215759,
231
+ "tpp_threshold_500_unintended_diff_only": 0.37575003504753113
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.1549999862909317,
235
+ "tpp_threshold_2_intended_diff_only": 0.18900001049041748,
236
+ "tpp_threshold_2_unintended_diff_only": 0.03400002419948578,
237
+ "tpp_threshold_5_total_metric": 0.21400004625320435,
238
  "tpp_threshold_5_intended_diff_only": 0.28800004720687866,
239
+ "tpp_threshold_5_unintended_diff_only": 0.07400000095367432,
240
+ "tpp_threshold_10_total_metric": 0.1497500091791153,
241
+ "tpp_threshold_10_intended_diff_only": 0.3490000367164612,
242
+ "tpp_threshold_10_unintended_diff_only": 0.1992500275373459,
243
+ "tpp_threshold_20_total_metric": 0.11675000190734863,
244
+ "tpp_threshold_20_intended_diff_only": 0.3630000352859497,
245
+ "tpp_threshold_20_unintended_diff_only": 0.24625003337860107,
246
+ "tpp_threshold_50_total_metric": 0.09075003862380981,
247
+ "tpp_threshold_50_intended_diff_only": 0.3880000710487366,
248
+ "tpp_threshold_50_unintended_diff_only": 0.29725003242492676,
249
+ "tpp_threshold_100_total_metric": 0.053750038146972656,
250
+ "tpp_threshold_100_intended_diff_only": 0.4020000696182251,
251
+ "tpp_threshold_100_unintended_diff_only": 0.34825003147125244,
252
+ "tpp_threshold_500_total_metric": 0.04649999737739563,
253
+ "tpp_threshold_500_intended_diff_only": 0.4520000219345093,
254
+ "tpp_threshold_500_unintended_diff_only": 0.40550002455711365
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.010499969124794006,
258
  "tpp_threshold_2_intended_diff_only": 0.009999990463256836,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704,
260
+ "tpp_threshold_5_total_metric": 0.19349999725818634,
261
+ "tpp_threshold_5_intended_diff_only": 0.22100001573562622,
262
+ "tpp_threshold_5_unintended_diff_only": 0.02750001847743988,
263
+ "tpp_threshold_10_total_metric": 0.30150002241134644,
264
  "tpp_threshold_10_intended_diff_only": 0.4180000424385071,
265
+ "tpp_threshold_10_unintended_diff_only": 0.11650002002716064,
266
+ "tpp_threshold_20_total_metric": 0.30000001192092896,
267
+ "tpp_threshold_20_intended_diff_only": 0.4610000252723694,
268
+ "tpp_threshold_20_unintended_diff_only": 0.16100001335144043,
269
+ "tpp_threshold_50_total_metric": 0.24650003015995026,
270
+ "tpp_threshold_50_intended_diff_only": 0.4830000400543213,
271
+ "tpp_threshold_50_unintended_diff_only": 0.23650000989437103,
272
+ "tpp_threshold_100_total_metric": 0.17675000429153442,
273
+ "tpp_threshold_100_intended_diff_only": 0.484000027179718,
274
+ "tpp_threshold_100_unintended_diff_only": 0.3072500228881836,
275
+ "tpp_threshold_500_total_metric": 0.12324999272823334,
276
+ "tpp_threshold_500_intended_diff_only": 0.49400001764297485,
277
+ "tpp_threshold_500_unintended_diff_only": 0.3707500249147415
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.006000027060508728,
281
+ "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224,
283
+ "tpp_threshold_5_total_metric": 0.1600000560283661,
284
+ "tpp_threshold_5_intended_diff_only": 0.24500006437301636,
285
+ "tpp_threshold_5_unintended_diff_only": 0.08500000834465027,
286
+ "tpp_threshold_10_total_metric": 0.19625002145767212,
287
+ "tpp_threshold_10_intended_diff_only": 0.28800004720687866,
288
+ "tpp_threshold_10_unintended_diff_only": 0.09175002574920654,
289
+ "tpp_threshold_20_total_metric": 0.23600003123283386,
290
+ "tpp_threshold_20_intended_diff_only": 0.3340000510215759,
291
+ "tpp_threshold_20_unintended_diff_only": 0.09800001978874207,
292
+ "tpp_threshold_50_total_metric": 0.257250040769577,
293
+ "tpp_threshold_50_intended_diff_only": 0.41300004720687866,
294
+ "tpp_threshold_50_unintended_diff_only": 0.15575000643730164,
295
+ "tpp_threshold_100_total_metric": 0.200250044465065,
296
+ "tpp_threshold_100_intended_diff_only": 0.45500004291534424,
297
+ "tpp_threshold_100_unintended_diff_only": 0.25474999845027924,
298
+ "tpp_threshold_500_total_metric": 0.11725004017353058,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.35875003039836884
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.00599999725818634,
306
+ "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
307
+ "tpp_threshold_2_unintended_diff_only": 0.003000006079673767,
308
+ "tpp_threshold_5_total_metric": 0.005499988794326782,
309
+ "tpp_threshold_5_intended_diff_only": 0.0059999823570251465,
310
+ "tpp_threshold_5_unintended_diff_only": 0.0004999935626983643,
311
+ "tpp_threshold_10_total_metric": 0.004499971866607666,
312
+ "tpp_threshold_10_intended_diff_only": 0.006999969482421875,
313
+ "tpp_threshold_10_unintended_diff_only": 0.002499997615814209,
314
+ "tpp_threshold_20_total_metric": 0.025499999523162842,
315
+ "tpp_threshold_20_intended_diff_only": 0.02799999713897705,
316
+ "tpp_threshold_20_unintended_diff_only": 0.002499997615814209,
317
+ "tpp_threshold_50_total_metric": 0.07224999368190765,
318
+ "tpp_threshold_50_intended_diff_only": 0.07099997997283936,
319
+ "tpp_threshold_50_unintended_diff_only": -0.0012500137090682983,
320
+ "tpp_threshold_100_total_metric": 0.14424999058246613,
321
+ "tpp_threshold_100_intended_diff_only": 0.15399998426437378,
322
+ "tpp_threshold_100_unintended_diff_only": 0.009749993681907654,
323
+ "tpp_threshold_500_total_metric": 0.41724999248981476,
324
+ "tpp_threshold_500_intended_diff_only": 0.4350000023841858,
325
+ "tpp_threshold_500_unintended_diff_only": 0.017750009894371033
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.015750020742416382,
329
+ "tpp_threshold_2_intended_diff_only": 0.013999998569488525,
330
+ "tpp_threshold_2_unintended_diff_only": -0.0017500221729278564,
331
+ "tpp_threshold_5_total_metric": 0.008250042796134949,
332
+ "tpp_threshold_5_intended_diff_only": 0.016000032424926758,
333
+ "tpp_threshold_5_unintended_diff_only": 0.007749989628791809,
334
+ "tpp_threshold_10_total_metric": 0.037250056862831116,
335
  "tpp_threshold_10_intended_diff_only": 0.04200005531311035,
336
+ "tpp_threshold_10_unintended_diff_only": 0.004749998450279236,
337
+ "tpp_threshold_20_total_metric": 0.08600001037120819,
338
+ "tpp_threshold_20_intended_diff_only": 0.09200000762939453,
339
+ "tpp_threshold_20_unintended_diff_only": 0.00599999725818634,
340
+ "tpp_threshold_50_total_metric": 0.1597500443458557,
341
+ "tpp_threshold_50_intended_diff_only": 0.1730000376701355,
342
+ "tpp_threshold_50_unintended_diff_only": 0.013249993324279785,
343
+ "tpp_threshold_100_total_metric": 0.2812500447034836,
344
+ "tpp_threshold_100_intended_diff_only": 0.3020000457763672,
345
+ "tpp_threshold_100_unintended_diff_only": 0.020750001072883606,
346
+ "tpp_threshold_500_total_metric": 0.4060000777244568,
347
+ "tpp_threshold_500_intended_diff_only": 0.43900007009506226,
348
+ "tpp_threshold_500_unintended_diff_only": 0.03299999237060547
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.0005000084638595581,
352
+ "tpp_threshold_2_intended_diff_only": 0.0,
353
+ "tpp_threshold_2_unintended_diff_only": -0.0005000084638595581,
354
+ "tpp_threshold_5_total_metric": 0.011250019073486328,
355
+ "tpp_threshold_5_intended_diff_only": 0.00700002908706665,
356
+ "tpp_threshold_5_unintended_diff_only": -0.004249989986419678,
357
+ "tpp_threshold_10_total_metric": 0.02000001072883606,
358
+ "tpp_threshold_10_intended_diff_only": 0.018000006675720215,
359
+ "tpp_threshold_10_unintended_diff_only": -0.0020000040531158447,
360
+ "tpp_threshold_20_total_metric": 0.010000035166740417,
361
+ "tpp_threshold_20_intended_diff_only": 0.012000024318695068,
362
+ "tpp_threshold_20_unintended_diff_only": 0.001999989151954651,
363
+ "tpp_threshold_50_total_metric": 0.03924998641014099,
364
+ "tpp_threshold_50_intended_diff_only": 0.038999974727630615,
365
+ "tpp_threshold_50_unintended_diff_only": -0.000250011682510376,
366
+ "tpp_threshold_100_total_metric": 0.09149999916553497,
367
+ "tpp_threshold_100_intended_diff_only": 0.09799998998641968,
368
+ "tpp_threshold_100_unintended_diff_only": 0.006499990820884705,
369
+ "tpp_threshold_500_total_metric": 0.3537500202655792,
370
+ "tpp_threshold_500_intended_diff_only": 0.3840000033378601,
371
+ "tpp_threshold_500_unintended_diff_only": 0.030249983072280884
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.004749983549118042,
375
  "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
376
+ "tpp_threshold_2_unintended_diff_only": 0.000250011682510376,
377
+ "tpp_threshold_5_total_metric": 0.020749971270561218,
378
+ "tpp_threshold_5_intended_diff_only": 0.0209999680519104,
379
+ "tpp_threshold_5_unintended_diff_only": 0.00024999678134918213,
380
+ "tpp_threshold_10_total_metric": 0.01675000786781311,
381
+ "tpp_threshold_10_intended_diff_only": 0.023000001907348633,
382
+ "tpp_threshold_10_unintended_diff_only": 0.0062499940395355225,
383
+ "tpp_threshold_20_total_metric": 0.040499985218048096,
384
+ "tpp_threshold_20_intended_diff_only": 0.041999995708465576,
385
+ "tpp_threshold_20_unintended_diff_only": 0.0015000104904174805,
386
+ "tpp_threshold_50_total_metric": 0.1472499966621399,
387
+ "tpp_threshold_50_intended_diff_only": 0.15700000524520874,
388
+ "tpp_threshold_50_unintended_diff_only": 0.009750008583068848,
389
+ "tpp_threshold_100_total_metric": 0.25699999928474426,
390
+ "tpp_threshold_100_intended_diff_only": 0.27300000190734863,
391
+ "tpp_threshold_100_unintended_diff_only": 0.01600000262260437,
392
+ "tpp_threshold_500_total_metric": 0.3722499758005142,
393
+ "tpp_threshold_500_intended_diff_only": 0.4179999828338623,
394
+ "tpp_threshold_500_unintended_diff_only": 0.045750007033348083
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.008249998092651367,
398
+ "tpp_threshold_2_intended_diff_only": 0.013000011444091797,
399
+ "tpp_threshold_2_unintended_diff_only": 0.00475001335144043,
400
+ "tpp_threshold_5_total_metric": 0.018000006675720215,
401
+ "tpp_threshold_5_intended_diff_only": 0.018000006675720215,
402
+ "tpp_threshold_5_unintended_diff_only": 0.0,
403
+ "tpp_threshold_10_total_metric": 0.06799997389316559,
404
+ "tpp_threshold_10_intended_diff_only": 0.07099997997283936,
405
+ "tpp_threshold_10_unintended_diff_only": 0.003000006079673767,
406
+ "tpp_threshold_20_total_metric": 0.10100001096725464,
407
+ "tpp_threshold_20_intended_diff_only": 0.11000001430511475,
408
+ "tpp_threshold_20_unintended_diff_only": 0.009000003337860107,
409
+ "tpp_threshold_50_total_metric": 0.18924999237060547,
410
+ "tpp_threshold_50_intended_diff_only": 0.19599997997283936,
411
+ "tpp_threshold_50_unintended_diff_only": 0.006749987602233887,
412
+ "tpp_threshold_100_total_metric": 0.27125002443790436,
413
+ "tpp_threshold_100_intended_diff_only": 0.28600001335144043,
414
+ "tpp_threshold_100_unintended_diff_only": 0.014749988913536072,
415
+ "tpp_threshold_500_total_metric": 0.3307500183582306,
416
+ "tpp_threshold_500_intended_diff_only": 0.37300002574920654,
417
+ "tpp_threshold_500_unintended_diff_only": 0.04225000739097595
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-48828-tokens-200000000/scr/saebench_scr_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,8 +69,8 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "5cc6d973-70a7-4cd6-b5f7-bf900f7b29c1",
69
- "datetime_epoch_millis": 1776886607378,
70
  "eval_result_metrics": {
71
  "scr_metrics": {
72
  "scr_dir1_threshold_2": 0.21586031401333375,
@@ -78,18 +82,18 @@
78
  "scr_dir1_threshold_10": 0.1613699480060317,
79
  "scr_metric_threshold_10": 0.25517605668515764,
80
  "scr_dir2_threshold_10": 0.2593694763087202,
81
- "scr_dir1_threshold_20": null,
82
- "scr_metric_threshold_20": null,
83
- "scr_dir2_threshold_20": null,
84
- "scr_dir1_threshold_50": null,
85
- "scr_metric_threshold_50": null,
86
- "scr_dir2_threshold_50": null,
87
- "scr_dir1_threshold_100": null,
88
- "scr_metric_threshold_100": null,
89
- "scr_dir2_threshold_100": null,
90
- "scr_dir1_threshold_500": null,
91
- "scr_metric_threshold_500": null,
92
- "scr_dir2_threshold_500": null
93
  }
94
  },
95
  "eval_result_details": [
@@ -104,18 +108,18 @@
104
  "scr_dir1_threshold_10": 0.5384614208981502,
105
  "scr_metric_threshold_10": 0.12435226279901532,
106
  "scr_dir2_threshold_10": 0.12435226279901532,
107
- "scr_dir1_threshold_20": null,
108
- "scr_metric_threshold_20": null,
109
- "scr_dir2_threshold_20": null,
110
- "scr_dir1_threshold_50": null,
111
- "scr_metric_threshold_50": null,
112
- "scr_dir2_threshold_50": null,
113
- "scr_dir1_threshold_100": null,
114
- "scr_metric_threshold_100": null,
115
- "scr_dir2_threshold_100": null,
116
- "scr_dir1_threshold_500": null,
117
- "scr_metric_threshold_500": null,
118
- "scr_dir2_threshold_500": null
119
  },
120
  {
121
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
@@ -128,18 +132,18 @@
128
  "scr_dir1_threshold_10": 0.10606057185225456,
129
  "scr_metric_threshold_10": 0.34862389836223945,
130
  "scr_dir2_threshold_10": 0.34862389836223945,
131
- "scr_dir1_threshold_20": null,
132
- "scr_metric_threshold_20": null,
133
- "scr_dir2_threshold_20": null,
134
- "scr_dir1_threshold_50": null,
135
- "scr_metric_threshold_50": null,
136
- "scr_dir2_threshold_50": null,
137
- "scr_dir1_threshold_100": null,
138
- "scr_metric_threshold_100": null,
139
- "scr_dir2_threshold_100": null,
140
- "scr_dir1_threshold_500": null,
141
- "scr_metric_threshold_500": null,
142
- "scr_dir2_threshold_500": null
143
  },
144
  {
145
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
@@ -152,18 +156,18 @@
152
  "scr_dir1_threshold_10": 0.17187462165026607,
153
  "scr_metric_threshold_10": 0.128140681316927,
154
  "scr_dir2_threshold_10": 0.128140681316927,
155
- "scr_dir1_threshold_20": null,
156
- "scr_metric_threshold_20": null,
157
- "scr_dir2_threshold_20": null,
158
- "scr_dir1_threshold_50": null,
159
- "scr_metric_threshold_50": null,
160
- "scr_dir2_threshold_50": null,
161
- "scr_dir1_threshold_100": null,
162
- "scr_metric_threshold_100": null,
163
- "scr_dir2_threshold_100": null,
164
- "scr_dir1_threshold_500": null,
165
- "scr_metric_threshold_500": null,
166
- "scr_dir2_threshold_500": null
167
  },
168
  {
169
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
@@ -176,18 +180,18 @@
176
  "scr_dir1_threshold_10": -0.1744968107512386,
177
  "scr_metric_threshold_10": 0.2611465427598772,
178
  "scr_dir2_threshold_10": 0.2611465427598772,
179
- "scr_dir1_threshold_20": null,
180
- "scr_metric_threshold_20": null,
181
- "scr_dir2_threshold_20": null,
182
- "scr_dir1_threshold_50": null,
183
- "scr_metric_threshold_50": null,
184
- "scr_dir2_threshold_50": null,
185
- "scr_dir1_threshold_100": null,
186
- "scr_metric_threshold_100": null,
187
- "scr_dir2_threshold_100": null,
188
- "scr_dir1_threshold_500": null,
189
- "scr_metric_threshold_500": null,
190
- "scr_dir2_threshold_500": null
191
  },
192
  {
193
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
@@ -200,18 +204,18 @@
200
  "scr_dir1_threshold_10": 0.08421069803000372,
201
  "scr_metric_threshold_10": 0.5396826147702655,
202
  "scr_dir2_threshold_10": 0.5396826147702655,
203
- "scr_dir1_threshold_20": null,
204
- "scr_metric_threshold_20": null,
205
- "scr_dir2_threshold_20": null,
206
- "scr_dir1_threshold_50": null,
207
- "scr_metric_threshold_50": null,
208
- "scr_dir2_threshold_50": null,
209
- "scr_dir1_threshold_100": null,
210
- "scr_metric_threshold_100": null,
211
- "scr_dir2_threshold_100": null,
212
- "scr_dir1_threshold_500": null,
213
- "scr_metric_threshold_500": null,
214
- "scr_dir2_threshold_500": null
215
  },
216
  {
217
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
@@ -224,18 +228,18 @@
224
  "scr_dir1_threshold_10": 0.2950818016891243,
225
  "scr_metric_threshold_10": 0.12977090206161526,
226
  "scr_dir2_threshold_10": 0.12977090206161526,
227
- "scr_dir1_threshold_20": null,
228
- "scr_metric_threshold_20": null,
229
- "scr_dir2_threshold_20": null,
230
- "scr_dir1_threshold_50": null,
231
- "scr_metric_threshold_50": null,
232
- "scr_dir2_threshold_50": null,
233
- "scr_dir1_threshold_100": null,
234
- "scr_metric_threshold_100": null,
235
- "scr_dir2_threshold_100": null,
236
- "scr_dir1_threshold_500": null,
237
- "scr_metric_threshold_500": null,
238
- "scr_dir2_threshold_500": null
239
  },
240
  {
241
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
@@ -248,18 +252,18 @@
248
  "scr_dir1_threshold_10": 0.17209299875634235,
249
  "scr_metric_threshold_10": 0.41201726948797046,
250
  "scr_dir2_threshold_10": 0.41201726948797046,
251
- "scr_dir1_threshold_20": null,
252
- "scr_metric_threshold_20": null,
253
- "scr_dir2_threshold_20": null,
254
- "scr_dir1_threshold_50": null,
255
- "scr_metric_threshold_50": null,
256
- "scr_dir2_threshold_50": null,
257
- "scr_dir1_threshold_100": null,
258
- "scr_metric_threshold_100": null,
259
- "scr_dir2_threshold_100": null,
260
- "scr_dir1_threshold_500": null,
261
- "scr_metric_threshold_500": null,
262
- "scr_dir2_threshold_500": null
263
  },
264
  {
265
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
@@ -272,18 +276,18 @@
272
  "scr_dir1_threshold_10": 0.09767428192335063,
273
  "scr_metric_threshold_10": 0.09767428192335063,
274
  "scr_dir2_threshold_10": 0.1312216389118512,
275
- "scr_dir1_threshold_20": null,
276
- "scr_metric_threshold_20": null,
277
- "scr_dir2_threshold_20": null,
278
- "scr_dir1_threshold_50": null,
279
- "scr_metric_threshold_50": null,
280
- "scr_dir2_threshold_50": null,
281
- "scr_dir1_threshold_100": null,
282
- "scr_metric_threshold_100": null,
283
- "scr_dir2_threshold_100": null,
284
- "scr_dir1_threshold_500": null,
285
- "scr_metric_threshold_500": null,
286
- "scr_dir2_threshold_500": null
287
  }
288
  ],
289
  "sae_bench_commit_hash": "Unknown",
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "a37c4bb3-d5b0-4272-91e6-4c8751e20e9e",
73
+ "datetime_epoch_millis": 1777317623684,
74
  "eval_result_metrics": {
75
  "scr_metrics": {
76
  "scr_dir1_threshold_2": 0.21586031401333375,
 
82
  "scr_dir1_threshold_10": 0.1613699480060317,
83
  "scr_metric_threshold_10": 0.25517605668515764,
84
  "scr_dir2_threshold_10": 0.2593694763087202,
85
+ "scr_dir1_threshold_20": 0.1910093029330642,
86
+ "scr_metric_threshold_20": 0.3165838101358981,
87
+ "scr_dir2_threshold_20": 0.312574556185402,
88
+ "scr_dir1_threshold_50": 0.16166733397527638,
89
+ "scr_metric_threshold_50": 0.39608407394084566,
90
+ "scr_dir2_threshold_50": 0.3930008270806805,
91
+ "scr_dir1_threshold_100": 0.18280301469479515,
92
+ "scr_metric_threshold_100": 0.4187884297226195,
93
+ "scr_dir2_threshold_100": 0.4200722327355718,
94
+ "scr_dir1_threshold_500": -0.2977460896134362,
95
+ "scr_metric_threshold_500": 0.2694611342827654,
96
+ "scr_dir2_threshold_500": 0.27021090255243746
97
  }
98
  },
99
  "eval_result_details": [
 
108
  "scr_dir1_threshold_10": 0.5384614208981502,
109
  "scr_metric_threshold_10": 0.12435226279901532,
110
  "scr_dir2_threshold_10": 0.12435226279901532,
111
+ "scr_dir1_threshold_20": 0.5512825313325533,
112
+ "scr_metric_threshold_20": 0.20207254286052237,
113
+ "scr_dir2_threshold_20": 0.20207254286052237,
114
+ "scr_dir1_threshold_50": 0.5384614208981502,
115
+ "scr_metric_threshold_50": 0.29792745713947766,
116
+ "scr_dir2_threshold_50": 0.29792745713947766,
117
+ "scr_dir1_threshold_100": 0.5000003820810118,
118
+ "scr_metric_threshold_100": 0.3212435257163134,
119
+ "scr_dir2_threshold_100": 0.3212435257163134,
120
+ "scr_dir1_threshold_500": -1.0512813850895177,
121
+ "scr_metric_threshold_500": 0.3704663028415971,
122
+ "scr_dir2_threshold_500": 0.3704663028415971
123
  },
124
  {
125
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
 
132
  "scr_dir1_threshold_10": 0.10606057185225456,
133
  "scr_metric_threshold_10": 0.34862389836223945,
134
  "scr_dir2_threshold_10": 0.34862389836223945,
135
+ "scr_dir1_threshold_20": 0.04545485332970889,
136
+ "scr_metric_threshold_20": 0.35474002715930114,
137
+ "scr_dir2_threshold_20": 0.35474002715930114,
138
+ "scr_dir1_threshold_50": 0.03787874340793094,
139
+ "scr_metric_threshold_50": 0.5412845073503276,
140
+ "scr_dir2_threshold_50": 0.5412845073503276,
141
+ "scr_dir1_threshold_100": 0.2878790820706107,
142
+ "scr_metric_threshold_100": 0.5779816446870757,
143
+ "scr_dir2_threshold_100": 0.5779816446870757,
144
+ "scr_dir1_threshold_500": -0.8636358915611131,
145
+ "scr_metric_threshold_500": 0.29969413887699026,
146
+ "scr_dir2_threshold_500": 0.29969413887699026
147
  },
148
  {
149
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
 
156
  "scr_dir1_threshold_10": 0.17187462165026607,
157
  "scr_metric_threshold_10": 0.128140681316927,
158
  "scr_dir2_threshold_10": 0.128140681316927,
159
+ "scr_dir1_threshold_20": 0.28125005820765137,
160
+ "scr_metric_threshold_20": 0.16834176159853043,
161
+ "scr_dir2_threshold_20": 0.16834176159853043,
162
+ "scr_dir1_threshold_50": 0.3593749708961743,
163
+ "scr_metric_threshold_50": 0.2386935023309456,
164
+ "scr_dir2_threshold_50": 0.2386935023309456,
165
+ "scr_dir1_threshold_100": -0.09374970896174313,
166
+ "scr_metric_threshold_100": 0.042713610359105945,
167
+ "scr_dir2_threshold_100": 0.042713610359105945,
168
+ "scr_dir1_threshold_500": 0.0,
169
+ "scr_metric_threshold_500": -0.04522614043660845,
170
+ "scr_dir2_threshold_500": -0.04522614043660845
171
  },
172
  {
173
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
 
180
  "scr_dir1_threshold_10": -0.1744968107512386,
181
  "scr_metric_threshold_10": 0.2611465427598772,
182
  "scr_dir2_threshold_10": 0.2611465427598772,
183
+ "scr_dir1_threshold_20": -0.1409397933038777,
184
+ "scr_metric_threshold_20": 0.3598727118176265,
185
+ "scr_dir2_threshold_20": 0.3598727118176265,
186
+ "scr_dir1_threshold_50": -0.40268460939953316,
187
+ "scr_metric_threshold_50": 0.4522293465167161,
188
+ "scr_dir2_threshold_50": 0.4522293465167161,
189
+ "scr_dir1_threshold_100": -0.12751714633741432,
190
+ "scr_metric_threshold_100": 0.5732484112705166,
191
+ "scr_dir2_threshold_100": 0.5732484112705166,
192
+ "scr_dir1_threshold_500": -1.2684565396100895,
193
+ "scr_metric_threshold_500": -0.38853495204863436,
194
+ "scr_dir2_threshold_500": -0.38853495204863436
195
  },
196
  {
197
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
 
204
  "scr_dir1_threshold_10": 0.08421069803000372,
205
  "scr_metric_threshold_10": 0.5396826147702655,
206
  "scr_dir2_threshold_10": 0.5396826147702655,
207
+ "scr_dir1_threshold_20": 0.0631580235225028,
208
+ "scr_metric_threshold_20": 0.6190476077844602,
209
+ "scr_dir2_threshold_20": 0.6190476077844602,
210
+ "scr_dir1_threshold_50": -0.17894726275076694,
211
+ "scr_metric_threshold_50": 0.6865079937623274,
212
+ "scr_dir2_threshold_50": 0.6865079937623274,
213
+ "scr_dir1_threshold_100": -0.15263149804355594,
214
+ "scr_metric_threshold_100": 0.7063492420158761,
215
+ "scr_dir2_threshold_100": 0.7063492420158761,
216
+ "scr_dir1_threshold_500": -0.2473683764729798,
217
+ "scr_metric_threshold_500": 0.6428570583834513,
218
+ "scr_dir2_threshold_500": 0.6428570583834513
219
  },
220
  {
221
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
 
228
  "scr_dir1_threshold_10": 0.2950818016891243,
229
  "scr_metric_threshold_10": 0.12977090206161526,
230
  "scr_dir2_threshold_10": 0.12977090206161526,
231
+ "scr_dir1_threshold_20": 0.3278689218723027,
232
+ "scr_metric_threshold_20": 0.17938926174649328,
233
+ "scr_dir2_threshold_20": 0.17938926174649328,
234
+ "scr_dir1_threshold_50": 0.4043716594522235,
235
+ "scr_metric_threshold_50": 0.22519087703221677,
236
+ "scr_dir2_threshold_50": 0.22519087703221677,
237
+ "scr_dir1_threshold_100": 0.3879780993606342,
238
+ "scr_metric_threshold_100": 0.31679388010504067,
239
+ "scr_dir2_threshold_100": 0.31679388010504067,
240
+ "scr_dir1_threshold_500": 0.5464479859815062,
241
+ "scr_metric_threshold_500": 0.4694657035588292,
242
+ "scr_dir2_threshold_500": 0.4694657035588292
243
  },
244
  {
245
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
 
252
  "scr_dir1_threshold_10": 0.17209299875634235,
253
  "scr_metric_threshold_10": 0.41201726948797046,
254
  "scr_dir2_threshold_10": 0.41201726948797046,
255
+ "scr_dir1_threshold_20": 0.21860457508548414,
256
+ "scr_metric_threshold_20": 0.46781131470206244,
257
+ "scr_dir2_threshold_20": 0.46781131470206244,
258
+ "scr_dir1_threshold_50": 0.2930233189968013,
259
+ "scr_metric_threshold_50": 0.48497847709352376,
260
+ "scr_dir2_threshold_50": 0.48497847709352376,
261
+ "scr_dir1_threshold_100": 0.3720928878639972,
262
+ "scr_metric_threshold_100": 0.5236051041022068,
263
+ "scr_dir2_threshold_100": 0.5236051041022068,
264
+ "scr_dir1_threshold_500": 0.22325567727222573,
265
+ "scr_metric_threshold_500": 0.5278971505140198,
266
+ "scr_dir2_threshold_500": 0.5278971505140198
267
  },
268
  {
269
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
 
276
  "scr_dir1_threshold_10": 0.09767428192335063,
277
  "scr_metric_threshold_10": 0.09767428192335063,
278
  "scr_dir2_threshold_10": 0.1312216389118512,
279
+ "scr_dir1_threshold_20": 0.18139525341818818,
280
+ "scr_metric_threshold_20": 0.18139525341818818,
281
+ "scr_dir2_threshold_20": 0.1493212218142196,
282
+ "scr_dir1_threshold_50": 0.24186043030123083,
283
+ "scr_metric_threshold_50": 0.24186043030123083,
284
+ "scr_dir2_threshold_50": 0.2171944554199093,
285
+ "scr_dir1_threshold_100": 0.2883720195248206,
286
+ "scr_metric_threshold_100": 0.2883720195248206,
287
+ "scr_dir2_threshold_100": 0.2986424436284392,
288
+ "scr_dir1_threshold_500": 0.27906981257247854,
289
+ "scr_metric_threshold_500": 0.27906981257247854,
290
+ "scr_dir2_threshold_500": 0.2850679587298548
291
  }
292
  ],
293
  "sae_bench_commit_hash": "Unknown",
gemma-2-2b-btk/k-100/seed-0/snapshots/step-48828-tokens-200000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "78264222-05e3-4676-a40e-b5ca2e6e59c4",
69
- "datetime_epoch_millis": 1776886768606,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.009649993479251861,
73
- "tpp_threshold_2_intended_diff_only": 0.009399998188018798,
74
- "tpp_threshold_2_unintended_diff_only": -0.0002499952912330627,
75
- "tpp_threshold_5_total_metric": 0.03132500797510147,
76
- "tpp_threshold_5_intended_diff_only": 0.034900009632110596,
77
- "tpp_threshold_5_unintended_diff_only": 0.0035750016570091246,
78
- "tpp_threshold_10_total_metric": 0.07525000125169753,
79
- "tpp_threshold_10_intended_diff_only": 0.08310000896453858,
80
- "tpp_threshold_10_unintended_diff_only": 0.007850007712841034,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.01219998300075531,
99
- "tpp_threshold_2_intended_diff_only": 0.011599993705749512,
100
- "tpp_threshold_2_unintended_diff_only": -0.0005999892950057983,
101
- "tpp_threshold_5_total_metric": 0.049300011992454526,
102
- "tpp_threshold_5_intended_diff_only": 0.0538000226020813,
103
- "tpp_threshold_5_unintended_diff_only": 0.00450001060962677,
104
- "tpp_threshold_10_total_metric": 0.12895000278949736,
105
- "tpp_threshold_10_intended_diff_only": 0.1414000153541565,
106
- "tpp_threshold_10_unintended_diff_only": 0.012450012564659118,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.007100003957748413,
123
- "tpp_threshold_2_intended_diff_only": 0.007200002670288086,
124
- "tpp_threshold_2_unintended_diff_only": 9.999871253967285e-05,
125
- "tpp_threshold_5_total_metric": 0.013350003957748413,
126
- "tpp_threshold_5_intended_diff_only": 0.015999996662139894,
127
- "tpp_threshold_5_unintended_diff_only": 0.0026499927043914795,
128
- "tpp_threshold_10_total_metric": 0.021549999713897705,
129
- "tpp_threshold_10_intended_diff_only": 0.024800002574920654,
130
- "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.0012500137090682983,
190
  "tpp_threshold_10_total_metric": 0.009249940514564514,
191
  "tpp_threshold_10_intended_diff_only": 0.011999964714050293,
192
- "tpp_threshold_10_unintended_diff_only": 0.002750024199485779
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": 0.0002499818801879883,
196
- "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
197
- "tpp_threshold_2_unintended_diff_only": 0.0007500052452087402,
198
- "tpp_threshold_5_total_metric": 0.06250002980232239,
199
- "tpp_threshold_5_intended_diff_only": 0.07200002670288086,
200
- "tpp_threshold_5_unintended_diff_only": 0.009499996900558472,
201
- "tpp_threshold_10_total_metric": 0.14600001275539398,
202
- "tpp_threshold_10_intended_diff_only": 0.16500002145767212,
203
- "tpp_threshold_10_unintended_diff_only": 0.019000008702278137
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.034000009298324585,
207
- "tpp_threshold_2_intended_diff_only": 0.03100001811981201,
208
- "tpp_threshold_2_unintended_diff_only": -0.0029999911785125732,
209
- "tpp_threshold_5_total_metric": 0.13975003361701965,
210
- "tpp_threshold_5_intended_diff_only": 0.14400005340576172,
211
- "tpp_threshold_5_unintended_diff_only": 0.004250019788742065,
212
- "tpp_threshold_10_total_metric": 0.2150000035762787,
213
- "tpp_threshold_10_intended_diff_only": 0.2250000238418579,
214
- "tpp_threshold_10_unintended_diff_only": 0.010000020265579224
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.011500000953674316,
218
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
219
- "tpp_threshold_2_unintended_diff_only": -0.002499997615814209,
220
- "tpp_threshold_5_total_metric": 0.020250022411346436,
221
- "tpp_threshold_5_intended_diff_only": 0.026000022888183594,
222
- "tpp_threshold_5_unintended_diff_only": 0.005750000476837158,
223
- "tpp_threshold_10_total_metric": 0.16025002300739288,
224
  "tpp_threshold_10_intended_diff_only": 0.17400002479553223,
225
- "tpp_threshold_10_unintended_diff_only": 0.013750001788139343
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.00449998676776886,
229
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
230
- "tpp_threshold_2_unintended_diff_only": 0.0005000084638595581,
231
- "tpp_threshold_5_total_metric": 0.010250002145767212,
232
- "tpp_threshold_5_intended_diff_only": 0.012000024318695068,
233
- "tpp_threshold_5_unintended_diff_only": 0.0017500221729278564,
234
- "tpp_threshold_10_total_metric": 0.11425003409385681,
235
- "tpp_threshold_10_intended_diff_only": 0.13100004196166992,
236
- "tpp_threshold_10_unintended_diff_only": 0.01675000786781311
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.013250023126602173,
242
- "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
243
- "tpp_threshold_2_unintended_diff_only": -0.005250006914138794,
244
- "tpp_threshold_5_total_metric": 0.012250006198883057,
245
- "tpp_threshold_5_intended_diff_only": 0.009999990463256836,
246
- "tpp_threshold_5_unintended_diff_only": -0.0022500157356262207,
247
- "tpp_threshold_10_total_metric": 0.0040000081062316895,
248
- "tpp_threshold_10_intended_diff_only": 0.008000016212463379,
249
- "tpp_threshold_10_unintended_diff_only": 0.0040000081062316895
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": -0.0007499456405639648,
253
- "tpp_threshold_2_intended_diff_only": 0.005000054836273193,
254
- "tpp_threshold_2_unintended_diff_only": 0.005750000476837158,
255
- "tpp_threshold_5_total_metric": 0.006250053644180298,
256
- "tpp_threshold_5_intended_diff_only": 0.01500004529953003,
257
- "tpp_threshold_5_unintended_diff_only": 0.008749991655349731,
258
- "tpp_threshold_10_total_metric": 0.027500063180923462,
259
- "tpp_threshold_10_intended_diff_only": 0.033000051975250244,
260
- "tpp_threshold_10_unintended_diff_only": 0.005499988794326782
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.012750014662742615,
264
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
265
- "tpp_threshold_2_unintended_diff_only": -0.007750019431114197,
266
- "tpp_threshold_5_total_metric": 0.017999976873397827,
267
- "tpp_threshold_5_intended_diff_only": 0.015999972820281982,
268
- "tpp_threshold_5_unintended_diff_only": -0.0020000040531158447,
269
- "tpp_threshold_10_total_metric": 0.01724998652935028,
270
- "tpp_threshold_10_intended_diff_only": 0.018999993801116943,
271
- "tpp_threshold_10_unintended_diff_only": 0.0017500072717666626
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.00625002384185791,
275
- "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
276
- "tpp_threshold_2_unintended_diff_only": 0.007250010967254639,
277
- "tpp_threshold_5_total_metric": 0.0007499754428863525,
278
- "tpp_threshold_5_intended_diff_only": 0.001999974250793457,
279
- "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045,
280
- "tpp_threshold_10_total_metric": 0.004749968647956848,
281
- "tpp_threshold_10_intended_diff_only": 0.006999969482421875,
282
- "tpp_threshold_10_unintended_diff_only": 0.002250000834465027
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.016499951481819153,
286
- "tpp_threshold_2_intended_diff_only": 0.01699995994567871,
287
- "tpp_threshold_2_unintended_diff_only": 0.0005000084638595581,
288
- "tpp_threshold_5_total_metric": 0.02950000762939453,
289
- "tpp_threshold_5_intended_diff_only": 0.03700000047683716,
290
- "tpp_threshold_5_unintended_diff_only": 0.007499992847442627,
291
- "tpp_threshold_10_total_metric": 0.054249972105026245,
292
- "tpp_threshold_10_intended_diff_only": 0.05699998140335083,
293
- "tpp_threshold_10_unintended_diff_only": 0.002750009298324585
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "ed5e8dd3-ebe7-4034-9a63-cce13150c43a",
73
+ "datetime_epoch_millis": 1777317899194,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.007799991965293884,
77
+ "tpp_threshold_2_intended_diff_only": 0.008899998664855958,
78
+ "tpp_threshold_2_unintended_diff_only": 0.0011000066995620727,
79
+ "tpp_threshold_5_total_metric": 0.031874993443489076,
80
+ "tpp_threshold_5_intended_diff_only": 0.033799999952316286,
81
+ "tpp_threshold_5_unintended_diff_only": 0.0019250065088272095,
82
+ "tpp_threshold_10_total_metric": 0.07579998970031739,
83
+ "tpp_threshold_10_intended_diff_only": 0.08429999351501465,
84
+ "tpp_threshold_10_unintended_diff_only": 0.008500003814697265,
85
+ "tpp_threshold_20_total_metric": 0.12705000191926957,
86
+ "tpp_threshold_20_intended_diff_only": 0.13920000791549683,
87
+ "tpp_threshold_20_unintended_diff_only": 0.012150005996227264,
88
+ "tpp_threshold_50_total_metric": 0.19905001521110532,
89
+ "tpp_threshold_50_intended_diff_only": 0.22660001516342163,
90
+ "tpp_threshold_50_unintended_diff_only": 0.027549999952316283,
91
+ "tpp_threshold_100_total_metric": 0.255075016617775,
92
+ "tpp_threshold_100_intended_diff_only": 0.3065000236034393,
93
+ "tpp_threshold_100_unintended_diff_only": 0.051425006985664365,
94
+ "tpp_threshold_500_total_metric": 0.2917000249028206,
95
+ "tpp_threshold_500_intended_diff_only": 0.4251000344753265,
96
+ "tpp_threshold_500_unintended_diff_only": 0.13340000957250595
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.010449990630149841,
103
+ "tpp_threshold_2_intended_diff_only": 0.01140000820159912,
104
+ "tpp_threshold_2_unintended_diff_only": 0.0009500175714492798,
105
+ "tpp_threshold_5_total_metric": 0.05114998817443848,
106
+ "tpp_threshold_5_intended_diff_only": 0.05460000038146973,
107
+ "tpp_threshold_5_unintended_diff_only": 0.00345001220703125,
108
+ "tpp_threshold_10_total_metric": 0.12804999053478242,
109
+ "tpp_threshold_10_intended_diff_only": 0.14140000343322753,
110
+ "tpp_threshold_10_unintended_diff_only": 0.013350012898445129,
111
+ "tpp_threshold_20_total_metric": 0.20720000565052032,
112
+ "tpp_threshold_20_intended_diff_only": 0.22640001773834229,
113
+ "tpp_threshold_20_unintended_diff_only": 0.01920001208782196,
114
+ "tpp_threshold_50_total_metric": 0.3053500235080719,
115
+ "tpp_threshold_50_intended_diff_only": 0.3548000335693359,
116
+ "tpp_threshold_50_unintended_diff_only": 0.04945001006126404,
117
+ "tpp_threshold_100_total_metric": 0.33660002052783966,
118
+ "tpp_threshold_100_intended_diff_only": 0.42760003805160524,
119
+ "tpp_threshold_100_unintended_diff_only": 0.09100001752376556,
120
+ "tpp_threshold_500_total_metric": 0.22840003073215484,
121
+ "tpp_threshold_500_intended_diff_only": 0.466200053691864,
122
+ "tpp_threshold_500_unintended_diff_only": 0.23780002295970917
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.005149993300437927,
127
+ "tpp_threshold_2_intended_diff_only": 0.006399989128112793,
128
+ "tpp_threshold_2_unintended_diff_only": 0.0012499958276748658,
129
+ "tpp_threshold_5_total_metric": 0.012599998712539673,
130
+ "tpp_threshold_5_intended_diff_only": 0.012999999523162841,
131
+ "tpp_threshold_5_unintended_diff_only": 0.0004000008106231689,
132
+ "tpp_threshold_10_total_metric": 0.023549988865852356,
133
+ "tpp_threshold_10_intended_diff_only": 0.027199983596801758,
134
+ "tpp_threshold_10_unintended_diff_only": 0.003649994730949402,
135
+ "tpp_threshold_20_total_metric": 0.0468999981880188,
136
+ "tpp_threshold_20_intended_diff_only": 0.051999998092651364,
137
+ "tpp_threshold_20_unintended_diff_only": 0.0050999999046325685,
138
+ "tpp_threshold_50_total_metric": 0.09275000691413879,
139
+ "tpp_threshold_50_intended_diff_only": 0.09839999675750732,
140
+ "tpp_threshold_50_unintended_diff_only": 0.005649989843368531,
141
+ "tpp_threshold_100_total_metric": 0.17355001270771026,
142
+ "tpp_threshold_100_intended_diff_only": 0.18540000915527344,
143
+ "tpp_threshold_100_unintended_diff_only": 0.011849996447563172,
144
+ "tpp_threshold_500_total_metric": 0.35500001907348633,
145
+ "tpp_threshold_500_intended_diff_only": 0.3840000152587891,
146
+ "tpp_threshold_500_unintended_diff_only": 0.028999996185302735
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.0012500137090682983,
194
  "tpp_threshold_10_total_metric": 0.009249940514564514,
195
  "tpp_threshold_10_intended_diff_only": 0.011999964714050293,
196
+ "tpp_threshold_10_unintended_diff_only": 0.002750024199485779,
197
+ "tpp_threshold_20_total_metric": 0.1772499680519104,
198
+ "tpp_threshold_20_intended_diff_only": 0.20399999618530273,
199
+ "tpp_threshold_20_unintended_diff_only": 0.026750028133392334,
200
+ "tpp_threshold_50_total_metric": 0.2902500182390213,
201
+ "tpp_threshold_50_intended_diff_only": 0.33100003004074097,
202
+ "tpp_threshold_50_unintended_diff_only": 0.040750011801719666,
203
+ "tpp_threshold_100_total_metric": 0.3554999530315399,
204
+ "tpp_threshold_100_intended_diff_only": 0.4129999876022339,
205
+ "tpp_threshold_100_unintended_diff_only": 0.05750003457069397,
206
+ "tpp_threshold_500_total_metric": 0.24500000476837158,
207
+ "tpp_threshold_500_intended_diff_only": 0.45600003004074097,
208
+ "tpp_threshold_500_unintended_diff_only": 0.21100002527236938
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.004499971866607666,
212
+ "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
213
+ "tpp_threshold_2_unintended_diff_only": 0.000500023365020752,
214
+ "tpp_threshold_5_total_metric": 0.06474997103214264,
215
+ "tpp_threshold_5_intended_diff_only": 0.07099997997283936,
216
+ "tpp_threshold_5_unintended_diff_only": 0.006250008940696716,
217
+ "tpp_threshold_10_total_metric": 0.14949999749660492,
218
+ "tpp_threshold_10_intended_diff_only": 0.16600000858306885,
219
+ "tpp_threshold_10_unintended_diff_only": 0.016500011086463928,
220
+ "tpp_threshold_20_total_metric": 0.17675000429153442,
221
+ "tpp_threshold_20_intended_diff_only": 0.1940000057220459,
222
+ "tpp_threshold_20_unintended_diff_only": 0.017250001430511475,
223
+ "tpp_threshold_50_total_metric": 0.2604999840259552,
224
+ "tpp_threshold_50_intended_diff_only": 0.3059999942779541,
225
+ "tpp_threshold_50_unintended_diff_only": 0.0455000102519989,
226
+ "tpp_threshold_100_total_metric": 0.3087500333786011,
227
+ "tpp_threshold_100_intended_diff_only": 0.40400004386901855,
228
+ "tpp_threshold_100_unintended_diff_only": 0.09525001049041748,
229
+ "tpp_threshold_500_total_metric": 0.1952500343322754,
230
+ "tpp_threshold_500_intended_diff_only": 0.4590000510215759,
231
+ "tpp_threshold_500_unintended_diff_only": 0.26375001668930054
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.021750018000602722,
235
+ "tpp_threshold_2_intended_diff_only": 0.025000035762786865,
236
+ "tpp_threshold_2_unintended_diff_only": 0.003250017762184143,
237
+ "tpp_threshold_5_total_metric": 0.13774999976158142,
238
+ "tpp_threshold_5_intended_diff_only": 0.14300000667572021,
239
+ "tpp_threshold_5_unintended_diff_only": 0.005250006914138794,
240
+ "tpp_threshold_10_total_metric": 0.21074999868869781,
241
+ "tpp_threshold_10_intended_diff_only": 0.22600001096725464,
242
+ "tpp_threshold_10_unintended_diff_only": 0.015250012278556824,
243
+ "tpp_threshold_20_total_metric": 0.27675004303455353,
244
+ "tpp_threshold_20_intended_diff_only": 0.3020000457763672,
245
+ "tpp_threshold_20_unintended_diff_only": 0.02525000274181366,
246
+ "tpp_threshold_50_total_metric": 0.2890000492334366,
247
+ "tpp_threshold_50_intended_diff_only": 0.35200005769729614,
248
+ "tpp_threshold_50_unintended_diff_only": 0.06300000846385956,
249
+ "tpp_threshold_100_total_metric": 0.27775004506111145,
250
+ "tpp_threshold_100_intended_diff_only": 0.3890000581741333,
251
+ "tpp_threshold_100_unintended_diff_only": 0.11125001311302185,
252
+ "tpp_threshold_500_total_metric": 0.13625003397464752,
253
+ "tpp_threshold_500_intended_diff_only": 0.4490000605583191,
254
+ "tpp_threshold_500_unintended_diff_only": 0.31275002658367157
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.009499981999397278,
258
  "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704,
260
+ "tpp_threshold_5_total_metric": 0.025499999523162842,
261
+ "tpp_threshold_5_intended_diff_only": 0.027000010013580322,
262
+ "tpp_threshold_5_unintended_diff_only": 0.0015000104904174805,
263
+ "tpp_threshold_10_total_metric": 0.1600000113248825,
264
  "tpp_threshold_10_intended_diff_only": 0.17400002479553223,
265
+ "tpp_threshold_10_unintended_diff_only": 0.01400001347064972,
266
+ "tpp_threshold_20_total_metric": 0.20149999856948853,
267
+ "tpp_threshold_20_intended_diff_only": 0.2120000123977661,
268
+ "tpp_threshold_20_unintended_diff_only": 0.010500013828277588,
269
+ "tpp_threshold_50_total_metric": 0.3760000169277191,
270
+ "tpp_threshold_50_intended_diff_only": 0.45600003004074097,
271
+ "tpp_threshold_50_unintended_diff_only": 0.08000001311302185,
272
+ "tpp_threshold_100_total_metric": 0.3530000150203705,
273
+ "tpp_threshold_100_intended_diff_only": 0.4790000319480896,
274
+ "tpp_threshold_100_unintended_diff_only": 0.12600001692771912,
275
+ "tpp_threshold_500_total_metric": 0.2722500413656235,
276
+ "tpp_threshold_500_intended_diff_only": 0.49100005626678467,
277
+ "tpp_threshold_500_unintended_diff_only": 0.2187500149011612
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.00575004518032074,
281
+ "tpp_threshold_2_intended_diff_only": 0.006000041961669922,
282
+ "tpp_threshold_2_unintended_diff_only": 0.00024999678134918213,
283
+ "tpp_threshold_5_total_metric": 0.013999998569488525,
284
+ "tpp_threshold_5_intended_diff_only": 0.017000019550323486,
285
+ "tpp_threshold_5_unintended_diff_only": 0.003000020980834961,
286
+ "tpp_threshold_10_total_metric": 0.11075000464916229,
287
+ "tpp_threshold_10_intended_diff_only": 0.1290000081062317,
288
+ "tpp_threshold_10_unintended_diff_only": 0.018250003457069397,
289
+ "tpp_threshold_20_total_metric": 0.20375001430511475,
290
+ "tpp_threshold_20_intended_diff_only": 0.2200000286102295,
291
+ "tpp_threshold_20_unintended_diff_only": 0.016250014305114746,
292
+ "tpp_threshold_50_total_metric": 0.3110000491142273,
293
+ "tpp_threshold_50_intended_diff_only": 0.3290000557899475,
294
+ "tpp_threshold_50_unintended_diff_only": 0.018000006675720215,
295
+ "tpp_threshold_100_total_metric": 0.3880000561475754,
296
+ "tpp_threshold_100_intended_diff_only": 0.4530000686645508,
297
+ "tpp_threshold_100_unintended_diff_only": 0.0650000125169754,
298
+ "tpp_threshold_500_total_metric": 0.29325003921985626,
299
+ "tpp_threshold_500_intended_diff_only": 0.4760000705718994,
300
+ "tpp_threshold_500_unintended_diff_only": 0.18275003135204315
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.0065000057220458984,
306
+ "tpp_threshold_2_intended_diff_only": 0.009999990463256836,
307
+ "tpp_threshold_2_unintended_diff_only": 0.0034999847412109375,
308
+ "tpp_threshold_5_total_metric": 0.0070000141859054565,
309
+ "tpp_threshold_5_intended_diff_only": 0.008000016212463379,
310
+ "tpp_threshold_5_unintended_diff_only": 0.0010000020265579224,
311
+ "tpp_threshold_10_total_metric": 0.004749983549118042,
312
+ "tpp_threshold_10_intended_diff_only": 0.006999969482421875,
313
+ "tpp_threshold_10_unintended_diff_only": 0.002249985933303833,
314
+ "tpp_threshold_20_total_metric": 0.015250012278556824,
315
+ "tpp_threshold_20_intended_diff_only": 0.017000019550323486,
316
+ "tpp_threshold_20_unintended_diff_only": 0.0017500072717666626,
317
+ "tpp_threshold_50_total_metric": 0.0429999977350235,
318
+ "tpp_threshold_50_intended_diff_only": 0.042999982833862305,
319
+ "tpp_threshold_50_unintended_diff_only": -1.4901161193847656e-08,
320
+ "tpp_threshold_100_total_metric": 0.10950000584125519,
321
+ "tpp_threshold_100_intended_diff_only": 0.11599999666213989,
322
+ "tpp_threshold_100_unintended_diff_only": 0.006499990820884705,
323
+ "tpp_threshold_500_total_metric": 0.3760000169277191,
324
+ "tpp_threshold_500_intended_diff_only": 0.3840000033378601,
325
+ "tpp_threshold_500_unintended_diff_only": 0.007999986410140991
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.010000020265579224,
329
+ "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
330
+ "tpp_threshold_2_unintended_diff_only": -0.0020000040531158447,
331
+ "tpp_threshold_5_total_metric": 0.0030000656843185425,
332
+ "tpp_threshold_5_intended_diff_only": 0.010000050067901611,
333
+ "tpp_threshold_5_unintended_diff_only": 0.006999984383583069,
334
+ "tpp_threshold_10_total_metric": 0.028750017285346985,
335
+ "tpp_threshold_10_intended_diff_only": 0.03600001335144043,
336
+ "tpp_threshold_10_unintended_diff_only": 0.007249996066093445,
337
+ "tpp_threshold_20_total_metric": 0.06525000929832458,
338
+ "tpp_threshold_20_intended_diff_only": 0.07400000095367432,
339
+ "tpp_threshold_20_unintended_diff_only": 0.008749991655349731,
340
+ "tpp_threshold_50_total_metric": 0.12475007772445679,
341
+ "tpp_threshold_50_intended_diff_only": 0.1350000500679016,
342
+ "tpp_threshold_50_unintended_diff_only": 0.010249972343444824,
343
+ "tpp_threshold_100_total_metric": 0.27225005626678467,
344
+ "tpp_threshold_100_intended_diff_only": 0.29700005054473877,
345
+ "tpp_threshold_100_unintended_diff_only": 0.0247499942779541,
346
+ "tpp_threshold_500_total_metric": 0.4017500728368759,
347
+ "tpp_threshold_500_intended_diff_only": 0.43500006198883057,
348
+ "tpp_threshold_500_unintended_diff_only": 0.03324998915195465
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.0015000104904174805,
352
+ "tpp_threshold_2_intended_diff_only": 0.0,
353
+ "tpp_threshold_2_unintended_diff_only": -0.0015000104904174805,
354
+ "tpp_threshold_5_total_metric": 0.012749984860420227,
355
+ "tpp_threshold_5_intended_diff_only": 0.0059999823570251465,
356
+ "tpp_threshold_5_unintended_diff_only": -0.006750002503395081,
357
+ "tpp_threshold_10_total_metric": 0.01849997043609619,
358
+ "tpp_threshold_10_intended_diff_only": 0.015999972820281982,
359
+ "tpp_threshold_10_unintended_diff_only": -0.002499997615814209,
360
+ "tpp_threshold_20_total_metric": 0.015750005841255188,
361
+ "tpp_threshold_20_intended_diff_only": 0.018000006675720215,
362
+ "tpp_threshold_20_unintended_diff_only": 0.002250000834465027,
363
+ "tpp_threshold_50_total_metric": 0.05575002729892731,
364
+ "tpp_threshold_50_intended_diff_only": 0.054000020027160645,
365
+ "tpp_threshold_50_unintended_diff_only": -0.0017500072717666626,
366
+ "tpp_threshold_100_total_metric": 0.10200001299381256,
367
+ "tpp_threshold_100_intended_diff_only": 0.10600000619888306,
368
+ "tpp_threshold_100_unintended_diff_only": 0.003999993205070496,
369
+ "tpp_threshold_500_total_metric": 0.33400000631809235,
370
+ "tpp_threshold_500_intended_diff_only": 0.34299999475479126,
371
+ "tpp_threshold_500_unintended_diff_only": 0.008999988436698914
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.0024999529123306274,
375
+ "tpp_threshold_2_intended_diff_only": 0.0029999613761901855,
376
+ "tpp_threshold_2_unintended_diff_only": 0.0005000084638595581,
377
+ "tpp_threshold_5_total_metric": 0.009499937295913696,
378
+ "tpp_threshold_5_intended_diff_only": 0.008999943733215332,
379
+ "tpp_threshold_5_unintended_diff_only": -0.0004999935626983643,
380
+ "tpp_threshold_10_total_metric": -0.0004999935626983643,
381
+ "tpp_threshold_10_intended_diff_only": 0.004999995231628418,
382
+ "tpp_threshold_10_unintended_diff_only": 0.005499988794326782,
383
+ "tpp_threshold_20_total_metric": 0.021249964833259583,
384
+ "tpp_threshold_20_intended_diff_only": 0.02199995517730713,
385
+ "tpp_threshold_20_unintended_diff_only": 0.0007499903440475464,
386
+ "tpp_threshold_50_total_metric": 0.054749950766563416,
387
+ "tpp_threshold_50_intended_diff_only": 0.06299996376037598,
388
+ "tpp_threshold_50_unintended_diff_only": 0.008250012993812561,
389
+ "tpp_threshold_100_total_metric": 0.11424997448921204,
390
+ "tpp_threshold_100_intended_diff_only": 0.12199997901916504,
391
+ "tpp_threshold_100_unintended_diff_only": 0.007750004529953003,
392
+ "tpp_threshold_500_total_metric": 0.3189999759197235,
393
+ "tpp_threshold_500_intended_diff_only": 0.38499999046325684,
394
+ "tpp_threshold_500_unintended_diff_only": 0.06600001454353333
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.005249977111816406,
398
+ "tpp_threshold_2_intended_diff_only": 0.010999977588653564,
399
+ "tpp_threshold_2_unintended_diff_only": 0.005750000476837158,
400
+ "tpp_threshold_5_total_metric": 0.030749991536140442,
401
+ "tpp_threshold_5_intended_diff_only": 0.03200000524520874,
402
+ "tpp_threshold_5_unintended_diff_only": 0.0012500137090682983,
403
+ "tpp_threshold_10_total_metric": 0.06624996662139893,
404
+ "tpp_threshold_10_intended_diff_only": 0.07199996709823608,
405
+ "tpp_threshold_10_unintended_diff_only": 0.005750000476837158,
406
+ "tpp_threshold_20_total_metric": 0.11699999868869781,
407
+ "tpp_threshold_20_intended_diff_only": 0.1290000081062317,
408
+ "tpp_threshold_20_unintended_diff_only": 0.012000009417533875,
409
+ "tpp_threshold_50_total_metric": 0.18549998104572296,
410
+ "tpp_threshold_50_intended_diff_only": 0.19699996709823608,
411
+ "tpp_threshold_50_unintended_diff_only": 0.011499986052513123,
412
+ "tpp_threshold_100_total_metric": 0.2697500139474869,
413
+ "tpp_threshold_100_intended_diff_only": 0.28600001335144043,
414
+ "tpp_threshold_100_unintended_diff_only": 0.016249999403953552,
415
+ "tpp_threshold_500_total_metric": 0.34425002336502075,
416
+ "tpp_threshold_500_intended_diff_only": 0.37300002574920654,
417
+ "tpp_threshold_500_unintended_diff_only": 0.02875000238418579
418
  }
419
  }
420
  }
gemma-2-2b-btk/k-100/seed-0/snapshots/step-6103-tokens-25000000/scr/saebench_scr_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,8 +69,8 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "2ff001da-cfea-436f-8611-67607a0453db",
69
- "datetime_epoch_millis": 1776886666503,
70
  "eval_result_metrics": {
71
  "scr_metrics": {
72
  "scr_dir1_threshold_2": 0.19620984140046563,
@@ -78,18 +82,18 @@
78
  "scr_dir1_threshold_10": 0.2351352741338071,
79
  "scr_metric_threshold_10": 0.28484493965498264,
80
  "scr_dir2_threshold_10": 0.2821089879477948,
81
- "scr_dir1_threshold_20": null,
82
- "scr_metric_threshold_20": null,
83
- "scr_dir2_threshold_20": null,
84
- "scr_dir1_threshold_50": null,
85
- "scr_metric_threshold_50": null,
86
- "scr_dir2_threshold_50": null,
87
- "scr_dir1_threshold_100": null,
88
- "scr_metric_threshold_100": null,
89
- "scr_dir2_threshold_100": null,
90
- "scr_dir1_threshold_500": null,
91
- "scr_metric_threshold_500": null,
92
- "scr_dir2_threshold_500": null
93
  }
94
  },
95
  "eval_result_details": [
@@ -104,18 +108,18 @@
104
  "scr_dir1_threshold_10": 0.5384614208981502,
105
  "scr_metric_threshold_10": 0.08808283994795557,
106
  "scr_dir2_threshold_10": 0.08808283994795557,
107
- "scr_dir1_threshold_20": null,
108
- "scr_metric_threshold_20": null,
109
- "scr_dir2_threshold_20": null,
110
- "scr_dir1_threshold_50": null,
111
- "scr_metric_threshold_50": null,
112
- "scr_dir2_threshold_50": null,
113
- "scr_dir1_threshold_100": null,
114
- "scr_metric_threshold_100": null,
115
- "scr_dir2_threshold_100": null,
116
- "scr_dir1_threshold_500": null,
117
- "scr_metric_threshold_500": null,
118
- "scr_dir2_threshold_500": null
119
  },
120
  {
121
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
@@ -128,18 +132,18 @@
128
  "scr_dir1_threshold_10": 0.12121234014557077,
129
  "scr_metric_threshold_10": 0.30275238555271,
130
  "scr_dir2_threshold_10": 0.30275238555271,
131
- "scr_dir1_threshold_20": null,
132
- "scr_metric_threshold_20": null,
133
- "scr_dir2_threshold_20": null,
134
- "scr_dir1_threshold_50": null,
135
- "scr_metric_threshold_50": null,
136
- "scr_dir2_threshold_50": null,
137
- "scr_dir1_threshold_100": null,
138
- "scr_metric_threshold_100": null,
139
- "scr_dir2_threshold_100": null,
140
- "scr_dir1_threshold_500": null,
141
- "scr_metric_threshold_500": null,
142
- "scr_dir2_threshold_500": null
143
  },
144
  {
145
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
@@ -152,18 +156,18 @@
152
  "scr_dir1_threshold_10": 0.3437501746229541,
153
  "scr_metric_threshold_10": 0.1482412214577287,
154
  "scr_dir2_threshold_10": 0.1482412214577287,
155
- "scr_dir1_threshold_20": null,
156
- "scr_metric_threshold_20": null,
157
- "scr_dir2_threshold_20": null,
158
- "scr_dir1_threshold_50": null,
159
- "scr_metric_threshold_50": null,
160
- "scr_dir2_threshold_50": null,
161
- "scr_dir1_threshold_100": null,
162
- "scr_metric_threshold_100": null,
163
- "scr_dir2_threshold_100": null,
164
- "scr_dir1_threshold_500": null,
165
- "scr_metric_threshold_500": null,
166
- "scr_dir2_threshold_500": null
167
  },
168
  {
169
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
@@ -176,18 +180,18 @@
176
  "scr_dir1_threshold_10": 0.22818779864829455,
177
  "scr_metric_threshold_10": 0.30573252397568285,
178
  "scr_dir2_threshold_10": 0.30573252397568285,
179
- "scr_dir1_threshold_20": null,
180
- "scr_metric_threshold_20": null,
181
- "scr_dir2_threshold_20": null,
182
- "scr_dir1_threshold_50": null,
183
- "scr_metric_threshold_50": null,
184
- "scr_dir2_threshold_50": null,
185
- "scr_dir1_threshold_100": null,
186
- "scr_metric_threshold_100": null,
187
- "scr_dir2_threshold_100": null,
188
- "scr_dir1_threshold_500": null,
189
- "scr_metric_threshold_500": null,
190
- "scr_dir2_threshold_500": null
191
  },
192
  {
193
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
@@ -200,18 +204,18 @@
200
  "scr_dir1_threshold_10": 0.09473687842942387,
201
  "scr_metric_threshold_10": 0.595238157185469,
202
  "scr_dir2_threshold_10": 0.595238157185469,
203
- "scr_dir1_threshold_20": null,
204
- "scr_metric_threshold_20": null,
205
- "scr_dir2_threshold_20": null,
206
- "scr_dir1_threshold_50": null,
207
- "scr_metric_threshold_50": null,
208
- "scr_dir2_threshold_50": null,
209
- "scr_dir1_threshold_100": null,
210
- "scr_metric_threshold_100": null,
211
- "scr_dir2_threshold_100": null,
212
- "scr_dir1_threshold_500": null,
213
- "scr_metric_threshold_500": null,
214
- "scr_dir2_threshold_500": null
215
  },
216
  {
217
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
@@ -224,18 +228,18 @@
224
  "scr_dir1_threshold_10": 0.2896173902280937,
225
  "scr_metric_threshold_10": 0.17557251734733875,
226
  "scr_dir2_threshold_10": 0.17557251734733875,
227
- "scr_dir1_threshold_20": null,
228
- "scr_metric_threshold_20": null,
229
- "scr_dir2_threshold_20": null,
230
- "scr_dir1_threshold_50": null,
231
- "scr_metric_threshold_50": null,
232
- "scr_dir2_threshold_50": null,
233
- "scr_dir1_threshold_100": null,
234
- "scr_metric_threshold_100": null,
235
- "scr_dir2_threshold_100": null,
236
- "scr_dir1_threshold_500": null,
237
- "scr_metric_threshold_500": null,
238
- "scr_dir2_threshold_500": null
239
  },
240
  {
241
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
@@ -248,18 +252,18 @@
248
  "scr_dir1_threshold_10": 0.12558142242720055,
249
  "scr_metric_threshold_10": 0.5236051041022068,
250
  "scr_dir2_threshold_10": 0.5236051041022068,
251
- "scr_dir1_threshold_20": null,
252
- "scr_metric_threshold_20": null,
253
- "scr_dir2_threshold_20": null,
254
- "scr_dir1_threshold_50": null,
255
- "scr_metric_threshold_50": null,
256
- "scr_dir2_threshold_50": null,
257
- "scr_dir1_threshold_100": null,
258
- "scr_metric_threshold_100": null,
259
- "scr_dir2_threshold_100": null,
260
- "scr_dir1_threshold_500": null,
261
- "scr_metric_threshold_500": null,
262
- "scr_dir2_threshold_500": null
263
  },
264
  {
265
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
@@ -272,18 +276,18 @@
272
  "scr_dir1_threshold_10": 0.13953476767076942,
273
  "scr_metric_threshold_10": 0.13953476767076942,
274
  "scr_dir2_threshold_10": 0.11764715401326675,
275
- "scr_dir1_threshold_20": null,
276
- "scr_metric_threshold_20": null,
277
- "scr_dir2_threshold_20": null,
278
- "scr_dir1_threshold_50": null,
279
- "scr_metric_threshold_50": null,
280
- "scr_dir2_threshold_50": null,
281
- "scr_dir1_threshold_100": null,
282
- "scr_metric_threshold_100": null,
283
- "scr_dir2_threshold_100": null,
284
- "scr_dir1_threshold_500": null,
285
- "scr_metric_threshold_500": null,
286
- "scr_dir2_threshold_500": null
287
  }
288
  ],
289
  "sae_bench_commit_hash": "Unknown",
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "24e5bcc8-e66e-4b73-a08d-524b591774b5",
73
+ "datetime_epoch_millis": 1777317728636,
74
  "eval_result_metrics": {
75
  "scr_metrics": {
76
  "scr_dir1_threshold_2": 0.19620984140046563,
 
82
  "scr_dir1_threshold_10": 0.2351352741338071,
83
  "scr_metric_threshold_10": 0.28484493965498264,
84
  "scr_dir2_threshold_10": 0.2821089879477948,
85
+ "scr_dir1_threshold_20": 0.20285064387076104,
86
+ "scr_metric_threshold_20": 0.326805875878005,
87
+ "scr_dir2_threshold_20": 0.32857903549932893,
88
+ "scr_dir1_threshold_50": 0.06344311799392961,
89
+ "scr_metric_threshold_50": 0.39347829473738455,
90
+ "scr_dir2_threshold_50": 0.39913178990759024,
91
+ "scr_dir1_threshold_100": 0.09300308309625738,
92
+ "scr_metric_threshold_100": 0.4048802664204696,
93
+ "scr_dir2_threshold_100": 0.40823974399288304,
94
+ "scr_dir1_threshold_500": 0.20965666674342906,
95
+ "scr_metric_threshold_500": 0.3054615213045736,
96
+ "scr_dir2_threshold_500": 0.3119463499548245
97
  }
98
  },
99
  "eval_result_details": [
 
108
  "scr_dir1_threshold_10": 0.5384614208981502,
109
  "scr_metric_threshold_10": 0.08808283994795557,
110
  "scr_dir2_threshold_10": 0.08808283994795557,
111
+ "scr_dir1_threshold_20": 0.5384614208981502,
112
+ "scr_metric_threshold_20": 0.11139890852479135,
113
+ "scr_dir2_threshold_20": 0.11139890852479135,
114
+ "scr_dir1_threshold_50": 0.48718003580863245,
115
+ "scr_metric_threshold_50": 0.18911918858629842,
116
+ "scr_dir2_threshold_50": 0.18911918858629842,
117
+ "scr_dir1_threshold_100": 0.48718003580863245,
118
+ "scr_metric_threshold_100": 0.2616580342884179,
119
+ "scr_dir2_threshold_100": 0.2616580342884179,
120
+ "scr_dir1_threshold_500": 0.39743608357792887,
121
+ "scr_metric_threshold_500": 0.3108808114137016,
122
+ "scr_dir2_threshold_500": 0.3108808114137016
123
  },
124
  {
125
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
 
132
  "scr_dir1_threshold_10": 0.12121234014557077,
133
  "scr_metric_threshold_10": 0.30275238555271,
134
  "scr_dir2_threshold_10": 0.30275238555271,
135
+ "scr_dir1_threshold_20": 0.2045454853329709,
136
+ "scr_metric_threshold_20": 0.35474002715930114,
137
+ "scr_dir2_threshold_20": 0.35474002715930114,
138
+ "scr_dir1_threshold_50": -0.4469697140738727,
139
+ "scr_metric_threshold_50": 0.4311927307857058,
140
+ "scr_dir2_threshold_50": 0.4311927307857058,
141
+ "scr_dir1_threshold_100": -0.5833329194122803,
142
+ "scr_metric_threshold_100": 0.48623861906801674,
143
+ "scr_dir2_threshold_100": 0.48623861906801674,
144
+ "scr_dir1_threshold_500": 0.2878790820706107,
145
+ "scr_metric_threshold_500": 0.08562689682199721,
146
+ "scr_dir2_threshold_500": 0.08562689682199721
147
  },
148
  {
149
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
 
156
  "scr_dir1_threshold_10": 0.3437501746229541,
157
  "scr_metric_threshold_10": 0.1482412214577287,
158
  "scr_dir2_threshold_10": 0.1482412214577287,
159
+ "scr_dir1_threshold_20": -0.031249592546440393,
160
+ "scr_metric_threshold_20": 0.18341709182393628,
161
+ "scr_dir2_threshold_20": 0.18341709182393628,
162
+ "scr_dir1_threshold_50": -0.6406250291038257,
163
+ "scr_metric_threshold_50": 0.28140711269005153,
164
+ "scr_dir2_threshold_50": 0.28140711269005153,
165
+ "scr_dir1_threshold_100": 0.06250011641530274,
166
+ "scr_metric_threshold_100": 0.06783921065491268,
167
+ "scr_dir2_threshold_100": 0.06783921065491268,
168
+ "scr_dir1_threshold_500": 0.04687532014208255,
169
+ "scr_metric_threshold_500": 0.10301508102112025,
170
+ "scr_dir2_threshold_500": 0.10301508102112025
171
  },
172
  {
173
  "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
 
180
  "scr_dir1_threshold_10": 0.22818779864829455,
181
  "scr_metric_threshold_10": 0.30573252397568285,
182
  "scr_dir2_threshold_10": 0.30573252397568285,
183
+ "scr_dir1_threshold_20": 0.07382535837795348,
184
+ "scr_metric_threshold_20": 0.39171981413981577,
185
+ "scr_dir2_threshold_20": 0.39171981413981577,
186
+ "scr_dir1_threshold_50": 0.14093939327267527,
187
+ "scr_metric_threshold_50": 0.5414013089483273,
188
+ "scr_dir2_threshold_50": 0.5414013089483273,
189
+ "scr_dir1_threshold_100": 0.08724800534441686,
190
+ "scr_metric_threshold_100": 0.5987261690577493,
191
+ "scr_dir2_threshold_100": 0.5987261690577493,
192
+ "scr_dir1_threshold_500": -0.07382575840915591,
193
+ "scr_metric_threshold_500": 0.13694280573859832,
194
+ "scr_dir2_threshold_500": 0.13694280573859832
195
  },
196
  {
197
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
 
204
  "scr_dir1_threshold_10": 0.09473687842942387,
205
  "scr_metric_threshold_10": 0.595238157185469,
206
  "scr_dir2_threshold_10": 0.595238157185469,
207
+ "scr_dir1_threshold_20": 0.16842108235134678,
208
+ "scr_metric_threshold_20": 0.6587301042915575,
209
+ "scr_dir2_threshold_20": 0.6587301042915575,
210
+ "scr_dir1_threshold_50": 0.09473687842942387,
211
+ "scr_metric_threshold_50": 0.6587301042915575,
212
+ "scr_dir2_threshold_50": 0.6587301042915575,
213
+ "scr_dir1_threshold_100": 0.0631580235225028,
214
+ "scr_metric_threshold_100": 0.7063492420158761,
215
+ "scr_dir2_threshold_100": 0.7063492420158761,
216
+ "scr_dir1_threshold_500": 0.07894729412163301,
217
+ "scr_metric_threshold_500": 0.7023808031440972,
218
+ "scr_dir2_threshold_500": 0.7023808031440972
219
  },
220
  {
221
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
 
228
  "scr_dir1_threshold_10": 0.2896173902280937,
229
  "scr_metric_threshold_10": 0.17557251734733875,
230
  "scr_dir2_threshold_10": 0.17557251734733875,
231
+ "scr_dir1_threshold_20": 0.3060109503196829,
232
+ "scr_metric_threshold_20": 0.22519087703221677,
233
+ "scr_dir2_threshold_20": 0.22519087703221677,
234
+ "scr_dir1_threshold_50": 0.3606557163469838,
235
+ "scr_metric_threshold_50": 0.28625946991455836,
236
+ "scr_dir2_threshold_50": 0.28625946991455836,
237
+ "scr_dir1_threshold_100": 0.4644808369405549,
238
+ "scr_metric_threshold_100": 0.297709930610645,
239
+ "scr_dir2_threshold_100": 0.297709930610645,
240
+ "scr_dir1_threshold_500": 0.502732368584764,
241
+ "scr_metric_threshold_500": 0.41984734387395123,
242
+ "scr_dir2_threshold_500": 0.41984734387395123
243
  },
244
  {
245
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
 
252
  "scr_dir1_threshold_10": 0.12558142242720055,
253
  "scr_metric_threshold_10": 0.5236051041022068,
254
  "scr_dir2_threshold_10": 0.5236051041022068,
255
+ "scr_dir1_threshold_20": 0.21860457508548414,
256
+ "scr_metric_threshold_20": 0.5450643129054811,
257
+ "scr_dir2_threshold_20": 0.5450643129054811,
258
+ "scr_dir1_threshold_50": 0.3441859975126847,
259
+ "scr_metric_threshold_50": 0.5922747769238424,
260
+ "scr_dir2_threshold_50": 0.5922747769238424,
261
+ "scr_dir1_threshold_100": -0.013953583791087694,
262
+ "scr_metric_threshold_100": 0.6437767757261215,
263
+ "scr_dir2_threshold_100": 0.6437767757261215,
264
+ "scr_dir1_threshold_500": 0.18139520312982554,
265
+ "scr_metric_threshold_500": 0.42918468769337936,
266
+ "scr_dir2_threshold_500": 0.42918468769337936
267
  },
268
  {
269
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
 
276
  "scr_dir1_threshold_10": 0.13953476767076942,
277
  "scr_metric_threshold_10": 0.13953476767076942,
278
  "scr_dir2_threshold_10": 0.11764715401326675,
279
+ "scr_dir1_threshold_20": 0.14418587114694043,
280
+ "scr_metric_threshold_20": 0.14418587114694043,
281
+ "scr_dir2_threshold_20": 0.15837114811753172,
282
+ "scr_dir1_threshold_50": 0.16744166575873534,
283
+ "scr_metric_threshold_50": 0.16744166575873534,
284
+ "scr_dir2_threshold_50": 0.21266962712038115,
285
+ "scr_dir1_threshold_100": 0.17674414994201715,
286
+ "scr_metric_threshold_100": 0.17674414994201715,
287
+ "scr_dir2_threshold_100": 0.20361997052132483,
288
+ "scr_dir1_threshold_500": 0.2558137407297439,
289
+ "scr_metric_threshold_500": 0.2558137407297439,
290
+ "scr_dir2_threshold_500": 0.30769236993175136
291
  }
292
  ],
293
  "sae_bench_commit_hash": "Unknown",
gemma-2-2b-btk/k-100/seed-0/snapshots/step-6103-tokens-25000000/tpp/saebench_tpp_custom_sae_eval_results.json CHANGED
@@ -24,7 +24,11 @@
24
  "n_values": [
25
  2,
26
  5,
27
- 10
 
 
 
 
28
  ],
29
  "column1_vals_lookup": {
30
  "LabHC/bias_in_bios_class_set1": [
@@ -65,81 +69,81 @@
65
  ]
66
  }
67
  },
68
- "eval_id": "16eef46d-fb0b-4386-a598-d932973537b3",
69
- "datetime_epoch_millis": 1776886817024,
70
  "eval_result_metrics": {
71
  "tpp_metrics": {
72
- "tpp_threshold_2_total_metric": 0.012399998307228087,
73
- "tpp_threshold_2_intended_diff_only": 0.0125,
74
- "tpp_threshold_2_unintended_diff_only": 0.00010000169277191162,
75
- "tpp_threshold_5_total_metric": 0.01602499634027481,
76
- "tpp_threshold_5_intended_diff_only": 0.018399995565414426,
77
- "tpp_threshold_5_unintended_diff_only": 0.002374999225139618,
78
- "tpp_threshold_10_total_metric": 0.020849990844726562,
79
- "tpp_threshold_10_intended_diff_only": 0.02369999885559082,
80
  "tpp_threshold_10_unintended_diff_only": 0.0028500080108642576,
81
- "tpp_threshold_20_total_metric": null,
82
- "tpp_threshold_20_intended_diff_only": null,
83
- "tpp_threshold_20_unintended_diff_only": null,
84
- "tpp_threshold_50_total_metric": null,
85
- "tpp_threshold_50_intended_diff_only": null,
86
- "tpp_threshold_50_unintended_diff_only": null,
87
- "tpp_threshold_100_total_metric": null,
88
- "tpp_threshold_100_intended_diff_only": null,
89
- "tpp_threshold_100_unintended_diff_only": null,
90
- "tpp_threshold_500_total_metric": null,
91
- "tpp_threshold_500_intended_diff_only": null,
92
- "tpp_threshold_500_unintended_diff_only": null
93
  }
94
  },
95
  "eval_result_details": [
96
  {
97
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
98
- "tpp_threshold_2_total_metric": 0.01654999554157257,
99
- "tpp_threshold_2_intended_diff_only": 0.01640000343322754,
100
- "tpp_threshold_2_unintended_diff_only": -0.00014999210834503173,
101
- "tpp_threshold_5_total_metric": 0.017199993133544922,
102
- "tpp_threshold_5_intended_diff_only": 0.019200003147125243,
103
- "tpp_threshold_5_unintended_diff_only": 0.002000010013580322,
104
- "tpp_threshold_10_total_metric": 0.020099982619285583,
105
- "tpp_threshold_10_intended_diff_only": 0.02239999771118164,
106
- "tpp_threshold_10_unintended_diff_only": 0.002300015091896057,
107
- "tpp_threshold_20_total_metric": null,
108
- "tpp_threshold_20_intended_diff_only": null,
109
- "tpp_threshold_20_unintended_diff_only": null,
110
- "tpp_threshold_50_total_metric": null,
111
- "tpp_threshold_50_intended_diff_only": null,
112
- "tpp_threshold_50_unintended_diff_only": null,
113
- "tpp_threshold_100_total_metric": null,
114
- "tpp_threshold_100_intended_diff_only": null,
115
- "tpp_threshold_100_unintended_diff_only": null,
116
- "tpp_threshold_500_total_metric": null,
117
- "tpp_threshold_500_intended_diff_only": null,
118
- "tpp_threshold_500_unintended_diff_only": null
119
  },
120
  {
121
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
122
- "tpp_threshold_2_total_metric": 0.008250001072883605,
123
- "tpp_threshold_2_intended_diff_only": 0.008599996566772461,
124
- "tpp_threshold_2_unintended_diff_only": 0.00034999549388885497,
125
- "tpp_threshold_5_total_metric": 0.0148499995470047,
126
- "tpp_threshold_5_intended_diff_only": 0.017599987983703613,
127
- "tpp_threshold_5_unintended_diff_only": 0.0027499884366989137,
128
- "tpp_threshold_10_total_metric": 0.02159999907016754,
129
- "tpp_threshold_10_intended_diff_only": 0.025,
130
- "tpp_threshold_10_unintended_diff_only": 0.0034000009298324587,
131
- "tpp_threshold_20_total_metric": null,
132
- "tpp_threshold_20_intended_diff_only": null,
133
- "tpp_threshold_20_unintended_diff_only": null,
134
- "tpp_threshold_50_total_metric": null,
135
- "tpp_threshold_50_intended_diff_only": null,
136
- "tpp_threshold_50_unintended_diff_only": null,
137
- "tpp_threshold_100_total_metric": null,
138
- "tpp_threshold_100_intended_diff_only": null,
139
- "tpp_threshold_100_unintended_diff_only": null,
140
- "tpp_threshold_500_total_metric": null,
141
- "tpp_threshold_500_intended_diff_only": null,
142
- "tpp_threshold_500_unintended_diff_only": null
143
  }
144
  ],
145
  "sae_bench_commit_hash": "Unknown",
@@ -189,108 +193,228 @@
189
  "tpp_threshold_5_unintended_diff_only": 0.0017500072717666626,
190
  "tpp_threshold_10_total_metric": 0.003999948501586914,
191
  "tpp_threshold_10_intended_diff_only": 0.006999969482421875,
192
- "tpp_threshold_10_unintended_diff_only": 0.003000020980834961
 
 
 
 
 
 
 
 
 
 
 
 
193
  },
194
  "1": {
195
- "tpp_threshold_2_total_metric": -0.0005000084638595581,
196
- "tpp_threshold_2_intended_diff_only": 0.0,
197
- "tpp_threshold_2_unintended_diff_only": 0.0005000084638595581,
198
- "tpp_threshold_5_total_metric": 0.0032499879598617554,
199
- "tpp_threshold_5_intended_diff_only": 0.004999995231628418,
200
- "tpp_threshold_5_unintended_diff_only": 0.0017500072717666626,
201
- "tpp_threshold_10_total_metric": -0.0022500306367874146,
202
- "tpp_threshold_10_intended_diff_only": 0.0009999871253967285,
203
- "tpp_threshold_10_unintended_diff_only": 0.003250017762184143
 
 
 
 
 
 
 
 
 
 
 
 
204
  },
205
  "2": {
206
- "tpp_threshold_2_total_metric": 0.06250004470348358,
207
- "tpp_threshold_2_intended_diff_only": 0.061000049114227295,
208
- "tpp_threshold_2_unintended_diff_only": -0.0014999955892562866,
209
- "tpp_threshold_5_total_metric": 0.06450001895427704,
210
- "tpp_threshold_5_intended_diff_only": 0.06700003147125244,
211
- "tpp_threshold_5_unintended_diff_only": 0.002500012516975403,
212
- "tpp_threshold_10_total_metric": 0.07500001788139343,
213
- "tpp_threshold_10_intended_diff_only": 0.07600003480911255,
214
- "tpp_threshold_10_unintended_diff_only": 0.0010000169277191162
 
 
 
 
 
 
 
 
 
 
 
 
215
  },
216
  "6": {
217
- "tpp_threshold_2_total_metric": 0.010500013828277588,
218
  "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
219
- "tpp_threshold_2_unintended_diff_only": -0.002499997615814209,
220
- "tpp_threshold_5_total_metric": 0.002249985933303833,
221
- "tpp_threshold_5_intended_diff_only": 0.004999995231628418,
222
- "tpp_threshold_5_unintended_diff_only": 0.002750009298324585,
223
- "tpp_threshold_10_total_metric": 0.007749974727630615,
224
- "tpp_threshold_10_intended_diff_only": 0.009999990463256836,
225
- "tpp_threshold_10_unintended_diff_only": 0.0022500157356262207
 
 
 
 
 
 
 
 
 
 
 
 
226
  },
227
  "9": {
228
- "tpp_threshold_2_total_metric": 0.0034999847412109375,
229
- "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
230
- "tpp_threshold_2_unintended_diff_only": 0.0015000104904174805,
231
- "tpp_threshold_5_total_metric": 0.006750002503395081,
232
- "tpp_threshold_5_intended_diff_only": 0.008000016212463379,
233
- "tpp_threshold_5_unintended_diff_only": 0.0012500137090682983,
234
- "tpp_threshold_10_total_metric": 0.01600000262260437,
235
- "tpp_threshold_10_intended_diff_only": 0.018000006675720215,
236
- "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447
 
 
 
 
 
 
 
 
 
 
 
 
237
  }
238
  },
239
  "canrager/amazon_reviews_mcauley_1and5": {
240
  "1": {
241
- "tpp_threshold_2_total_metric": 0.008500009775161743,
242
- "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
243
- "tpp_threshold_2_unintended_diff_only": -0.004500001668930054,
244
- "tpp_threshold_5_total_metric": 0.007249996066093445,
245
- "tpp_threshold_5_intended_diff_only": 0.004999995231628418,
246
- "tpp_threshold_5_unintended_diff_only": -0.002250000834465027,
247
- "tpp_threshold_10_total_metric": 0.002249971032142639,
248
  "tpp_threshold_10_intended_diff_only": 0.006999969482421875,
249
- "tpp_threshold_10_unintended_diff_only": 0.004749998450279236
 
 
 
 
 
 
 
 
 
 
 
 
250
  },
251
  "2": {
252
- "tpp_threshold_2_total_metric": 0.011250019073486328,
253
- "tpp_threshold_2_intended_diff_only": 0.017000019550323486,
254
- "tpp_threshold_2_unintended_diff_only": 0.005750000476837158,
255
- "tpp_threshold_5_total_metric": 0.01775003969669342,
256
- "tpp_threshold_5_intended_diff_only": 0.026000022888183594,
257
- "tpp_threshold_5_unintended_diff_only": 0.008249983191490173,
258
- "tpp_threshold_10_total_metric": 0.03500005602836609,
259
- "tpp_threshold_10_intended_diff_only": 0.03800004720687866,
260
- "tpp_threshold_10_unintended_diff_only": 0.0029999911785125732
 
 
 
 
 
 
 
 
 
 
 
 
261
  },
262
  "3": {
263
- "tpp_threshold_2_total_metric": 0.011000007390975952,
264
- "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
265
- "tpp_threshold_2_unintended_diff_only": -0.006999999284744263,
266
- "tpp_threshold_5_total_metric": 0.015499994158744812,
267
- "tpp_threshold_5_intended_diff_only": 0.015999972820281982,
268
- "tpp_threshold_5_unintended_diff_only": 0.0004999786615371704,
269
- "tpp_threshold_10_total_metric": 0.011749982833862305,
270
- "tpp_threshold_10_intended_diff_only": 0.014999985694885254,
271
- "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492
 
 
 
 
 
 
 
 
 
 
 
 
272
  },
273
  "5": {
274
- "tpp_threshold_2_total_metric": -0.004500031471252441,
275
- "tpp_threshold_2_intended_diff_only": 0.0029999613761901855,
276
- "tpp_threshold_2_unintended_diff_only": 0.007499992847442627,
277
- "tpp_threshold_5_total_metric": 0.0007499754428863525,
278
- "tpp_threshold_5_intended_diff_only": 0.0029999613761901855,
279
- "tpp_threshold_5_unintended_diff_only": 0.002249985933303833,
280
- "tpp_threshold_10_total_metric": 0.0009999722242355347,
281
- "tpp_threshold_10_intended_diff_only": 0.0059999823570251465,
282
- "tpp_threshold_10_unintended_diff_only": 0.005000010132789612
 
 
 
 
 
 
 
 
 
 
 
 
283
  },
284
  "6": {
285
- "tpp_threshold_2_total_metric": 0.015000000596046448,
286
- "tpp_threshold_2_intended_diff_only": 0.014999985694885254,
287
- "tpp_threshold_2_unintended_diff_only": -1.4901161193847656e-08,
288
- "tpp_threshold_5_total_metric": 0.03299999237060547,
289
- "tpp_threshold_5_intended_diff_only": 0.03799998760223389,
290
- "tpp_threshold_5_unintended_diff_only": 0.004999995231628418,
291
- "tpp_threshold_10_total_metric": 0.05800001323223114,
292
- "tpp_threshold_10_intended_diff_only": 0.05900001525878906,
293
- "tpp_threshold_10_unintended_diff_only": 0.0010000020265579224
 
 
 
 
 
 
 
 
 
 
 
 
294
  }
295
  }
296
  }
 
24
  "n_values": [
25
  2,
26
  5,
27
+ 10,
28
+ 20,
29
+ 50,
30
+ 100,
31
+ 500
32
  ],
33
  "column1_vals_lookup": {
34
  "LabHC/bias_in_bios_class_set1": [
 
69
  ]
70
  }
71
  },
72
+ "eval_id": "63f7ee90-d531-42e1-b7b1-985ab429c5c9",
73
+ "datetime_epoch_millis": 1777318009874,
74
  "eval_result_metrics": {
75
  "tpp_metrics": {
76
+ "tpp_threshold_2_total_metric": 0.011625003814697266,
77
+ "tpp_threshold_2_intended_diff_only": 0.0127000093460083,
78
+ "tpp_threshold_2_unintended_diff_only": 0.001075005531311035,
79
+ "tpp_threshold_5_total_metric": 0.01549999862909317,
80
+ "tpp_threshold_5_intended_diff_only": 0.016700005531311034,
81
+ "tpp_threshold_5_unintended_diff_only": 0.001200006902217865,
82
+ "tpp_threshold_10_total_metric": 0.02384999394416809,
83
+ "tpp_threshold_10_intended_diff_only": 0.02670000195503235,
84
  "tpp_threshold_10_unintended_diff_only": 0.0028500080108642576,
85
+ "tpp_threshold_20_total_metric": 0.044850008189678194,
86
+ "tpp_threshold_20_intended_diff_only": 0.048200011253356934,
87
+ "tpp_threshold_20_unintended_diff_only": 0.0033500030636787414,
88
+ "tpp_threshold_50_total_metric": 0.09422501623630525,
89
+ "tpp_threshold_50_intended_diff_only": 0.0992000162601471,
90
+ "tpp_threshold_50_unintended_diff_only": 0.0049750000238418576,
91
+ "tpp_threshold_100_total_metric": 0.18182501643896104,
92
+ "tpp_threshold_100_intended_diff_only": 0.1898000180721283,
93
+ "tpp_threshold_100_unintended_diff_only": 0.007975001633167268,
94
+ "tpp_threshold_500_total_metric": 0.3744000270962715,
95
+ "tpp_threshold_500_intended_diff_only": 0.3871000289916992,
96
+ "tpp_threshold_500_unintended_diff_only": 0.012700001895427703
97
  }
98
  },
99
  "eval_result_details": [
100
  {
101
  "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
102
+ "tpp_threshold_2_total_metric": 0.015200001001358033,
103
+ "tpp_threshold_2_intended_diff_only": 0.01660001277923584,
104
+ "tpp_threshold_2_unintended_diff_only": 0.0014000117778778076,
105
+ "tpp_threshold_5_total_metric": 0.01814999580383301,
106
+ "tpp_threshold_5_intended_diff_only": 0.018800008296966552,
107
+ "tpp_threshold_5_unintended_diff_only": 0.0006500124931335449,
108
+ "tpp_threshold_10_total_metric": 0.02024998962879181,
109
+ "tpp_threshold_10_intended_diff_only": 0.022600007057189942,
110
+ "tpp_threshold_10_unintended_diff_only": 0.0023500174283981323,
111
+ "tpp_threshold_20_total_metric": 0.044200006127357486,
112
+ "tpp_threshold_20_intended_diff_only": 0.04660001993179321,
113
+ "tpp_threshold_20_unintended_diff_only": 0.00240001380443573,
114
+ "tpp_threshold_50_total_metric": 0.09805001616477967,
115
+ "tpp_threshold_50_intended_diff_only": 0.10240002870559692,
116
+ "tpp_threshold_50_unintended_diff_only": 0.004350012540817261,
117
+ "tpp_threshold_100_total_metric": 0.18940001726150513,
118
+ "tpp_threshold_100_intended_diff_only": 0.19420002698898314,
119
+ "tpp_threshold_100_unintended_diff_only": 0.0048000097274780275,
120
+ "tpp_threshold_500_total_metric": 0.4010000228881836,
121
+ "tpp_threshold_500_intended_diff_only": 0.40760003328323363,
122
+ "tpp_threshold_500_unintended_diff_only": 0.006600010395050049
123
  },
124
  {
125
  "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
126
+ "tpp_threshold_2_total_metric": 0.0080500066280365,
127
+ "tpp_threshold_2_intended_diff_only": 0.008800005912780762,
128
+ "tpp_threshold_2_unintended_diff_only": 0.0007499992847442627,
129
+ "tpp_threshold_5_total_metric": 0.012850001454353333,
130
+ "tpp_threshold_5_intended_diff_only": 0.014600002765655517,
131
+ "tpp_threshold_5_unintended_diff_only": 0.0017500013113021851,
132
+ "tpp_threshold_10_total_metric": 0.027449998259544372,
133
+ "tpp_threshold_10_intended_diff_only": 0.030799996852874757,
134
+ "tpp_threshold_10_unintended_diff_only": 0.0033499985933303834,
135
+ "tpp_threshold_20_total_metric": 0.0455000102519989,
136
+ "tpp_threshold_20_intended_diff_only": 0.049800002574920656,
137
+ "tpp_threshold_20_unintended_diff_only": 0.004299992322921753,
138
+ "tpp_threshold_50_total_metric": 0.09040001630783082,
139
+ "tpp_threshold_50_intended_diff_only": 0.09600000381469727,
140
+ "tpp_threshold_50_unintended_diff_only": 0.005599987506866455,
141
+ "tpp_threshold_100_total_metric": 0.17425001561641693,
142
+ "tpp_threshold_100_intended_diff_only": 0.18540000915527344,
143
+ "tpp_threshold_100_unintended_diff_only": 0.011149993538856507,
144
+ "tpp_threshold_500_total_metric": 0.34780003130435944,
145
+ "tpp_threshold_500_intended_diff_only": 0.3666000247001648,
146
+ "tpp_threshold_500_unintended_diff_only": 0.018799993395805358
147
  }
148
  ],
149
  "sae_bench_commit_hash": "Unknown",
 
193
  "tpp_threshold_5_unintended_diff_only": 0.0017500072717666626,
194
  "tpp_threshold_10_total_metric": 0.003999948501586914,
195
  "tpp_threshold_10_intended_diff_only": 0.006999969482421875,
196
+ "tpp_threshold_10_unintended_diff_only": 0.003000020980834961,
197
+ "tpp_threshold_20_total_metric": 0.033749982714653015,
198
+ "tpp_threshold_20_intended_diff_only": 0.03600001335144043,
199
+ "tpp_threshold_20_unintended_diff_only": 0.0022500306367874146,
200
+ "tpp_threshold_50_total_metric": 0.07224996387958527,
201
+ "tpp_threshold_50_intended_diff_only": 0.07499998807907104,
202
+ "tpp_threshold_50_unintended_diff_only": 0.002750024199485779,
203
+ "tpp_threshold_100_total_metric": 0.1887499988079071,
204
+ "tpp_threshold_100_intended_diff_only": 0.19300001859664917,
205
+ "tpp_threshold_100_unintended_diff_only": 0.004250019788742065,
206
+ "tpp_threshold_500_total_metric": 0.416499987244606,
207
+ "tpp_threshold_500_intended_diff_only": 0.42100000381469727,
208
+ "tpp_threshold_500_unintended_diff_only": 0.0045000165700912476
209
  },
210
  "1": {
211
+ "tpp_threshold_2_total_metric": 0.0034999847412109375,
212
+ "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
213
+ "tpp_threshold_2_unintended_diff_only": 0.000500023365020752,
214
+ "tpp_threshold_5_total_metric": -0.0005000084638595581,
215
+ "tpp_threshold_5_intended_diff_only": 0.0,
216
+ "tpp_threshold_5_unintended_diff_only": 0.0005000084638595581,
217
+ "tpp_threshold_10_total_metric": 0.00449998676776886,
218
+ "tpp_threshold_10_intended_diff_only": 0.0040000081062316895,
219
+ "tpp_threshold_10_unintended_diff_only": -0.0004999786615371704,
220
+ "tpp_threshold_20_total_metric": 0.009499981999397278,
221
+ "tpp_threshold_20_intended_diff_only": 0.013999998569488525,
222
+ "tpp_threshold_20_unintended_diff_only": 0.0045000165700912476,
223
+ "tpp_threshold_50_total_metric": 0.04450002312660217,
224
+ "tpp_threshold_50_intended_diff_only": 0.053000032901763916,
225
+ "tpp_threshold_50_unintended_diff_only": 0.008500009775161743,
226
+ "tpp_threshold_100_total_metric": 0.12999998033046722,
227
+ "tpp_threshold_100_intended_diff_only": 0.13499999046325684,
228
+ "tpp_threshold_100_unintended_diff_only": 0.005000010132789612,
229
+ "tpp_threshold_500_total_metric": 0.41425004601478577,
230
+ "tpp_threshold_500_intended_diff_only": 0.4180000424385071,
231
+ "tpp_threshold_500_unintended_diff_only": 0.0037499964237213135
232
  },
233
  "2": {
234
+ "tpp_threshold_2_total_metric": 0.05150005221366882,
235
+ "tpp_threshold_2_intended_diff_only": 0.05600005388259888,
236
+ "tpp_threshold_2_unintended_diff_only": 0.004500001668930054,
237
+ "tpp_threshold_5_total_metric": 0.061250001192092896,
238
+ "tpp_threshold_5_intended_diff_only": 0.06400001049041748,
239
+ "tpp_threshold_5_unintended_diff_only": 0.002750009298324585,
240
+ "tpp_threshold_10_total_metric": 0.0727500170469284,
241
+ "tpp_threshold_10_intended_diff_only": 0.07700002193450928,
242
+ "tpp_threshold_10_unintended_diff_only": 0.004250004887580872,
243
+ "tpp_threshold_20_total_metric": 0.1120000034570694,
244
+ "tpp_threshold_20_intended_diff_only": 0.11400002241134644,
245
+ "tpp_threshold_20_unintended_diff_only": 0.0020000189542770386,
246
+ "tpp_threshold_50_total_metric": 0.20750002562999725,
247
+ "tpp_threshold_50_intended_diff_only": 0.21000003814697266,
248
+ "tpp_threshold_50_unintended_diff_only": 0.002500012516975403,
249
+ "tpp_threshold_100_total_metric": 0.3072500675916672,
250
+ "tpp_threshold_100_intended_diff_only": 0.31000006198883057,
251
+ "tpp_threshold_100_unintended_diff_only": 0.002749994397163391,
252
+ "tpp_threshold_500_total_metric": 0.42875005304813385,
253
+ "tpp_threshold_500_intended_diff_only": 0.440000057220459,
254
+ "tpp_threshold_500_unintended_diff_only": 0.011250004172325134
255
  },
256
  "6": {
257
+ "tpp_threshold_2_total_metric": 0.00849999487400055,
258
  "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
259
+ "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704,
260
+ "tpp_threshold_5_total_metric": 0.00875002145767212,
261
+ "tpp_threshold_5_intended_diff_only": 0.006000041961669922,
262
+ "tpp_threshold_5_unintended_diff_only": -0.0027499794960021973,
263
+ "tpp_threshold_10_total_metric": 0.010500028729438782,
264
+ "tpp_threshold_10_intended_diff_only": 0.01100003719329834,
265
+ "tpp_threshold_10_unintended_diff_only": 0.0005000084638595581,
266
+ "tpp_threshold_20_total_metric": 0.01500004529953003,
267
+ "tpp_threshold_20_intended_diff_only": 0.01500004529953003,
268
+ "tpp_threshold_20_unintended_diff_only": 0.0,
269
+ "tpp_threshold_50_total_metric": 0.013500049710273743,
270
+ "tpp_threshold_50_intended_diff_only": 0.01500004529953003,
271
+ "tpp_threshold_50_unintended_diff_only": 0.0014999955892562866,
272
+ "tpp_threshold_100_total_metric": 0.021250024437904358,
273
+ "tpp_threshold_100_intended_diff_only": 0.025000035762786865,
274
+ "tpp_threshold_100_unintended_diff_only": 0.0037500113248825073,
275
+ "tpp_threshold_500_total_metric": 0.2887500077486038,
276
+ "tpp_threshold_500_intended_diff_only": 0.2900000214576721,
277
+ "tpp_threshold_500_unintended_diff_only": 0.0012500137090682983
278
  },
279
  "9": {
280
+ "tpp_threshold_2_total_metric": 0.005750030279159546,
281
+ "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
282
+ "tpp_threshold_2_unintended_diff_only": 0.0012499988079071045,
283
+ "tpp_threshold_5_total_metric": 0.01199999451637268,
284
+ "tpp_threshold_5_intended_diff_only": 0.013000011444091797,
285
+ "tpp_threshold_5_unintended_diff_only": 0.0010000169277191162,
286
+ "tpp_threshold_10_total_metric": 0.009499967098236084,
287
+ "tpp_threshold_10_intended_diff_only": 0.013999998569488525,
288
+ "tpp_threshold_10_unintended_diff_only": 0.004500031471252441,
289
+ "tpp_threshold_20_total_metric": 0.050750017166137695,
290
+ "tpp_threshold_20_intended_diff_only": 0.054000020027160645,
291
+ "tpp_threshold_20_unintended_diff_only": 0.0032500028610229492,
292
+ "tpp_threshold_50_total_metric": 0.15250001847743988,
293
+ "tpp_threshold_50_intended_diff_only": 0.15900003910064697,
294
+ "tpp_threshold_50_unintended_diff_only": 0.006500020623207092,
295
+ "tpp_threshold_100_total_metric": 0.2997500151395798,
296
+ "tpp_threshold_100_intended_diff_only": 0.30800002813339233,
297
+ "tpp_threshold_100_unintended_diff_only": 0.008250012993812561,
298
+ "tpp_threshold_500_total_metric": 0.4567500203847885,
299
+ "tpp_threshold_500_intended_diff_only": 0.46900004148483276,
300
+ "tpp_threshold_500_unintended_diff_only": 0.01225002110004425
301
  }
302
  },
303
  "canrager/amazon_reviews_mcauley_1and5": {
304
  "1": {
305
+ "tpp_threshold_2_total_metric": 0.0034999698400497437,
306
+ "tpp_threshold_2_intended_diff_only": 0.006999969482421875,
307
+ "tpp_threshold_2_unintended_diff_only": 0.0034999996423721313,
308
+ "tpp_threshold_5_total_metric": 0.000250011682510376,
309
+ "tpp_threshold_5_intended_diff_only": 0.003000020980834961,
310
+ "tpp_threshold_5_unintended_diff_only": 0.002750009298324585,
311
+ "tpp_threshold_10_total_metric": 0.00424996018409729,
312
  "tpp_threshold_10_intended_diff_only": 0.006999969482421875,
313
+ "tpp_threshold_10_unintended_diff_only": 0.002750009298324585,
314
+ "tpp_threshold_20_total_metric": 0.015000030398368835,
315
+ "tpp_threshold_20_intended_diff_only": 0.017000019550323486,
316
+ "tpp_threshold_20_unintended_diff_only": 0.001999989151954651,
317
+ "tpp_threshold_50_total_metric": 0.038750022649765015,
318
+ "tpp_threshold_50_intended_diff_only": 0.03600001335144043,
319
+ "tpp_threshold_50_unintended_diff_only": -0.002750009298324585,
320
+ "tpp_threshold_100_total_metric": 0.11225003004074097,
321
+ "tpp_threshold_100_intended_diff_only": 0.11800003051757812,
322
+ "tpp_threshold_100_unintended_diff_only": 0.005750000476837158,
323
+ "tpp_threshold_500_total_metric": 0.3412500023841858,
324
+ "tpp_threshold_500_intended_diff_only": 0.34200000762939453,
325
+ "tpp_threshold_500_unintended_diff_only": 0.0007500052452087402
326
  },
327
  "2": {
328
+ "tpp_threshold_2_total_metric": 0.0205000638961792,
329
+ "tpp_threshold_2_intended_diff_only": 0.01900005340576172,
330
+ "tpp_threshold_2_unintended_diff_only": -0.0015000104904174805,
331
+ "tpp_threshold_5_total_metric": 0.014500051736831665,
332
+ "tpp_threshold_5_intended_diff_only": 0.024000048637390137,
333
+ "tpp_threshold_5_unintended_diff_only": 0.009499996900558472,
334
+ "tpp_threshold_10_total_metric": 0.045750051736831665,
335
+ "tpp_threshold_10_intended_diff_only": 0.05200004577636719,
336
+ "tpp_threshold_10_unintended_diff_only": 0.0062499940395355225,
337
+ "tpp_threshold_20_total_metric": 0.05725005269050598,
338
+ "tpp_threshold_20_intended_diff_only": 0.061000049114227295,
339
+ "tpp_threshold_20_unintended_diff_only": 0.0037499964237213135,
340
+ "tpp_threshold_50_total_metric": 0.1077500581741333,
341
+ "tpp_threshold_50_intended_diff_only": 0.11800003051757812,
342
+ "tpp_threshold_50_unintended_diff_only": 0.010249972343444824,
343
+ "tpp_threshold_100_total_metric": 0.3240000903606415,
344
+ "tpp_threshold_100_intended_diff_only": 0.3420000672340393,
345
+ "tpp_threshold_100_unintended_diff_only": 0.017999976873397827,
346
+ "tpp_threshold_500_total_metric": 0.39475010335445404,
347
+ "tpp_threshold_500_intended_diff_only": 0.43400007486343384,
348
+ "tpp_threshold_500_unintended_diff_only": 0.0392499715089798
349
  },
350
  "3": {
351
+ "tpp_threshold_2_total_metric": 0.004500031471252441,
352
+ "tpp_threshold_2_intended_diff_only": 0.003000020980834961,
353
+ "tpp_threshold_2_unintended_diff_only": -0.0015000104904174805,
354
+ "tpp_threshold_5_total_metric": 0.006499990820884705,
355
+ "tpp_threshold_5_intended_diff_only": 0.004999995231628418,
356
+ "tpp_threshold_5_unintended_diff_only": -0.0014999955892562866,
357
+ "tpp_threshold_10_total_metric": 0.014749988913536072,
358
+ "tpp_threshold_10_intended_diff_only": 0.013999998569488525,
359
+ "tpp_threshold_10_unintended_diff_only": -0.0007499903440475464,
360
+ "tpp_threshold_20_total_metric": 0.007499992847442627,
361
+ "tpp_threshold_20_intended_diff_only": 0.010999977588653564,
362
+ "tpp_threshold_20_unintended_diff_only": 0.0034999847412109375,
363
+ "tpp_threshold_50_total_metric": 0.03800000250339508,
364
+ "tpp_threshold_50_intended_diff_only": 0.042999982833862305,
365
+ "tpp_threshold_50_unintended_diff_only": 0.004999980330467224,
366
+ "tpp_threshold_100_total_metric": 0.08099998533725739,
367
+ "tpp_threshold_100_intended_diff_only": 0.08899998664855957,
368
+ "tpp_threshold_100_unintended_diff_only": 0.008000001311302185,
369
+ "tpp_threshold_500_total_metric": 0.3212500512599945,
370
+ "tpp_threshold_500_intended_diff_only": 0.33000004291534424,
371
+ "tpp_threshold_500_unintended_diff_only": 0.008749991655349731
372
  },
373
  "5": {
374
+ "tpp_threshold_2_total_metric": 0.006749972701072693,
375
+ "tpp_threshold_2_intended_diff_only": 0.0059999823570251465,
376
+ "tpp_threshold_2_unintended_diff_only": -0.0007499903440475464,
377
+ "tpp_threshold_5_total_metric": 0.0059999823570251465,
378
+ "tpp_threshold_5_intended_diff_only": 0.0059999823570251465,
379
+ "tpp_threshold_5_unintended_diff_only": 0.0,
380
+ "tpp_threshold_10_total_metric": 0.0009999871253967285,
381
+ "tpp_threshold_10_intended_diff_only": 0.006999969482421875,
382
+ "tpp_threshold_10_unintended_diff_only": 0.0059999823570251465,
383
+ "tpp_threshold_20_total_metric": 0.031499966979026794,
384
+ "tpp_threshold_20_intended_diff_only": 0.034999966621398926,
385
+ "tpp_threshold_20_unintended_diff_only": 0.0034999996423721313,
386
+ "tpp_threshold_50_total_metric": 0.054499998688697815,
387
+ "tpp_threshold_50_intended_diff_only": 0.06099998950958252,
388
+ "tpp_threshold_50_unintended_diff_only": 0.006499990820884705,
389
+ "tpp_threshold_100_total_metric": 0.09499998390674591,
390
+ "tpp_threshold_100_intended_diff_only": 0.10399997234344482,
391
+ "tpp_threshold_100_unintended_diff_only": 0.008999988436698914,
392
+ "tpp_threshold_500_total_metric": 0.3355000168085098,
393
+ "tpp_threshold_500_intended_diff_only": 0.35500001907348633,
394
+ "tpp_threshold_500_unintended_diff_only": 0.0195000022649765
395
  },
396
  "6": {
397
+ "tpp_threshold_2_total_metric": 0.004999995231628418,
398
+ "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
399
+ "tpp_threshold_2_unintended_diff_only": 0.0040000081062316895,
400
+ "tpp_threshold_5_total_metric": 0.03699997067451477,
401
+ "tpp_threshold_5_intended_diff_only": 0.034999966621398926,
402
+ "tpp_threshold_5_unintended_diff_only": -0.0020000040531158447,
403
+ "tpp_threshold_10_total_metric": 0.07150000333786011,
404
+ "tpp_threshold_10_intended_diff_only": 0.07400000095367432,
405
+ "tpp_threshold_10_unintended_diff_only": 0.002499997615814209,
406
+ "tpp_threshold_20_total_metric": 0.11625000834465027,
407
+ "tpp_threshold_20_intended_diff_only": 0.125,
408
+ "tpp_threshold_20_unintended_diff_only": 0.008749991655349731,
409
+ "tpp_threshold_50_total_metric": 0.21299999952316284,
410
+ "tpp_threshold_50_intended_diff_only": 0.22200000286102295,
411
+ "tpp_threshold_50_unintended_diff_only": 0.009000003337860107,
412
+ "tpp_threshold_100_total_metric": 0.2589999884366989,
413
+ "tpp_threshold_100_intended_diff_only": 0.27399998903274536,
414
+ "tpp_threshold_100_unintended_diff_only": 0.015000000596046448,
415
+ "tpp_threshold_500_total_metric": 0.346249982714653,
416
+ "tpp_threshold_500_intended_diff_only": 0.37199997901916504,
417
+ "tpp_threshold_500_unintended_diff_only": 0.025749996304512024
418
  }
419
  }
420
  }