diff --git a/gemma-2-2b-btk-mat/k-100/seed-0/snapshots/step-317382-tokens-1300000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk-mat/k-100/seed-0/snapshots/step-317382-tokens-1300000000/tpp/saebench_tpp_custom_sae_eval_results.json index c140acfc03051503cc4384a8d39ec12656d10e46..b822df75d07436da4dab2a8f64206aafcfae4ebf 100644 --- a/gemma-2-2b-btk-mat/k-100/seed-0/snapshots/step-317382-tokens-1300000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk-mat/k-100/seed-0/snapshots/step-317382-tokens-1300000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "52cd9c67-1bb8-4261-8090-dac372146614", - "datetime_epoch_millis": 1776881466884, + "eval_id": "8fce6eea-45ea-4c59-8c5a-6af79f7a8d8d", + "datetime_epoch_millis": 1777314582341, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.04565000385046005, - "tpp_threshold_2_intended_diff_only": 0.057700008153915405, - "tpp_threshold_2_unintended_diff_only": 0.012050004303455352, - "tpp_threshold_5_total_metric": 0.10462500154972076, - "tpp_threshold_5_intended_diff_only": 0.13940000534057617, - "tpp_threshold_5_unintended_diff_only": 0.03477500379085541, - "tpp_threshold_10_total_metric": 0.1483999952673912, - "tpp_threshold_10_intended_diff_only": 0.21450000405311584, - "tpp_threshold_10_unintended_diff_only": 0.06610000878572464, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.04417499750852585, + "tpp_threshold_2_intended_diff_only": 0.05649999976158142, + "tpp_threshold_2_unintended_diff_only": 0.012325002253055573, + "tpp_threshold_5_total_metric": 0.10682500898838043, + "tpp_threshold_5_intended_diff_only": 0.13920001387596131, + "tpp_threshold_5_unintended_diff_only": 0.03237500488758087, + "tpp_threshold_10_total_metric": 0.15090001076459886, + "tpp_threshold_10_intended_diff_only": 0.21670001745224, + "tpp_threshold_10_unintended_diff_only": 0.06580000668764115, + "tpp_threshold_20_total_metric": 0.1770750045776367, + "tpp_threshold_20_intended_diff_only": 0.2630000114440918, + "tpp_threshold_20_unintended_diff_only": 0.08592500686645507, + "tpp_threshold_50_total_metric": 0.21272501796483995, + "tpp_threshold_50_intended_diff_only": 0.3382000267505646, + "tpp_threshold_50_unintended_diff_only": 0.12547500878572465, + "tpp_threshold_100_total_metric": 0.2501500189304352, + "tpp_threshold_100_intended_diff_only": 0.40690003037452693, + "tpp_threshold_100_unintended_diff_only": 0.15675001144409179, + "tpp_threshold_500_total_metric": 0.23062503784894944, + "tpp_threshold_500_intended_diff_only": 0.4453000485897064, + "tpp_threshold_500_unintended_diff_only": 0.21467501074075698 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.07639999985694886, - "tpp_threshold_2_intended_diff_only": 0.09900001287460328, - "tpp_threshold_2_unintended_diff_only": 0.022600013017654418, - "tpp_threshold_5_total_metric": 0.17915000319480895, - "tpp_threshold_5_intended_diff_only": 0.24400001764297485, - "tpp_threshold_5_unintended_diff_only": 0.06485001444816589, - "tpp_threshold_10_total_metric": 0.2423500031232834, + "tpp_threshold_2_total_metric": 0.07420000433921814, + "tpp_threshold_2_intended_diff_only": 0.0974000096321106, + "tpp_threshold_2_unintended_diff_only": 0.023200005292892456, + "tpp_threshold_5_total_metric": 0.1817000150680542, + "tpp_threshold_5_intended_diff_only": 0.24420002698898316, + "tpp_threshold_5_unintended_diff_only": 0.06250001192092895, + "tpp_threshold_10_total_metric": 0.24305000603199006, "tpp_threshold_10_intended_diff_only": 0.3666000247001648, - "tpp_threshold_10_unintended_diff_only": 0.1242500215768814, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_10_unintended_diff_only": 0.12355001866817475, + "tpp_threshold_20_total_metric": 0.2477500170469284, + "tpp_threshold_20_intended_diff_only": 0.41080002784729003, + "tpp_threshold_20_unintended_diff_only": 0.16305001080036163, + "tpp_threshold_50_total_metric": 0.2190000206232071, + "tpp_threshold_50_intended_diff_only": 0.4552000403404236, + "tpp_threshold_50_unintended_diff_only": 0.2362000197172165, + "tpp_threshold_100_total_metric": 0.18680003583431243, + "tpp_threshold_100_intended_diff_only": 0.4652000546455383, + "tpp_threshold_100_unintended_diff_only": 0.2784000188112259, + "tpp_threshold_500_total_metric": 0.09815002977848053, + "tpp_threshold_500_intended_diff_only": 0.46780005693435667, + "tpp_threshold_500_unintended_diff_only": 0.36965002715587614 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.014900007843971252, - "tpp_threshold_2_intended_diff_only": 0.01640000343322754, - "tpp_threshold_2_unintended_diff_only": 0.0014999955892562866, - "tpp_threshold_5_total_metric": 0.03009999990463257, - "tpp_threshold_5_intended_diff_only": 0.03479999303817749, - "tpp_threshold_5_unintended_diff_only": 0.004699993133544922, - "tpp_threshold_10_total_metric": 0.05444998741149902, - "tpp_threshold_10_intended_diff_only": 0.062399983406066895, - "tpp_threshold_10_unintended_diff_only": 0.00794999599456787, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.014149990677833558, + "tpp_threshold_2_intended_diff_only": 0.015599989891052246, + "tpp_threshold_2_unintended_diff_only": 0.001449999213218689, + "tpp_threshold_5_total_metric": 0.031950002908706664, + "tpp_threshold_5_intended_diff_only": 0.034200000762939456, + "tpp_threshold_5_unintended_diff_only": 0.002249997854232788, + "tpp_threshold_10_total_metric": 0.05875001549720764, + "tpp_threshold_10_intended_diff_only": 0.06680001020431518, + "tpp_threshold_10_unintended_diff_only": 0.008049994707107544, + "tpp_threshold_20_total_metric": 0.10639999210834503, + "tpp_threshold_20_intended_diff_only": 0.11519999504089355, + "tpp_threshold_20_unintended_diff_only": 0.008800002932548522, + "tpp_threshold_50_total_metric": 0.20645001530647278, + "tpp_threshold_50_intended_diff_only": 0.22120001316070556, + "tpp_threshold_50_unintended_diff_only": 0.014749997854232788, + "tpp_threshold_100_total_metric": 0.3135000020265579, + "tpp_threshold_100_intended_diff_only": 0.3486000061035156, + "tpp_threshold_100_unintended_diff_only": 0.0351000040769577, + "tpp_threshold_500_total_metric": 0.36310004591941836, + "tpp_threshold_500_intended_diff_only": 0.42280004024505613, + "tpp_threshold_500_unintended_diff_only": 0.059699994325637815 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.10175003111362457, "tpp_threshold_10_total_metric": 0.21299993991851807, "tpp_threshold_10_intended_diff_only": 0.3489999771118164, - "tpp_threshold_10_unintended_diff_only": 0.13600003719329834 + "tpp_threshold_10_unintended_diff_only": 0.13600003719329834, + "tpp_threshold_20_total_metric": 0.20374999940395355, + "tpp_threshold_20_intended_diff_only": 0.3880000114440918, + "tpp_threshold_20_unintended_diff_only": 0.18425001204013824, + "tpp_threshold_50_total_metric": 0.17900000512599945, + "tpp_threshold_50_intended_diff_only": 0.4520000219345093, + "tpp_threshold_50_unintended_diff_only": 0.2730000168085098, + "tpp_threshold_100_total_metric": 0.14824999868869781, + "tpp_threshold_100_intended_diff_only": 0.45600003004074097, + "tpp_threshold_100_unintended_diff_only": 0.30775003135204315, + "tpp_threshold_500_total_metric": 0.07350000739097595, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.382500022649765 }, "1": { - "tpp_threshold_2_total_metric": 0.12025000154972076, - "tpp_threshold_2_intended_diff_only": 0.1600000262260437, - "tpp_threshold_2_unintended_diff_only": 0.03975002467632294, - "tpp_threshold_5_total_metric": 0.1772499680519104, + "tpp_threshold_2_total_metric": 0.12274999916553497, + "tpp_threshold_2_intended_diff_only": 0.15700000524520874, + "tpp_threshold_2_unintended_diff_only": 0.03425000607967377, + "tpp_threshold_5_total_metric": 0.17999999225139618, "tpp_threshold_5_intended_diff_only": 0.22699999809265137, - "tpp_threshold_5_unintended_diff_only": 0.04975003004074097, - "tpp_threshold_10_total_metric": 0.2952499985694885, + "tpp_threshold_5_unintended_diff_only": 0.04700000584125519, + "tpp_threshold_10_total_metric": 0.296999990940094, "tpp_threshold_10_intended_diff_only": 0.35100001096725464, - "tpp_threshold_10_unintended_diff_only": 0.05575001239776611 + "tpp_threshold_10_unintended_diff_only": 0.054000020027160645, + "tpp_threshold_20_total_metric": 0.32349999248981476, + "tpp_threshold_20_intended_diff_only": 0.42100000381469727, + "tpp_threshold_20_unintended_diff_only": 0.09750001132488251, + "tpp_threshold_50_total_metric": 0.2900000214576721, + "tpp_threshold_50_intended_diff_only": 0.4500000476837158, + "tpp_threshold_50_unintended_diff_only": 0.1600000262260437, + "tpp_threshold_100_total_metric": 0.25050003826618195, + "tpp_threshold_100_intended_diff_only": 0.4590000510215759, + "tpp_threshold_100_unintended_diff_only": 0.20850001275539398, + "tpp_threshold_500_total_metric": 0.08950002491474152, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.3695000261068344 }, "2": { - "tpp_threshold_2_total_metric": 0.02124999463558197, - "tpp_threshold_2_intended_diff_only": 0.018000006675720215, - "tpp_threshold_2_unintended_diff_only": -0.0032499879598617554, - "tpp_threshold_5_total_metric": 0.1820000559091568, + "tpp_threshold_2_total_metric": 0.010250017046928406, + "tpp_threshold_2_intended_diff_only": 0.013000011444091797, + "tpp_threshold_2_unintended_diff_only": 0.002749994397163391, + "tpp_threshold_5_total_metric": 0.19400005042552948, "tpp_threshold_5_intended_diff_only": 0.29600006341934204, - "tpp_threshold_5_unintended_diff_only": 0.11400000751018524, - "tpp_threshold_10_total_metric": 0.16375002264976501, + "tpp_threshold_5_unintended_diff_only": 0.10200001299381256, + "tpp_threshold_10_total_metric": 0.1627500206232071, "tpp_threshold_10_intended_diff_only": 0.3400000333786011, - "tpp_threshold_10_unintended_diff_only": 0.17625001072883606 + "tpp_threshold_10_unintended_diff_only": 0.17725001275539398, + "tpp_threshold_20_total_metric": 0.16425004601478577, + "tpp_threshold_20_intended_diff_only": 0.3800000548362732, + "tpp_threshold_20_unintended_diff_only": 0.21575000882148743, + "tpp_threshold_50_total_metric": 0.148499995470047, + "tpp_threshold_50_intended_diff_only": 0.42900002002716064, + "tpp_threshold_50_unintended_diff_only": 0.28050002455711365, + "tpp_threshold_100_total_metric": 0.15000003576278687, + "tpp_threshold_100_intended_diff_only": 0.4490000605583191, + "tpp_threshold_100_unintended_diff_only": 0.2990000247955322, + "tpp_threshold_500_total_metric": 0.058750033378601074, + "tpp_threshold_500_intended_diff_only": 0.4530000686645508, + "tpp_threshold_500_unintended_diff_only": 0.3942500352859497 }, "6": { - "tpp_threshold_2_total_metric": 0.04650002717971802, - "tpp_threshold_2_intended_diff_only": 0.04900002479553223, - "tpp_threshold_2_unintended_diff_only": 0.002499997615814209, - "tpp_threshold_5_total_metric": 0.1340000033378601, - "tpp_threshold_5_intended_diff_only": 0.15299999713897705, - "tpp_threshold_5_unintended_diff_only": 0.018999993801116943, - "tpp_threshold_10_total_metric": 0.343500018119812, + "tpp_threshold_2_total_metric": 0.042000025510787964, + "tpp_threshold_2_intended_diff_only": 0.0480000376701355, + "tpp_threshold_2_unintended_diff_only": 0.006000012159347534, + "tpp_threshold_5_total_metric": 0.137750044465065, + "tpp_threshold_5_intended_diff_only": 0.15400004386901855, + "tpp_threshold_5_unintended_diff_only": 0.016249999403953552, + "tpp_threshold_10_total_metric": 0.34550002217292786, "tpp_threshold_10_intended_diff_only": 0.437000036239624, - "tpp_threshold_10_unintended_diff_only": 0.09350001811981201 + "tpp_threshold_10_unintended_diff_only": 0.09150001406669617, + "tpp_threshold_20_total_metric": 0.3540000170469284, + "tpp_threshold_20_intended_diff_only": 0.4610000252723694, + "tpp_threshold_20_unintended_diff_only": 0.10700000822544098, + "tpp_threshold_50_total_metric": 0.2565000355243683, + "tpp_threshold_50_intended_diff_only": 0.48200005292892456, + "tpp_threshold_50_unintended_diff_only": 0.22550001740455627, + "tpp_threshold_100_total_metric": 0.2092500478029251, + "tpp_threshold_100_intended_diff_only": 0.48600006103515625, + "tpp_threshold_100_unintended_diff_only": 0.27675001323223114, + "tpp_threshold_500_total_metric": 0.16175003349781036, + "tpp_threshold_500_intended_diff_only": 0.49500006437301636, + "tpp_threshold_500_unintended_diff_only": 0.333250030875206 }, "9": { - "tpp_threshold_2_total_metric": 0.009750008583068848, - "tpp_threshold_2_intended_diff_only": 0.012000024318695068, - "tpp_threshold_2_unintended_diff_only": 0.0022500157356262207, - "tpp_threshold_5_total_metric": 0.19025000929832458, + "tpp_threshold_2_total_metric": 0.011750012636184692, + "tpp_threshold_2_intended_diff_only": 0.013000011444091797, + "tpp_threshold_2_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_5_total_metric": 0.18450000882148743, "tpp_threshold_5_intended_diff_only": 0.23000001907348633, - "tpp_threshold_5_unintended_diff_only": 0.03975000977516174, - "tpp_threshold_10_total_metric": 0.1962500363588333, + "tpp_threshold_5_unintended_diff_only": 0.0455000102519989, + "tpp_threshold_10_total_metric": 0.19700005650520325, "tpp_threshold_10_intended_diff_only": 0.35600006580352783, - "tpp_threshold_10_unintended_diff_only": 0.15975002944469452 + "tpp_threshold_10_unintended_diff_only": 0.15900000929832458, + "tpp_threshold_20_total_metric": 0.19325003027915955, + "tpp_threshold_20_intended_diff_only": 0.40400004386901855, + "tpp_threshold_20_unintended_diff_only": 0.210750013589859, + "tpp_threshold_50_total_metric": 0.2210000455379486, + "tpp_threshold_50_intended_diff_only": 0.4630000591278076, + "tpp_threshold_50_unintended_diff_only": 0.242000013589859, + "tpp_threshold_100_total_metric": 0.17600005865097046, + "tpp_threshold_100_intended_diff_only": 0.4760000705718994, + "tpp_threshold_100_unintended_diff_only": 0.30000001192092896, + "tpp_threshold_500_total_metric": 0.10725004971027374, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.36875002086162567 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.008000001311302185, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": -0.003999993205070496, - "tpp_threshold_5_total_metric": 0.012750014662742615, - "tpp_threshold_5_intended_diff_only": 0.013000011444091797, - "tpp_threshold_5_unintended_diff_only": 0.00024999678134918213, - "tpp_threshold_10_total_metric": 0.011250004172325134, + "tpp_threshold_2_total_metric": 0.005750000476837158, + "tpp_threshold_2_intended_diff_only": 0.009000003337860107, + "tpp_threshold_2_unintended_diff_only": 0.0032500028610229492, + "tpp_threshold_5_total_metric": 0.008000031113624573, + "tpp_threshold_5_intended_diff_only": 0.012000024318695068, + "tpp_threshold_5_unintended_diff_only": 0.003999993205070496, + "tpp_threshold_10_total_metric": 0.012999996542930603, "tpp_threshold_10_intended_diff_only": 0.018999993801116943, - "tpp_threshold_10_unintended_diff_only": 0.007749989628791809 + "tpp_threshold_10_unintended_diff_only": 0.00599999725818634, + "tpp_threshold_20_total_metric": 0.07224997878074646, + "tpp_threshold_20_intended_diff_only": 0.07599997520446777, + "tpp_threshold_20_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_50_total_metric": 0.19425000250339508, + "tpp_threshold_50_intended_diff_only": 0.19999998807907104, + "tpp_threshold_50_unintended_diff_only": 0.005749985575675964, + "tpp_threshold_100_total_metric": 0.3400000035762787, + "tpp_threshold_100_intended_diff_only": 0.35600000619888306, + "tpp_threshold_100_unintended_diff_only": 0.01600000262260437, + "tpp_threshold_500_total_metric": 0.42875005304813385, + "tpp_threshold_500_intended_diff_only": 0.45500004291534424, + "tpp_threshold_500_unintended_diff_only": 0.026249989867210388 }, "2": { - "tpp_threshold_2_total_metric": 0.006500020623207092, - "tpp_threshold_2_intended_diff_only": 0.013000011444091797, - "tpp_threshold_2_unintended_diff_only": 0.006499990820884705, - "tpp_threshold_5_total_metric": 0.02000005543231964, - "tpp_threshold_5_intended_diff_only": 0.030000030994415283, - "tpp_threshold_5_unintended_diff_only": 0.009999975562095642, - "tpp_threshold_10_total_metric": 0.07975001633167267, - "tpp_threshold_10_intended_diff_only": 0.0910000205039978, - "tpp_threshold_10_unintended_diff_only": 0.011250004172325134 + "tpp_threshold_2_total_metric": 0.016500040888786316, + "tpp_threshold_2_intended_diff_only": 0.01500004529953003, + "tpp_threshold_2_unintended_diff_only": -0.0014999955892562866, + "tpp_threshold_5_total_metric": 0.024500057101249695, + "tpp_threshold_5_intended_diff_only": 0.033000051975250244, + "tpp_threshold_5_unintended_diff_only": 0.00849999487400055, + "tpp_threshold_10_total_metric": 0.07975004613399506, + "tpp_threshold_10_intended_diff_only": 0.09400004148483276, + "tpp_threshold_10_unintended_diff_only": 0.014249995350837708, + "tpp_threshold_20_total_metric": 0.11650000512599945, + "tpp_threshold_20_intended_diff_only": 0.1290000081062317, + "tpp_threshold_20_unintended_diff_only": 0.012500002980232239, + "tpp_threshold_50_total_metric": 0.2482500672340393, + "tpp_threshold_50_intended_diff_only": 0.2640000581741333, + "tpp_threshold_50_unintended_diff_only": 0.015749990940093994, + "tpp_threshold_100_total_metric": 0.36225005984306335, + "tpp_threshold_100_intended_diff_only": 0.3850000500679016, + "tpp_threshold_100_unintended_diff_only": 0.022749990224838257, + "tpp_threshold_500_total_metric": 0.3890000879764557, + "tpp_threshold_500_intended_diff_only": 0.43900007009506226, + "tpp_threshold_500_unintended_diff_only": 0.04999998211860657 }, "3": { - "tpp_threshold_2_total_metric": 0.022750020027160645, - "tpp_threshold_2_intended_diff_only": 0.018000006675720215, - "tpp_threshold_2_unintended_diff_only": -0.00475001335144043, - "tpp_threshold_5_total_metric": 0.02399998903274536, - "tpp_threshold_5_intended_diff_only": 0.02499997615814209, - "tpp_threshold_5_unintended_diff_only": 0.0009999871253967285, - "tpp_threshold_10_total_metric": 0.02724999189376831, - "tpp_threshold_10_intended_diff_only": 0.0339999794960022, - "tpp_threshold_10_unintended_diff_only": 0.006749987602233887 + "tpp_threshold_2_total_metric": 0.014249980449676514, + "tpp_threshold_2_intended_diff_only": 0.015999972820281982, + "tpp_threshold_2_unintended_diff_only": 0.0017499923706054688, + "tpp_threshold_5_total_metric": 0.01449999213218689, + "tpp_threshold_5_intended_diff_only": 0.013999998569488525, + "tpp_threshold_5_unintended_diff_only": -0.0004999935626983643, + "tpp_threshold_10_total_metric": 0.03275004029273987, + "tpp_threshold_10_intended_diff_only": 0.0350000262260437, + "tpp_threshold_10_unintended_diff_only": 0.002249985933303833, + "tpp_threshold_20_total_metric": 0.05674998462200165, + "tpp_threshold_20_intended_diff_only": 0.06599998474121094, + "tpp_threshold_20_unintended_diff_only": 0.00925000011920929, + "tpp_threshold_50_total_metric": 0.14900000393390656, + "tpp_threshold_50_intended_diff_only": 0.16600000858306885, + "tpp_threshold_50_unintended_diff_only": 0.017000004649162292, + "tpp_threshold_100_total_metric": 0.2670000344514847, + "tpp_threshold_100_intended_diff_only": 0.3490000367164612, + "tpp_threshold_100_unintended_diff_only": 0.0820000022649765, + "tpp_threshold_500_total_metric": 0.3265000283718109, + "tpp_threshold_500_intended_diff_only": 0.4270000457763672, + "tpp_threshold_500_unintended_diff_only": 0.10050001740455627 }, "5": { - "tpp_threshold_2_total_metric": 0.0014999955892562866, - "tpp_threshold_2_intended_diff_only": 0.009999990463256836, - "tpp_threshold_2_unintended_diff_only": 0.00849999487400055, - "tpp_threshold_5_total_metric": 0.01349996030330658, - "tpp_threshold_5_intended_diff_only": 0.01699995994567871, - "tpp_threshold_5_unintended_diff_only": 0.0034999996423721313, - "tpp_threshold_10_total_metric": 0.030249953269958496, - "tpp_threshold_10_intended_diff_only": 0.035999953746795654, - "tpp_threshold_10_unintended_diff_only": 0.005750000476837158 + "tpp_threshold_2_total_metric": 0.008999943733215332, + "tpp_threshold_2_intended_diff_only": 0.008999943733215332, + "tpp_threshold_2_unintended_diff_only": 0.0, + "tpp_threshold_5_total_metric": 0.027249962091445923, + "tpp_threshold_5_intended_diff_only": 0.026999950408935547, + "tpp_threshold_5_unintended_diff_only": -0.000250011682510376, + "tpp_threshold_10_total_metric": 0.03299999237060547, + "tpp_threshold_10_intended_diff_only": 0.041999995708465576, + "tpp_threshold_10_unintended_diff_only": 0.009000003337860107, + "tpp_threshold_20_total_metric": 0.052249982953071594, + "tpp_threshold_20_intended_diff_only": 0.05699998140335083, + "tpp_threshold_20_unintended_diff_only": 0.004749998450279236, + "tpp_threshold_50_total_metric": 0.11225000023841858, + "tpp_threshold_50_intended_diff_only": 0.12999999523162842, + "tpp_threshold_50_unintended_diff_only": 0.01774999499320984, + "tpp_threshold_100_total_metric": 0.2544999420642853, + "tpp_threshold_100_intended_diff_only": 0.28099995851516724, + "tpp_threshold_100_unintended_diff_only": 0.026500016450881958, + "tpp_threshold_500_total_metric": 0.35075002908706665, + "tpp_threshold_500_intended_diff_only": 0.42000001668930054, + "tpp_threshold_500_unintended_diff_only": 0.06924998760223389 }, "6": { - "tpp_threshold_2_total_metric": 0.035750001668930054, - "tpp_threshold_2_intended_diff_only": 0.03700000047683716, - "tpp_threshold_2_unintended_diff_only": 0.0012499988079071045, - "tpp_threshold_5_total_metric": 0.08024998009204865, - "tpp_threshold_5_intended_diff_only": 0.08899998664855957, - "tpp_threshold_5_unintended_diff_only": 0.008750006556510925, - "tpp_threshold_10_total_metric": 0.12374997138977051, - "tpp_threshold_10_intended_diff_only": 0.13199996948242188, - "tpp_threshold_10_unintended_diff_only": 0.008249998092651367 + "tpp_threshold_2_total_metric": 0.025249987840652466, + "tpp_threshold_2_intended_diff_only": 0.02899998426437378, + "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_5_total_metric": 0.08549997210502625, + "tpp_threshold_5_intended_diff_only": 0.08499997854232788, + "tpp_threshold_5_unintended_diff_only": -0.0004999935626983643, + "tpp_threshold_10_total_metric": 0.1352500021457672, + "tpp_threshold_10_intended_diff_only": 0.14399999380111694, + "tpp_threshold_10_unintended_diff_only": 0.008749991655349731, + "tpp_threshold_20_total_metric": 0.234250009059906, + "tpp_threshold_20_intended_diff_only": 0.24800002574920654, + "tpp_threshold_20_unintended_diff_only": 0.013750016689300537, + "tpp_threshold_50_total_metric": 0.32850000262260437, + "tpp_threshold_50_intended_diff_only": 0.3460000157356262, + "tpp_threshold_50_unintended_diff_only": 0.01750001311302185, + "tpp_threshold_100_total_metric": 0.3437499701976776, + "tpp_threshold_100_intended_diff_only": 0.37199997901916504, + "tpp_threshold_100_unintended_diff_only": 0.028250008821487427, + "tpp_threshold_500_total_metric": 0.3205000311136246, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.05249999463558197 } } } diff --git a/gemma-2-2b-btk-mat/k-100/seed-0/snapshots/step-48828-tokens-200000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk-mat/k-100/seed-0/snapshots/step-48828-tokens-200000000/tpp/saebench_tpp_custom_sae_eval_results.json index b9747ad90749f22abe19e03176a518403505c48d..6d9d3aa6e5484c74065cf38b7404d1ed7aea87ae 100644 --- a/gemma-2-2b-btk-mat/k-100/seed-0/snapshots/step-48828-tokens-200000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk-mat/k-100/seed-0/snapshots/step-48828-tokens-200000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "03e860f7-ff02-427b-b6dc-682a08f89af5", - "datetime_epoch_millis": 1776882661358, + "eval_id": "7997e007-9853-4f5e-9dab-29873d03948e", + "datetime_epoch_millis": 1777314742336, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.013974991440773011, - "tpp_threshold_2_intended_diff_only": 0.014899992942810058, - "tpp_threshold_2_unintended_diff_only": 0.0009250015020370483, - "tpp_threshold_5_total_metric": 0.04457501173019409, - "tpp_threshold_5_intended_diff_only": 0.05280001759529114, - "tpp_threshold_5_unintended_diff_only": 0.008225005865097047, - "tpp_threshold_10_total_metric": 0.1080999955534935, - "tpp_threshold_10_intended_diff_only": 0.1331000030040741, - "tpp_threshold_10_unintended_diff_only": 0.025000007450580595, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.013749976456165313, + "tpp_threshold_2_intended_diff_only": 0.015399986505508423, + "tpp_threshold_2_unintended_diff_only": 0.0016500100493431092, + "tpp_threshold_5_total_metric": 0.04444999247789383, + "tpp_threshold_5_intended_diff_only": 0.0515999972820282, + "tpp_threshold_5_unintended_diff_only": 0.007150004804134368, + "tpp_threshold_10_total_metric": 0.10890000611543656, + "tpp_threshold_10_intended_diff_only": 0.1340000033378601, + "tpp_threshold_10_unintended_diff_only": 0.025099997222423554, + "tpp_threshold_20_total_metric": 0.189675010740757, + "tpp_threshold_20_intended_diff_only": 0.22810001969337462, + "tpp_threshold_20_unintended_diff_only": 0.03842500895261765, + "tpp_threshold_50_total_metric": 0.2488000199198723, + "tpp_threshold_50_intended_diff_only": 0.3137000262737274, + "tpp_threshold_50_unintended_diff_only": 0.06490000635385514, + "tpp_threshold_100_total_metric": 0.28297500461339953, + "tpp_threshold_100_intended_diff_only": 0.37920001745223997, + "tpp_threshold_100_unintended_diff_only": 0.09622501283884048, + "tpp_threshold_500_total_metric": 0.26877502501010897, + "tpp_threshold_500_intended_diff_only": 0.44490004181861875, + "tpp_threshold_500_unintended_diff_only": 0.1761250168085098 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.00914999544620514, - "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": -0.00014999210834503173, - "tpp_threshold_5_total_metric": 0.05970000922679901, - "tpp_threshold_5_intended_diff_only": 0.07140002250671387, - "tpp_threshold_5_unintended_diff_only": 0.011700013279914856, - "tpp_threshold_10_total_metric": 0.16279999315738677, - "tpp_threshold_10_intended_diff_only": 0.20660001039505005, - "tpp_threshold_10_unintended_diff_only": 0.04380001723766327, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.008649978041648864, + "tpp_threshold_2_intended_diff_only": 0.009399998188018798, + "tpp_threshold_2_unintended_diff_only": 0.0007500201463699341, + "tpp_threshold_5_total_metric": 0.05999999940395355, + "tpp_threshold_5_intended_diff_only": 0.07120001316070557, + "tpp_threshold_5_unintended_diff_only": 0.011200013756752013, + "tpp_threshold_10_total_metric": 0.16315000951290132, + "tpp_threshold_10_intended_diff_only": 0.2064000129699707, + "tpp_threshold_10_unintended_diff_only": 0.0432500034570694, + "tpp_threshold_20_total_metric": 0.28975001275539397, + "tpp_threshold_20_intended_diff_only": 0.3580000281333923, + "tpp_threshold_20_unintended_diff_only": 0.06825001537799835, + "tpp_threshold_50_total_metric": 0.3225500226020813, + "tpp_threshold_50_intended_diff_only": 0.4402000427246094, + "tpp_threshold_50_unintended_diff_only": 0.11765002012252808, + "tpp_threshold_100_total_metric": 0.28970001339912416, + "tpp_threshold_100_intended_diff_only": 0.4628000378608704, + "tpp_threshold_100_unintended_diff_only": 0.17310002446174622, + "tpp_threshold_500_total_metric": 0.15300002992153167, + "tpp_threshold_500_intended_diff_only": 0.46780005693435667, + "tpp_threshold_500_unintended_diff_only": 0.314800027012825 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.018799987435340882, - "tpp_threshold_2_intended_diff_only": 0.02079998254776001, - "tpp_threshold_2_unintended_diff_only": 0.0019999951124191283, - "tpp_threshold_5_total_metric": 0.029450014233589172, - "tpp_threshold_5_intended_diff_only": 0.03420001268386841, - "tpp_threshold_5_unintended_diff_only": 0.004749998450279236, - "tpp_threshold_10_total_metric": 0.05339999794960022, - "tpp_threshold_10_intended_diff_only": 0.059599995613098145, - "tpp_threshold_10_unintended_diff_only": 0.006199997663497925, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.01884997487068176, + "tpp_threshold_2_intended_diff_only": 0.021399974822998047, + "tpp_threshold_2_unintended_diff_only": 0.0025499999523162843, + "tpp_threshold_5_total_metric": 0.028899985551834106, + "tpp_threshold_5_intended_diff_only": 0.03199998140335083, + "tpp_threshold_5_unintended_diff_only": 0.003099995851516724, + "tpp_threshold_10_total_metric": 0.0546500027179718, + "tpp_threshold_10_intended_diff_only": 0.06159999370574951, + "tpp_threshold_10_unintended_diff_only": 0.00694999098777771, + "tpp_threshold_20_total_metric": 0.08960000872611999, + "tpp_threshold_20_intended_diff_only": 0.09820001125335694, + "tpp_threshold_20_unintended_diff_only": 0.008600002527236939, + "tpp_threshold_50_total_metric": 0.17505001723766328, + "tpp_threshold_50_intended_diff_only": 0.18720000982284546, + "tpp_threshold_50_unintended_diff_only": 0.01214999258518219, + "tpp_threshold_100_total_metric": 0.27624999582767484, + "tpp_threshold_100_intended_diff_only": 0.2955999970436096, + "tpp_threshold_100_unintended_diff_only": 0.019350001215934755, + "tpp_threshold_500_total_metric": 0.3845500200986862, + "tpp_threshold_500_intended_diff_only": 0.42200002670288084, + "tpp_threshold_500_unintended_diff_only": 0.03745000660419464 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.0020000189542770386, "tpp_threshold_10_total_metric": 0.03774997591972351, "tpp_threshold_10_intended_diff_only": 0.042999982833862305, - "tpp_threshold_10_unintended_diff_only": 0.005250006914138794 + "tpp_threshold_10_unintended_diff_only": 0.005250006914138794, + "tpp_threshold_20_total_metric": 0.231999933719635, + "tpp_threshold_20_intended_diff_only": 0.2709999680519104, + "tpp_threshold_20_unintended_diff_only": 0.03900003433227539, + "tpp_threshold_50_total_metric": 0.312749981880188, + "tpp_threshold_50_intended_diff_only": 0.42000001668930054, + "tpp_threshold_50_unintended_diff_only": 0.10725003480911255, + "tpp_threshold_100_total_metric": 0.3267499953508377, + "tpp_threshold_100_intended_diff_only": 0.4520000219345093, + "tpp_threshold_100_unintended_diff_only": 0.12525002658367157, + "tpp_threshold_500_total_metric": 0.1525000035762787, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.3035000264644623 }, "1": { - "tpp_threshold_2_total_metric": 0.0037499815225601196, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983, - "tpp_threshold_5_total_metric": 0.058500006794929504, - "tpp_threshold_5_intended_diff_only": 0.07700002193450928, - "tpp_threshold_5_unintended_diff_only": 0.018500015139579773, - "tpp_threshold_10_total_metric": 0.10574996471405029, - "tpp_threshold_10_intended_diff_only": 0.12599998712539673, - "tpp_threshold_10_unintended_diff_only": 0.020250022411346436 + "tpp_threshold_2_total_metric": 0.007749989628791809, + "tpp_threshold_2_intended_diff_only": 0.008000016212463379, + "tpp_threshold_2_unintended_diff_only": 0.0002500265836715698, + "tpp_threshold_5_total_metric": 0.06049996614456177, + "tpp_threshold_5_intended_diff_only": 0.07499998807907104, + "tpp_threshold_5_unintended_diff_only": 0.014500021934509277, + "tpp_threshold_10_total_metric": 0.10999999940395355, + "tpp_threshold_10_intended_diff_only": 0.125, + "tpp_threshold_10_unintended_diff_only": 0.015000000596046448, + "tpp_threshold_20_total_metric": 0.31074999272823334, + "tpp_threshold_20_intended_diff_only": 0.36000001430511475, + "tpp_threshold_20_unintended_diff_only": 0.04925002157688141, + "tpp_threshold_50_total_metric": 0.34825000166893005, + "tpp_threshold_50_intended_diff_only": 0.4390000104904175, + "tpp_threshold_50_unintended_diff_only": 0.09075000882148743, + "tpp_threshold_100_total_metric": 0.2982500195503235, + "tpp_threshold_100_intended_diff_only": 0.4540000557899475, + "tpp_threshold_100_unintended_diff_only": 0.15575003623962402, + "tpp_threshold_500_total_metric": 0.14525002241134644, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.3137500286102295 }, "2": { - "tpp_threshold_2_total_metric": 0.008750036358833313, - "tpp_threshold_2_intended_diff_only": 0.006000041961669922, - "tpp_threshold_2_unintended_diff_only": -0.002749994397163391, - "tpp_threshold_5_total_metric": 0.06275001168251038, + "tpp_threshold_2_total_metric": 0.0037499964237213135, + "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_2_unintended_diff_only": 0.000250011682510376, + "tpp_threshold_5_total_metric": 0.06024999916553497, "tpp_threshold_5_intended_diff_only": 0.07300001382827759, - "tpp_threshold_5_unintended_diff_only": 0.010250002145767212, - "tpp_threshold_10_total_metric": 0.2070000171661377, - "tpp_threshold_10_intended_diff_only": 0.2890000343322754, - "tpp_threshold_10_unintended_diff_only": 0.0820000171661377 + "tpp_threshold_5_unintended_diff_only": 0.012750014662742615, + "tpp_threshold_10_total_metric": 0.20650000870227814, + "tpp_threshold_10_intended_diff_only": 0.2900000214576721, + "tpp_threshold_10_unintended_diff_only": 0.08350001275539398, + "tpp_threshold_20_total_metric": 0.21900004148483276, + "tpp_threshold_20_intended_diff_only": 0.3380000591278076, + "tpp_threshold_20_unintended_diff_only": 0.11900001764297485, + "tpp_threshold_50_total_metric": 0.2367500513792038, + "tpp_threshold_50_intended_diff_only": 0.4070000648498535, + "tpp_threshold_50_unintended_diff_only": 0.17025001347064972, + "tpp_threshold_100_total_metric": 0.23674999177455902, + "tpp_threshold_100_intended_diff_only": 0.43800002336502075, + "tpp_threshold_100_unintended_diff_only": 0.20125003159046173, + "tpp_threshold_500_total_metric": 0.10350003838539124, + "tpp_threshold_500_intended_diff_only": 0.4530000686645508, + "tpp_threshold_500_unintended_diff_only": 0.34950003027915955 }, "6": { - "tpp_threshold_2_total_metric": 0.015000015497207642, + "tpp_threshold_2_total_metric": 0.012249991297721863, "tpp_threshold_2_intended_diff_only": 0.013000011444091797, - "tpp_threshold_2_unintended_diff_only": -0.0020000040531158447, - "tpp_threshold_5_total_metric": 0.05674999952316284, - "tpp_threshold_5_intended_diff_only": 0.07300001382827759, - "tpp_threshold_5_unintended_diff_only": 0.016250014305114746, - "tpp_threshold_10_total_metric": 0.24774999916553497, + "tpp_threshold_2_unintended_diff_only": 0.0007500201463699341, + "tpp_threshold_5_total_metric": 0.062499985098838806, + "tpp_threshold_5_intended_diff_only": 0.07400000095367432, + "tpp_threshold_5_unintended_diff_only": 0.01150001585483551, + "tpp_threshold_10_total_metric": 0.24575001001358032, "tpp_threshold_10_intended_diff_only": 0.281000018119812, - "tpp_threshold_10_unintended_diff_only": 0.03325001895427704 + "tpp_threshold_10_unintended_diff_only": 0.03525000810623169, + "tpp_threshold_20_total_metric": 0.4050000458955765, + "tpp_threshold_20_intended_diff_only": 0.4500000476837158, + "tpp_threshold_20_unintended_diff_only": 0.04500000178813934, + "tpp_threshold_50_total_metric": 0.39475004374980927, + "tpp_threshold_50_intended_diff_only": 0.48100006580352783, + "tpp_threshold_50_unintended_diff_only": 0.08625002205371857, + "tpp_threshold_100_total_metric": 0.3282500058412552, + "tpp_threshold_100_intended_diff_only": 0.49400001764297485, + "tpp_threshold_100_unintended_diff_only": 0.16575001180171967, + "tpp_threshold_500_total_metric": 0.23000004887580872, + "tpp_threshold_500_intended_diff_only": 0.49500006437301636, + "tpp_threshold_500_unintended_diff_only": 0.26500001549720764 }, "9": { - "tpp_threshold_2_total_metric": 0.01150001585483551, - "tpp_threshold_2_intended_diff_only": 0.013000011444091797, - "tpp_threshold_2_unintended_diff_only": 0.0014999955892562866, - "tpp_threshold_5_total_metric": 0.10050003230571747, + "tpp_threshold_2_total_metric": 0.012749984860420227, + "tpp_threshold_2_intended_diff_only": 0.013999998569488525, + "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983, + "tpp_threshold_5_total_metric": 0.09675005078315735, "tpp_threshold_5_intended_diff_only": 0.11200004816055298, - "tpp_threshold_5_unintended_diff_only": 0.01150001585483551, - "tpp_threshold_10_total_metric": 0.21575000882148743, - "tpp_threshold_10_intended_diff_only": 0.2940000295639038, - "tpp_threshold_10_unintended_diff_only": 0.07825002074241638 + "tpp_threshold_5_unintended_diff_only": 0.01524999737739563, + "tpp_threshold_10_total_metric": 0.215750053524971, + "tpp_threshold_10_intended_diff_only": 0.2930000424385071, + "tpp_threshold_10_unintended_diff_only": 0.07724998891353607, + "tpp_threshold_20_total_metric": 0.2820000499486923, + "tpp_threshold_20_intended_diff_only": 0.3710000514984131, + "tpp_threshold_20_unintended_diff_only": 0.08900000154972076, + "tpp_threshold_50_total_metric": 0.3202500343322754, + "tpp_threshold_50_intended_diff_only": 0.4540000557899475, + "tpp_threshold_50_unintended_diff_only": 0.13375002145767212, + "tpp_threshold_100_total_metric": 0.2585000544786453, + "tpp_threshold_100_intended_diff_only": 0.4760000705718994, + "tpp_threshold_100_unintended_diff_only": 0.2175000160932541, + "tpp_threshold_500_total_metric": 0.1337500363588333, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.3422500342130661 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.0104999840259552, - "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_2_unintended_diff_only": -0.004500001668930054, - "tpp_threshold_5_total_metric": 0.012750014662742615, - "tpp_threshold_5_intended_diff_only": 0.013000011444091797, - "tpp_threshold_5_unintended_diff_only": 0.00024999678134918213, - "tpp_threshold_10_total_metric": 0.02049998939037323, + "tpp_threshold_2_total_metric": 0.007249981164932251, + "tpp_threshold_2_intended_diff_only": 0.009999990463256836, + "tpp_threshold_2_unintended_diff_only": 0.002750009298324585, + "tpp_threshold_5_total_metric": 0.0059999823570251465, + "tpp_threshold_5_intended_diff_only": 0.010999977588653564, + "tpp_threshold_5_unintended_diff_only": 0.004999995231628418, + "tpp_threshold_10_total_metric": 0.02175000309944153, "tpp_threshold_10_intended_diff_only": 0.02799999713897705, - "tpp_threshold_10_unintended_diff_only": 0.007500007748603821 + "tpp_threshold_10_unintended_diff_only": 0.0062499940395355225, + "tpp_threshold_20_total_metric": 0.0532500296831131, + "tpp_threshold_20_intended_diff_only": 0.058000028133392334, + "tpp_threshold_20_unintended_diff_only": 0.004749998450279236, + "tpp_threshold_50_total_metric": 0.16500000655651093, + "tpp_threshold_50_intended_diff_only": 0.17100000381469727, + "tpp_threshold_50_unintended_diff_only": 0.00599999725818634, + "tpp_threshold_100_total_metric": 0.3009999990463257, + "tpp_threshold_100_intended_diff_only": 0.3149999976158142, + "tpp_threshold_100_unintended_diff_only": 0.013999998569488525, + "tpp_threshold_500_total_metric": 0.42924998700618744, + "tpp_threshold_500_intended_diff_only": 0.45399999618530273, + "tpp_threshold_500_unintended_diff_only": 0.024750009179115295 }, "2": { - "tpp_threshold_2_total_metric": 0.005250036716461182, - "tpp_threshold_2_intended_diff_only": 0.012000024318695068, - "tpp_threshold_2_unintended_diff_only": 0.006749987602233887, - "tpp_threshold_5_total_metric": 0.01175004243850708, - "tpp_threshold_5_intended_diff_only": 0.020000040531158447, - "tpp_threshold_5_unintended_diff_only": 0.008249998092651367, - "tpp_threshold_10_total_metric": 0.06925004720687866, - "tpp_threshold_10_intended_diff_only": 0.07600003480911255, - "tpp_threshold_10_unintended_diff_only": 0.006749987602233887 + "tpp_threshold_2_total_metric": 0.01524999737739563, + "tpp_threshold_2_intended_diff_only": 0.013999998569488525, + "tpp_threshold_2_unintended_diff_only": -0.0012499988079071045, + "tpp_threshold_5_total_metric": 0.008500024676322937, + "tpp_threshold_5_intended_diff_only": 0.017000019550323486, + "tpp_threshold_5_unintended_diff_only": 0.00849999487400055, + "tpp_threshold_10_total_metric": 0.0675000250339508, + "tpp_threshold_10_intended_diff_only": 0.078000009059906, + "tpp_threshold_10_unintended_diff_only": 0.0104999840259552, + "tpp_threshold_20_total_metric": 0.09450003504753113, + "tpp_threshold_20_intended_diff_only": 0.10800004005432129, + "tpp_threshold_20_unintended_diff_only": 0.013500005006790161, + "tpp_threshold_50_total_metric": 0.1817500740289688, + "tpp_threshold_50_intended_diff_only": 0.1990000605583191, + "tpp_threshold_50_unintended_diff_only": 0.01724998652935028, + "tpp_threshold_100_total_metric": 0.3047500252723694, + "tpp_threshold_100_intended_diff_only": 0.3370000123977661, + "tpp_threshold_100_unintended_diff_only": 0.03224998712539673, + "tpp_threshold_500_total_metric": 0.39600005745887756, + "tpp_threshold_500_intended_diff_only": 0.43900007009506226, + "tpp_threshold_500_unintended_diff_only": 0.04300001263618469 }, "3": { - "tpp_threshold_2_total_metric": 0.018749982118606567, + "tpp_threshold_2_total_metric": 0.012749984860420227, "tpp_threshold_2_intended_diff_only": 0.015999972820281982, - "tpp_threshold_2_unintended_diff_only": -0.002750009298324585, - "tpp_threshold_5_total_metric": 0.02050003409385681, - "tpp_threshold_5_intended_diff_only": 0.021000027656555176, - "tpp_threshold_5_unintended_diff_only": 0.0004999935626983643, - "tpp_threshold_10_total_metric": 0.013999998569488525, - "tpp_threshold_10_intended_diff_only": 0.019999980926513672, - "tpp_threshold_10_unintended_diff_only": 0.0059999823570251465 + "tpp_threshold_2_unintended_diff_only": 0.0032499879598617554, + "tpp_threshold_5_total_metric": 0.015000000596046448, + "tpp_threshold_5_intended_diff_only": 0.013999998569488525, + "tpp_threshold_5_unintended_diff_only": -0.0010000020265579224, + "tpp_threshold_10_total_metric": 0.022749990224838257, + "tpp_threshold_10_intended_diff_only": 0.02499997615814209, + "tpp_threshold_10_unintended_diff_only": 0.002249985933303833, + "tpp_threshold_20_total_metric": 0.02800002694129944, + "tpp_threshold_20_intended_diff_only": 0.0350000262260437, + "tpp_threshold_20_unintended_diff_only": 0.006999999284744263, + "tpp_threshold_50_total_metric": 0.13324999809265137, + "tpp_threshold_50_intended_diff_only": 0.14399999380111694, + "tpp_threshold_50_unintended_diff_only": 0.010749995708465576, + "tpp_threshold_100_total_metric": 0.23224999010562897, + "tpp_threshold_100_intended_diff_only": 0.25, + "tpp_threshold_100_unintended_diff_only": 0.017750009894371033, + "tpp_threshold_500_total_metric": 0.38700003921985626, + "tpp_threshold_500_intended_diff_only": 0.4270000457763672, + "tpp_threshold_500_unintended_diff_only": 0.040000006556510925 }, "5": { - "tpp_threshold_2_total_metric": 0.0002499520778656006, + "tpp_threshold_2_total_metric": 0.008249938488006592, "tpp_threshold_2_intended_diff_only": 0.008999943733215332, - "tpp_threshold_2_unintended_diff_only": 0.008749991655349731, - "tpp_threshold_5_total_metric": 0.009249970316886902, - "tpp_threshold_5_intended_diff_only": 0.011999964714050293, - "tpp_threshold_5_unintended_diff_only": 0.002749994397163391, - "tpp_threshold_10_total_metric": 0.02349993586540222, - "tpp_threshold_10_intended_diff_only": 0.026999950408935547, - "tpp_threshold_10_unintended_diff_only": 0.003500014543533325 + "tpp_threshold_2_unintended_diff_only": 0.0007500052452087402, + "tpp_threshold_5_total_metric": 0.018499955534934998, + "tpp_threshold_5_intended_diff_only": 0.01799994707107544, + "tpp_threshold_5_unintended_diff_only": -0.0005000084638595581, + "tpp_threshold_10_total_metric": 0.01774996519088745, + "tpp_threshold_10_intended_diff_only": 0.02499997615814209, + "tpp_threshold_10_unintended_diff_only": 0.007250010967254639, + "tpp_threshold_20_total_metric": 0.03374996781349182, + "tpp_threshold_20_intended_diff_only": 0.038999974727630615, + "tpp_threshold_20_unintended_diff_only": 0.005250006914138794, + "tpp_threshold_50_total_metric": 0.08224999904632568, + "tpp_threshold_50_intended_diff_only": 0.09299999475479126, + "tpp_threshold_50_unintended_diff_only": 0.010749995708465576, + "tpp_threshold_100_total_metric": 0.20024996995925903, + "tpp_threshold_100_intended_diff_only": 0.20999997854232788, + "tpp_threshold_100_unintended_diff_only": 0.009750008583068848, + "tpp_threshold_500_total_metric": 0.3814999908208847, + "tpp_threshold_500_intended_diff_only": 0.4169999957084656, + "tpp_threshold_500_unintended_diff_only": 0.03550000488758087 }, "6": { - "tpp_threshold_2_total_metric": 0.05924998223781586, - "tpp_threshold_2_intended_diff_only": 0.06099998950958252, - "tpp_threshold_2_unintended_diff_only": 0.0017500072717666626, - "tpp_threshold_5_total_metric": 0.09300000965595245, - "tpp_threshold_5_intended_diff_only": 0.10500001907348633, - "tpp_threshold_5_unintended_diff_only": 0.012000009417533875, - "tpp_threshold_10_total_metric": 0.13975001871585846, - "tpp_threshold_10_intended_diff_only": 0.1470000147819519, - "tpp_threshold_10_unintended_diff_only": 0.007249996066093445 + "tpp_threshold_2_total_metric": 0.050749972462654114, + "tpp_threshold_2_intended_diff_only": 0.05799996852874756, + "tpp_threshold_2_unintended_diff_only": 0.007249996066093445, + "tpp_threshold_5_total_metric": 0.096499964594841, + "tpp_threshold_5_intended_diff_only": 0.09999996423721313, + "tpp_threshold_5_unintended_diff_only": 0.0034999996423721313, + "tpp_threshold_10_total_metric": 0.14350003004074097, + "tpp_threshold_10_intended_diff_only": 0.15200001001358032, + "tpp_threshold_10_unintended_diff_only": 0.008499979972839355, + "tpp_threshold_20_total_metric": 0.2384999841451645, + "tpp_threshold_20_intended_diff_only": 0.25099998712539673, + "tpp_threshold_20_unintended_diff_only": 0.012500002980232239, + "tpp_threshold_50_total_metric": 0.31300000846385956, + "tpp_threshold_50_intended_diff_only": 0.32899999618530273, + "tpp_threshold_50_unintended_diff_only": 0.015999987721443176, + "tpp_threshold_100_total_metric": 0.34299999475479126, + "tpp_threshold_100_intended_diff_only": 0.3659999966621399, + "tpp_threshold_100_unintended_diff_only": 0.023000001907348633, + "tpp_threshold_500_total_metric": 0.3290000259876251, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.04399999976158142 } } } diff --git a/gemma-2-2b-btk-mat/k-100/seed-0/snapshots/step-61035-tokens-250000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk-mat/k-100/seed-0/snapshots/step-61035-tokens-250000000/tpp/saebench_tpp_custom_sae_eval_results.json index e03dd6df0248b0048109a9d5745f6e86feec34d2..819420e7ac40da5fb42932666e8af7670092ce95 100644 --- a/gemma-2-2b-btk-mat/k-100/seed-0/snapshots/step-61035-tokens-250000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk-mat/k-100/seed-0/snapshots/step-61035-tokens-250000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "258fd16b-d44d-4fbd-ba02-c9735bc19385", - "datetime_epoch_millis": 1776882735186, + "eval_id": "c9af90f5-0d76-4153-b35d-e71d8540b778", + "datetime_epoch_millis": 1777315179100, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.01367499530315399, - "tpp_threshold_2_intended_diff_only": 0.014899998903274536, - "tpp_threshold_2_unintended_diff_only": 0.0012250036001205443, - "tpp_threshold_5_total_metric": 0.04029998928308487, - "tpp_threshold_5_intended_diff_only": 0.04739999771118164, - "tpp_threshold_5_unintended_diff_only": 0.007100008428096771, - "tpp_threshold_10_total_metric": 0.11207500398159026, - "tpp_threshold_10_intended_diff_only": 0.13380001187324525, - "tpp_threshold_10_unintended_diff_only": 0.021725007891654966, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.012924993038177492, + "tpp_threshold_2_intended_diff_only": 0.01510000228881836, + "tpp_threshold_2_unintended_diff_only": 0.0021750092506408695, + "tpp_threshold_5_total_metric": 0.040825004875659945, + "tpp_threshold_5_intended_diff_only": 0.04670000672340393, + "tpp_threshold_5_unintended_diff_only": 0.005875001847743988, + "tpp_threshold_10_total_metric": 0.11460000276565552, + "tpp_threshold_10_intended_diff_only": 0.13590000867843627, + "tpp_threshold_10_unintended_diff_only": 0.02130000591278076, + "tpp_threshold_20_total_metric": 0.18427502512931823, + "tpp_threshold_20_intended_diff_only": 0.231900030374527, + "tpp_threshold_20_unintended_diff_only": 0.04762500524520874, + "tpp_threshold_50_total_metric": 0.23530001342296603, + "tpp_threshold_50_intended_diff_only": 0.3132000207901001, + "tpp_threshold_50_unintended_diff_only": 0.0779000073671341, + "tpp_threshold_100_total_metric": 0.27075001746416094, + "tpp_threshold_100_intended_diff_only": 0.3764000296592712, + "tpp_threshold_100_unintended_diff_only": 0.10565001219511033, + "tpp_threshold_500_total_metric": 0.25757503062486653, + "tpp_threshold_500_intended_diff_only": 0.4451000392436981, + "tpp_threshold_500_unintended_diff_only": 0.18752500861883165 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.011549988389015197, - "tpp_threshold_2_intended_diff_only": 0.012000000476837159, - "tpp_threshold_2_unintended_diff_only": 0.00045001208782196044, - "tpp_threshold_5_total_metric": 0.0490999847650528, - "tpp_threshold_5_intended_diff_only": 0.05839999914169312, - "tpp_threshold_5_unintended_diff_only": 0.00930001437664032, - "tpp_threshold_10_total_metric": 0.17535000443458557, + "tpp_threshold_2_total_metric": 0.01004999279975891, + "tpp_threshold_2_intended_diff_only": 0.01220000982284546, + "tpp_threshold_2_unintended_diff_only": 0.002150017023086548, + "tpp_threshold_5_total_metric": 0.049700000882148744, + "tpp_threshold_5_intended_diff_only": 0.05840001106262207, + "tpp_threshold_5_unintended_diff_only": 0.008700010180473328, + "tpp_threshold_10_total_metric": 0.17580000758171083, "tpp_threshold_10_intended_diff_only": 0.21220002174377442, - "tpp_threshold_10_unintended_diff_only": 0.03685001730918884, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_10_unintended_diff_only": 0.0364000141620636, + "tpp_threshold_20_total_metric": 0.2883000373840332, + "tpp_threshold_20_intended_diff_only": 0.3756000518798828, + "tpp_threshold_20_unintended_diff_only": 0.0873000144958496, + "tpp_threshold_50_total_metric": 0.29470000565052035, + "tpp_threshold_50_intended_diff_only": 0.4412000298500061, + "tpp_threshold_50_unintended_diff_only": 0.14650002419948577, + "tpp_threshold_100_total_metric": 0.26705001294612885, + "tpp_threshold_100_intended_diff_only": 0.46100003719329835, + "tpp_threshold_100_unintended_diff_only": 0.1939500242471695, + "tpp_threshold_500_total_metric": 0.13205003440380098, + "tpp_threshold_500_intended_diff_only": 0.46780005693435667, + "tpp_threshold_500_unintended_diff_only": 0.33575002253055575 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.015800002217292785, - "tpp_threshold_2_intended_diff_only": 0.017799997329711915, - "tpp_threshold_2_unintended_diff_only": 0.0019999951124191283, - "tpp_threshold_5_total_metric": 0.03149999380111694, - "tpp_threshold_5_intended_diff_only": 0.036399996280670165, - "tpp_threshold_5_unintended_diff_only": 0.004900002479553222, - "tpp_threshold_10_total_metric": 0.04880000352859497, - "tpp_threshold_10_intended_diff_only": 0.05540000200271607, - "tpp_threshold_10_unintended_diff_only": 0.006599998474121094, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.01579999327659607, + "tpp_threshold_2_intended_diff_only": 0.01799999475479126, + "tpp_threshold_2_unintended_diff_only": 0.0022000014781951905, + "tpp_threshold_5_total_metric": 0.03195000886917114, + "tpp_threshold_5_intended_diff_only": 0.03500000238418579, + "tpp_threshold_5_unintended_diff_only": 0.0030499935150146485, + "tpp_threshold_10_total_metric": 0.05339999794960022, + "tpp_threshold_10_intended_diff_only": 0.059599995613098145, + "tpp_threshold_10_unintended_diff_only": 0.006199997663497925, + "tpp_threshold_20_total_metric": 0.08025001287460327, + "tpp_threshold_20_intended_diff_only": 0.08820000886917115, + "tpp_threshold_20_unintended_diff_only": 0.00794999599456787, + "tpp_threshold_50_total_metric": 0.17590002119541168, + "tpp_threshold_50_intended_diff_only": 0.1852000117301941, + "tpp_threshold_50_unintended_diff_only": 0.00929999053478241, + "tpp_threshold_100_total_metric": 0.274450021982193, + "tpp_threshold_100_intended_diff_only": 0.29180002212524414, + "tpp_threshold_100_unintended_diff_only": 0.017350000143051148, + "tpp_threshold_500_total_metric": 0.38310002684593203, + "tpp_threshold_500_intended_diff_only": 0.4224000215530396, + "tpp_threshold_500_unintended_diff_only": 0.039299994707107544 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.0022500306367874146, "tpp_threshold_10_total_metric": 0.05924996733665466, "tpp_threshold_10_intended_diff_only": 0.06499999761581421, - "tpp_threshold_10_unintended_diff_only": 0.005750030279159546 + "tpp_threshold_10_unintended_diff_only": 0.005750030279159546, + "tpp_threshold_20_total_metric": 0.24324999749660492, + "tpp_threshold_20_intended_diff_only": 0.3270000219345093, + "tpp_threshold_20_unintended_diff_only": 0.08375002443790436, + "tpp_threshold_50_total_metric": 0.2789999693632126, + "tpp_threshold_50_intended_diff_only": 0.4350000023841858, + "tpp_threshold_50_unintended_diff_only": 0.1560000330209732, + "tpp_threshold_100_total_metric": 0.2512499839067459, + "tpp_threshold_100_intended_diff_only": 0.4520000219345093, + "tpp_threshold_100_unintended_diff_only": 0.20075003802776337, + "tpp_threshold_500_total_metric": 0.1184999942779541, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.33750003576278687 }, "1": { - "tpp_threshold_2_total_metric": 0.00024996697902679443, - "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 0.0007500201463699341, - "tpp_threshold_5_total_metric": 0.044249966740608215, - "tpp_threshold_5_intended_diff_only": 0.05699998140335083, - "tpp_threshold_5_unintended_diff_only": 0.012750014662742615, - "tpp_threshold_10_total_metric": 0.06199999153614044, + "tpp_threshold_2_total_metric": 0.0029999911785125732, + "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162, + "tpp_threshold_5_total_metric": 0.046499982476234436, + "tpp_threshold_5_intended_diff_only": 0.0559999942779541, + "tpp_threshold_5_unintended_diff_only": 0.009500011801719666, + "tpp_threshold_10_total_metric": 0.06549999117851257, "tpp_threshold_10_intended_diff_only": 0.078000009059906, - "tpp_threshold_10_unintended_diff_only": 0.016000017523765564 + "tpp_threshold_10_unintended_diff_only": 0.012500017881393433, + "tpp_threshold_20_total_metric": 0.29750002920627594, + "tpp_threshold_20_intended_diff_only": 0.34400004148483276, + "tpp_threshold_20_unintended_diff_only": 0.046500012278556824, + "tpp_threshold_50_total_metric": 0.31174999475479126, + "tpp_threshold_50_intended_diff_only": 0.42500001192092896, + "tpp_threshold_50_unintended_diff_only": 0.1132500171661377, + "tpp_threshold_100_total_metric": 0.28724998235702515, + "tpp_threshold_100_intended_diff_only": 0.4520000219345093, + "tpp_threshold_100_unintended_diff_only": 0.16475003957748413, + "tpp_threshold_500_total_metric": 0.12150003015995026, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.33750002086162567 }, "2": { - "tpp_threshold_2_total_metric": 0.020999997854232788, - "tpp_threshold_2_intended_diff_only": 0.022000014781951904, - "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162, - "tpp_threshold_5_total_metric": 0.048250019550323486, + "tpp_threshold_2_total_metric": 0.010499998927116394, + "tpp_threshold_2_intended_diff_only": 0.018000006675720215, + "tpp_threshold_2_unintended_diff_only": 0.007500007748603821, + "tpp_threshold_5_total_metric": 0.047500014305114746, "tpp_threshold_5_intended_diff_only": 0.054000020027160645, - "tpp_threshold_5_unintended_diff_only": 0.005750000476837158, - "tpp_threshold_10_total_metric": 0.216000035405159, - "tpp_threshold_10_intended_diff_only": 0.27400004863739014, - "tpp_threshold_10_unintended_diff_only": 0.05800001323223114 + "tpp_threshold_5_unintended_diff_only": 0.0065000057220458984, + "tpp_threshold_10_total_metric": 0.2172500342130661, + "tpp_threshold_10_intended_diff_only": 0.27500003576278687, + "tpp_threshold_10_unintended_diff_only": 0.057750001549720764, + "tpp_threshold_20_total_metric": 0.19175004959106445, + "tpp_threshold_20_intended_diff_only": 0.36100006103515625, + "tpp_threshold_20_unintended_diff_only": 0.1692500114440918, + "tpp_threshold_50_total_metric": 0.19675001502037048, + "tpp_threshold_50_intended_diff_only": 0.40400004386901855, + "tpp_threshold_50_unintended_diff_only": 0.20725002884864807, + "tpp_threshold_100_total_metric": 0.19050003588199615, + "tpp_threshold_100_intended_diff_only": 0.4360000491142273, + "tpp_threshold_100_unintended_diff_only": 0.24550001323223114, + "tpp_threshold_500_total_metric": 0.08900004625320435, + "tpp_threshold_500_intended_diff_only": 0.4530000686645508, + "tpp_threshold_500_unintended_diff_only": 0.36400002241134644 }, "6": { - "tpp_threshold_2_total_metric": 0.01600000262260437, + "tpp_threshold_2_total_metric": 0.013999983668327332, "tpp_threshold_2_intended_diff_only": 0.013999998569488525, - "tpp_threshold_2_unintended_diff_only": -0.0020000040531158447, - "tpp_threshold_5_total_metric": 0.02374999225139618, - "tpp_threshold_5_intended_diff_only": 0.03700000047683716, - "tpp_threshold_5_unintended_diff_only": 0.013250008225440979, - "tpp_threshold_10_total_metric": 0.30124999582767487, + "tpp_threshold_2_unintended_diff_only": 1.4901161193847656e-08, + "tpp_threshold_5_total_metric": 0.028000041842460632, + "tpp_threshold_5_intended_diff_only": 0.03800004720687866, + "tpp_threshold_5_unintended_diff_only": 0.01000000536441803, + "tpp_threshold_10_total_metric": 0.3007500022649765, "tpp_threshold_10_intended_diff_only": 0.328000009059906, - "tpp_threshold_10_unintended_diff_only": 0.02675001323223114 + "tpp_threshold_10_unintended_diff_only": 0.027250006794929504, + "tpp_threshold_20_total_metric": 0.41175004839897156, + "tpp_threshold_20_intended_diff_only": 0.4580000638961792, + "tpp_threshold_20_unintended_diff_only": 0.04625001549720764, + "tpp_threshold_50_total_metric": 0.36375002562999725, + "tpp_threshold_50_intended_diff_only": 0.48200005292892456, + "tpp_threshold_50_unintended_diff_only": 0.11825002729892731, + "tpp_threshold_100_total_metric": 0.33800001442432404, + "tpp_threshold_100_intended_diff_only": 0.48900002241134644, + "tpp_threshold_100_unintended_diff_only": 0.1510000079870224, + "tpp_threshold_500_total_metric": 0.2055000513792038, + "tpp_threshold_500_intended_diff_only": 0.49500006437301636, + "tpp_threshold_500_unintended_diff_only": 0.28950001299381256 }, "9": { - "tpp_threshold_2_total_metric": 0.011750012636184692, - "tpp_threshold_2_intended_diff_only": 0.013000011444091797, - "tpp_threshold_2_unintended_diff_only": 0.0012499988079071045, - "tpp_threshold_5_total_metric": 0.10749998688697815, + "tpp_threshold_2_total_metric": 0.014000028371810913, + "tpp_threshold_2_intended_diff_only": 0.01500004529953003, + "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162, + "tpp_threshold_5_total_metric": 0.10475000739097595, "tpp_threshold_5_intended_diff_only": 0.12000000476837158, - "tpp_threshold_5_unintended_diff_only": 0.012500017881393433, - "tpp_threshold_10_total_metric": 0.2382500320672989, - "tpp_threshold_10_intended_diff_only": 0.3160000443458557, - "tpp_threshold_10_unintended_diff_only": 0.07775001227855682 + "tpp_threshold_5_unintended_diff_only": 0.01524999737739563, + "tpp_threshold_10_total_metric": 0.23625004291534424, + "tpp_threshold_10_intended_diff_only": 0.315000057220459, + "tpp_threshold_10_unintended_diff_only": 0.07875001430511475, + "tpp_threshold_20_total_metric": 0.29725006222724915, + "tpp_threshold_20_intended_diff_only": 0.3880000710487366, + "tpp_threshold_20_unintended_diff_only": 0.09075000882148743, + "tpp_threshold_50_total_metric": 0.32225002348423004, + "tpp_threshold_50_intended_diff_only": 0.46000003814697266, + "tpp_threshold_50_unintended_diff_only": 0.13775001466274261, + "tpp_threshold_100_total_metric": 0.268250048160553, + "tpp_threshold_100_intended_diff_only": 0.4760000705718994, + "tpp_threshold_100_unintended_diff_only": 0.20775002241134644, + "tpp_threshold_500_total_metric": 0.12575004994869232, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.3502500206232071 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.012500002980232239, - "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": -0.0034999996423721313, - "tpp_threshold_5_total_metric": 0.014749974012374878, - "tpp_threshold_5_intended_diff_only": 0.015999972820281982, - "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045, - "tpp_threshold_10_total_metric": 0.021500006318092346, - "tpp_threshold_10_intended_diff_only": 0.03100001811981201, - "tpp_threshold_10_unintended_diff_only": 0.009500011801719666 + "tpp_threshold_2_total_metric": 0.009499996900558472, + "tpp_threshold_2_intended_diff_only": 0.013999998569488525, + "tpp_threshold_2_unintended_diff_only": 0.004500001668930054, + "tpp_threshold_5_total_metric": 0.009749993681907654, + "tpp_threshold_5_intended_diff_only": 0.014999985694885254, + "tpp_threshold_5_unintended_diff_only": 0.0052499920129776, + "tpp_threshold_10_total_metric": 0.02649998664855957, + "tpp_threshold_10_intended_diff_only": 0.03299999237060547, + "tpp_threshold_10_unintended_diff_only": 0.0065000057220458984, + "tpp_threshold_20_total_metric": 0.06700003147125244, + "tpp_threshold_20_intended_diff_only": 0.07200002670288086, + "tpp_threshold_20_unintended_diff_only": 0.004999995231628418, + "tpp_threshold_50_total_metric": 0.19675002992153168, + "tpp_threshold_50_intended_diff_only": 0.1980000138282776, + "tpp_threshold_50_unintended_diff_only": 0.0012499839067459106, + "tpp_threshold_100_total_metric": 0.30775003135204315, + "tpp_threshold_100_intended_diff_only": 0.32200002670288086, + "tpp_threshold_100_unintended_diff_only": 0.014249995350837708, + "tpp_threshold_500_total_metric": 0.4332500547170639, + "tpp_threshold_500_intended_diff_only": 0.45500004291534424, + "tpp_threshold_500_unintended_diff_only": 0.021749988198280334 }, "2": { - "tpp_threshold_2_total_metric": 0.004750043153762817, - "tpp_threshold_2_intended_diff_only": 0.01100003719329834, - "tpp_threshold_2_unintended_diff_only": 0.0062499940395355225, - "tpp_threshold_5_total_metric": 0.02850005030632019, - "tpp_threshold_5_intended_diff_only": 0.03800004720687866, - "tpp_threshold_5_unintended_diff_only": 0.009499996900558472, - "tpp_threshold_10_total_metric": 0.04950001835823059, - "tpp_threshold_10_intended_diff_only": 0.05500000715255737, - "tpp_threshold_10_unintended_diff_only": 0.005499988794326782 + "tpp_threshold_2_total_metric": 0.01475001871585846, + "tpp_threshold_2_intended_diff_only": 0.013000011444091797, + "tpp_threshold_2_unintended_diff_only": -0.0017500072717666626, + "tpp_threshold_5_total_metric": 0.026750028133392334, + "tpp_threshold_5_intended_diff_only": 0.0350000262260437, + "tpp_threshold_5_unintended_diff_only": 0.008249998092651367, + "tpp_threshold_10_total_metric": 0.0492500364780426, + "tpp_threshold_10_intended_diff_only": 0.058000028133392334, + "tpp_threshold_10_unintended_diff_only": 0.008749991655349731, + "tpp_threshold_20_total_metric": 0.07200005650520325, + "tpp_threshold_20_intended_diff_only": 0.08000004291534424, + "tpp_threshold_20_unintended_diff_only": 0.007999986410140991, + "tpp_threshold_50_total_metric": 0.16300006210803986, + "tpp_threshold_50_intended_diff_only": 0.1770000457763672, + "tpp_threshold_50_unintended_diff_only": 0.013999983668327332, + "tpp_threshold_100_total_metric": 0.28700006008148193, + "tpp_threshold_100_intended_diff_only": 0.3060000538825989, + "tpp_threshold_100_unintended_diff_only": 0.018999993801116943, + "tpp_threshold_500_total_metric": 0.40050002932548523, + "tpp_threshold_500_intended_diff_only": 0.43800002336502075, + "tpp_threshold_500_unintended_diff_only": 0.03749999403953552 }, "3": { - "tpp_threshold_2_total_metric": 0.015750005841255188, - "tpp_threshold_2_intended_diff_only": 0.013999998569488525, - "tpp_threshold_2_unintended_diff_only": -0.0017500072717666626, - "tpp_threshold_5_total_metric": 0.021750018000602722, - "tpp_threshold_5_intended_diff_only": 0.023000001907348633, - "tpp_threshold_5_unintended_diff_only": 0.0012499839067459106, - "tpp_threshold_10_total_metric": 0.01875002682209015, - "tpp_threshold_10_intended_diff_only": 0.026000022888183594, - "tpp_threshold_10_unintended_diff_only": 0.007249996066093445 + "tpp_threshold_2_total_metric": 0.010500013828277588, + "tpp_threshold_2_intended_diff_only": 0.013000011444091797, + "tpp_threshold_2_unintended_diff_only": 0.002499997615814209, + "tpp_threshold_5_total_metric": 0.01175004243850708, + "tpp_threshold_5_intended_diff_only": 0.012000024318695068, + "tpp_threshold_5_unintended_diff_only": 0.0002499818801879883, + "tpp_threshold_10_total_metric": 0.026749998331069946, + "tpp_threshold_10_intended_diff_only": 0.02899998426437378, + "tpp_threshold_10_unintended_diff_only": 0.002249985933303833, + "tpp_threshold_20_total_metric": 0.026249989867210388, + "tpp_threshold_20_intended_diff_only": 0.03299999237060547, + "tpp_threshold_20_unintended_diff_only": 0.006750002503395081, + "tpp_threshold_50_total_metric": 0.1250000298023224, + "tpp_threshold_50_intended_diff_only": 0.13300001621246338, + "tpp_threshold_50_unintended_diff_only": 0.007999986410140991, + "tpp_threshold_100_total_metric": 0.22925002872943878, + "tpp_threshold_100_intended_diff_only": 0.24700003862380981, + "tpp_threshold_100_unintended_diff_only": 0.017750009894371033, + "tpp_threshold_500_total_metric": 0.3862500488758087, + "tpp_threshold_500_intended_diff_only": 0.4270000457763672, + "tpp_threshold_500_unintended_diff_only": 0.04074999690055847 }, "5": { - "tpp_threshold_2_total_metric": -0.003000006079673767, - "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_2_unintended_diff_only": 0.008999988436698914, - "tpp_threshold_5_total_metric": 0.016249969601631165, - "tpp_threshold_5_intended_diff_only": 0.019999980926513672, - "tpp_threshold_5_unintended_diff_only": 0.0037500113248825073, - "tpp_threshold_10_total_metric": 0.02449999749660492, - "tpp_threshold_10_intended_diff_only": 0.02799999713897705, - "tpp_threshold_10_unintended_diff_only": 0.0034999996423721313 + "tpp_threshold_2_total_metric": 0.007249936461448669, + "tpp_threshold_2_intended_diff_only": 0.007999956607818604, + "tpp_threshold_2_unintended_diff_only": 0.0007500201463699341, + "tpp_threshold_5_total_metric": 0.030749961733818054, + "tpp_threshold_5_intended_diff_only": 0.030999958515167236, + "tpp_threshold_5_unintended_diff_only": 0.00024999678134918213, + "tpp_threshold_10_total_metric": 0.025249958038330078, + "tpp_threshold_10_intended_diff_only": 0.030999958515167236, + "tpp_threshold_10_unintended_diff_only": 0.005750000476837158, + "tpp_threshold_20_total_metric": 0.04999998211860657, + "tpp_threshold_20_intended_diff_only": 0.05299997329711914, + "tpp_threshold_20_unintended_diff_only": 0.0029999911785125732, + "tpp_threshold_50_total_metric": 0.10349997878074646, + "tpp_threshold_50_intended_diff_only": 0.1119999885559082, + "tpp_threshold_50_unintended_diff_only": 0.008500009775161743, + "tpp_threshold_100_total_metric": 0.21199999749660492, + "tpp_threshold_100_intended_diff_only": 0.22200000286102295, + "tpp_threshold_100_unintended_diff_only": 0.01000000536441803, + "tpp_threshold_500_total_metric": 0.37424996495246887, + "tpp_threshold_500_intended_diff_only": 0.41899996995925903, + "tpp_threshold_500_unintended_diff_only": 0.04475000500679016 }, "6": { - "tpp_threshold_2_total_metric": 0.04899996519088745, - "tpp_threshold_2_intended_diff_only": 0.04899996519088745, - "tpp_threshold_2_unintended_diff_only": 0.0, - "tpp_threshold_5_total_metric": 0.07624995708465576, - "tpp_threshold_5_intended_diff_only": 0.08499997854232788, - "tpp_threshold_5_unintended_diff_only": 0.00875002145767212, - "tpp_threshold_10_total_metric": 0.12974996864795685, - "tpp_threshold_10_intended_diff_only": 0.1369999647140503, - "tpp_threshold_10_unintended_diff_only": 0.007249996066093445 + "tpp_threshold_2_total_metric": 0.03700000047683716, + "tpp_threshold_2_intended_diff_only": 0.041999995708465576, + "tpp_threshold_2_unintended_diff_only": 0.004999995231628418, + "tpp_threshold_5_total_metric": 0.08075001835823059, + "tpp_threshold_5_intended_diff_only": 0.0820000171661377, + "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_10_total_metric": 0.1392500102519989, + "tpp_threshold_10_intended_diff_only": 0.1470000147819519, + "tpp_threshold_10_unintended_diff_only": 0.007750004529953003, + "tpp_threshold_20_total_metric": 0.1860000044107437, + "tpp_threshold_20_intended_diff_only": 0.203000009059906, + "tpp_threshold_20_unintended_diff_only": 0.017000004649162292, + "tpp_threshold_50_total_metric": 0.29125000536441803, + "tpp_threshold_50_intended_diff_only": 0.3059999942779541, + "tpp_threshold_50_unintended_diff_only": 0.014749988913536072, + "tpp_threshold_100_total_metric": 0.3362499922513962, + "tpp_threshold_100_intended_diff_only": 0.3619999885559082, + "tpp_threshold_100_unintended_diff_only": 0.025749996304512024, + "tpp_threshold_500_total_metric": 0.3212500363588333, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.05174998939037323 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-1-tokens-0/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-1-tokens-0/tpp/saebench_tpp_custom_sae_eval_results.json index 8dcac8eb69d8aa5489c736732c1b73f1ed4934bf..e799a61656ebe67b4102ff5fb578861e5cd94c1a 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-1-tokens-0/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-1-tokens-0/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "88ce73c6-9294-4e50-9eca-f41dc29bbd93", - "datetime_epoch_millis": 1776883332532, + "eval_id": "e1af4f20-5d92-44a9-9bac-6f65ac766765", + "datetime_epoch_millis": 1777315836333, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.002375009655952454, - "tpp_threshold_2_intended_diff_only": 0.0010000109672546388, - "tpp_threshold_2_unintended_diff_only": -0.001374998688697815, - "tpp_threshold_5_total_metric": -0.0002499982714653015, - "tpp_threshold_5_intended_diff_only": 0.0005000054836273194, - "tpp_threshold_5_unintended_diff_only": 0.0007500037550926209, - "tpp_threshold_10_total_metric": -0.0014749899506568908, - "tpp_threshold_10_intended_diff_only": -0.0011999845504760743, - "tpp_threshold_10_unintended_diff_only": 0.0002750054001808167, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.0010000094771385193, + "tpp_threshold_2_intended_diff_only": 0.000800013542175293, + "tpp_threshold_2_unintended_diff_only": -0.00019999593496322632, + "tpp_threshold_5_total_metric": -2.9802322387803733e-09, + "tpp_threshold_5_intended_diff_only": -0.0007999956607818604, + "tpp_threshold_5_unintended_diff_only": -0.0007999926805496216, + "tpp_threshold_10_total_metric": 0.0002999991178512574, + "tpp_threshold_10_intended_diff_only": 0.0008000016212463378, + "tpp_threshold_10_unintended_diff_only": 0.0005000025033950805, + "tpp_threshold_20_total_metric": 0.0010500028729438782, + "tpp_threshold_20_intended_diff_only": 0.0012000083923339843, + "tpp_threshold_20_unintended_diff_only": 0.0001500055193901062, + "tpp_threshold_50_total_metric": 0.003299999237060547, + "tpp_threshold_50_intended_diff_only": 0.0024000048637390133, + "tpp_threshold_50_unintended_diff_only": -0.0008999943733215332, + "tpp_threshold_100_total_metric": 0.0006500110030174255, + "tpp_threshold_100_intended_diff_only": 0.0023000121116638185, + "tpp_threshold_100_unintended_diff_only": 0.0016500011086463928, + "tpp_threshold_500_total_metric": 0.027400009334087372, + "tpp_threshold_500_intended_diff_only": 0.028900015354156493, + "tpp_threshold_500_unintended_diff_only": 0.0015000060200691223 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.002300018072128296, - "tpp_threshold_2_intended_diff_only": 0.0008000254631042481, - "tpp_threshold_2_unintended_diff_only": -0.001499992609024048, - "tpp_threshold_5_total_metric": -0.0015500009059906006, + "tpp_threshold_2_total_metric": 0.0014000117778778076, + "tpp_threshold_2_intended_diff_only": 0.0012000203132629395, + "tpp_threshold_2_unintended_diff_only": -0.00019999146461486817, + "tpp_threshold_5_total_metric": -0.00030000805854797364, "tpp_threshold_5_intended_diff_only": -0.0007999897003173828, - "tpp_threshold_5_unintended_diff_only": 0.0007500112056732178, - "tpp_threshold_10_total_metric": -0.0018499940633773804, - "tpp_threshold_10_intended_diff_only": -0.0015999794006347657, - "tpp_threshold_10_unintended_diff_only": 0.00025001466274261476, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_5_unintended_diff_only": -0.0004999816417694091, + "tpp_threshold_10_total_metric": -0.0014500021934509277, + "tpp_threshold_10_intended_diff_only": -0.001399993896484375, + "tpp_threshold_10_unintended_diff_only": 5.0008296966552734e-05, + "tpp_threshold_20_total_metric": 0.0032000094652175903, + "tpp_threshold_20_intended_diff_only": 0.002800023555755615, + "tpp_threshold_20_unintended_diff_only": -0.0003999859094619751, + "tpp_threshold_50_total_metric": 0.002300000190734863, + "tpp_threshold_50_intended_diff_only": 0.002000010013580322, + "tpp_threshold_50_unintended_diff_only": -0.000299990177154541, + "tpp_threshold_100_total_metric": -0.0005499899387359619, + "tpp_threshold_100_intended_diff_only": 0.00020002126693725587, + "tpp_threshold_100_unintended_diff_only": 0.0007500112056732178, + "tpp_threshold_500_total_metric": 0.011700013279914856, + "tpp_threshold_500_intended_diff_only": 0.011800026893615723, + "tpp_threshold_500_unintended_diff_only": 0.0001000136137008667 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.002450001239776611, - "tpp_threshold_2_intended_diff_only": 0.0011999964714050292, - "tpp_threshold_2_unintended_diff_only": -0.001250004768371582, - "tpp_threshold_5_total_metric": 0.0010500043630599976, - "tpp_threshold_5_intended_diff_only": 0.0018000006675720215, - "tpp_threshold_5_unintended_diff_only": 0.000749996304512024, - "tpp_threshold_10_total_metric": -0.0010999858379364014, - "tpp_threshold_10_intended_diff_only": -0.0007999897003173828, - "tpp_threshold_10_unintended_diff_only": 0.0002999961376190186, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.000600007176399231, + "tpp_threshold_2_intended_diff_only": 0.0004000067710876465, + "tpp_threshold_2_unintended_diff_only": -0.00020000040531158446, + "tpp_threshold_5_total_metric": 0.0003000020980834961, + "tpp_threshold_5_intended_diff_only": -0.0008000016212463378, + "tpp_threshold_5_unintended_diff_only": -0.001100003719329834, + "tpp_threshold_10_total_metric": 0.0020500004291534426, + "tpp_threshold_10_intended_diff_only": 0.0029999971389770507, + "tpp_threshold_10_unintended_diff_only": 0.0009499967098236084, + "tpp_threshold_20_total_metric": -0.001100003719329834, + "tpp_threshold_20_intended_diff_only": -0.0004000067710876465, + "tpp_threshold_20_unintended_diff_only": 0.0006999969482421875, + "tpp_threshold_50_total_metric": 0.0042999982833862305, + "tpp_threshold_50_intended_diff_only": 0.002799999713897705, + "tpp_threshold_50_unintended_diff_only": -0.0014999985694885253, + "tpp_threshold_100_total_metric": 0.001850011944770813, + "tpp_threshold_100_intended_diff_only": 0.004400002956390381, + "tpp_threshold_100_unintended_diff_only": 0.002549991011619568, + "tpp_threshold_500_total_metric": 0.043100005388259886, + "tpp_threshold_500_intended_diff_only": 0.046000003814697266, + "tpp_threshold_500_unintended_diff_only": 0.002899998426437378 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.000250011682510376, "tpp_threshold_10_total_metric": -0.004500031471252441, "tpp_threshold_10_intended_diff_only": -0.0040000081062316895, - "tpp_threshold_10_unintended_diff_only": 0.000500023365020752 + "tpp_threshold_10_unintended_diff_only": 0.000500023365020752, + "tpp_threshold_20_total_metric": 0.0034999698400497437, + "tpp_threshold_20_intended_diff_only": 0.001999974250793457, + "tpp_threshold_20_unintended_diff_only": -0.0014999955892562866, + "tpp_threshold_50_total_metric": -2.9802322387695312e-08, + "tpp_threshold_50_intended_diff_only": 0.0009999871253967285, + "tpp_threshold_50_unintended_diff_only": 0.0010000169277191162, + "tpp_threshold_100_total_metric": -0.0010000020265579224, + "tpp_threshold_100_intended_diff_only": -0.0009999871253967285, + "tpp_threshold_100_unintended_diff_only": 1.4901161193847656e-08, + "tpp_threshold_500_total_metric": 0.019999966025352478, + "tpp_threshold_500_intended_diff_only": 0.018999993801116943, + "tpp_threshold_500_unintended_diff_only": -0.0009999722242355347 }, "1": { - "tpp_threshold_2_total_metric": -0.001749977469444275, - "tpp_threshold_2_intended_diff_only": -0.001999974250793457, - "tpp_threshold_2_unintended_diff_only": -0.00024999678134918213, - "tpp_threshold_5_total_metric": -0.0010000169277191162, - "tpp_threshold_5_intended_diff_only": 0.0, - "tpp_threshold_5_unintended_diff_only": 0.0010000169277191162, - "tpp_threshold_10_total_metric": -0.007499992847442627, - "tpp_threshold_10_intended_diff_only": -0.0059999823570251465, - "tpp_threshold_10_unintended_diff_only": 0.0015000104904174805 + "tpp_threshold_2_total_metric": 0.003000020980834961, + "tpp_threshold_2_intended_diff_only": 0.0020000338554382324, + "tpp_threshold_2_unintended_diff_only": -0.0009999871253967285, + "tpp_threshold_5_total_metric": -0.004250019788742065, + "tpp_threshold_5_intended_diff_only": -0.004999995231628418, + "tpp_threshold_5_unintended_diff_only": -0.0007499754428863525, + "tpp_threshold_10_total_metric": -0.0022500157356262207, + "tpp_threshold_10_intended_diff_only": -0.0040000081062316895, + "tpp_threshold_10_unintended_diff_only": -0.0017499923706054688, + "tpp_threshold_20_total_metric": -0.0015000253915786743, + "tpp_threshold_20_intended_diff_only": 0.0, + "tpp_threshold_20_unintended_diff_only": 0.0015000253915786743, + "tpp_threshold_50_total_metric": -0.0002500265836715698, + "tpp_threshold_50_intended_diff_only": 0.0009999871253967285, + "tpp_threshold_50_unintended_diff_only": 0.0012500137090682983, + "tpp_threshold_100_total_metric": -0.006750017404556274, + "tpp_threshold_100_intended_diff_only": -0.0040000081062316895, + "tpp_threshold_100_unintended_diff_only": 0.002750009298324585, + "tpp_threshold_500_total_metric": 0.016750022768974304, + "tpp_threshold_500_intended_diff_only": 0.016000032424926758, + "tpp_threshold_500_unintended_diff_only": -0.0007499903440475464 }, "2": { - "tpp_threshold_2_total_metric": 0.005750030279159546, - "tpp_threshold_2_intended_diff_only": 0.0020000338554382324, - "tpp_threshold_2_unintended_diff_only": -0.0037499964237213135, - "tpp_threshold_5_total_metric": 0.0007499903440475464, - "tpp_threshold_5_intended_diff_only": 0.0, - "tpp_threshold_5_unintended_diff_only": -0.0007499903440475464, - "tpp_threshold_10_total_metric": 0.0045000165700912476, - "tpp_threshold_10_intended_diff_only": 0.0020000338554382324, - "tpp_threshold_10_unintended_diff_only": -0.002499982714653015 + "tpp_threshold_2_total_metric": -0.0017499923706054688, + "tpp_threshold_2_intended_diff_only": -0.0009999871253967285, + "tpp_threshold_2_unintended_diff_only": 0.0007500052452087402, + "tpp_threshold_5_total_metric": -0.0012499988079071045, + "tpp_threshold_5_intended_diff_only": -0.0009999871253967285, + "tpp_threshold_5_unintended_diff_only": 0.000250011682510376, + "tpp_threshold_10_total_metric": 0.003999993205070496, + "tpp_threshold_10_intended_diff_only": 0.004999995231628418, + "tpp_threshold_10_unintended_diff_only": 0.0010000020265579224, + "tpp_threshold_20_total_metric": 0.010750040411949158, + "tpp_threshold_20_intended_diff_only": 0.010000050067901611, + "tpp_threshold_20_unintended_diff_only": -0.0007499903440475464, + "tpp_threshold_50_total_metric": 0.005250006914138794, + "tpp_threshold_50_intended_diff_only": 0.003000020980834961, + "tpp_threshold_50_unintended_diff_only": -0.002249985933303833, + "tpp_threshold_100_total_metric": 0.004000023007392883, + "tpp_threshold_100_intended_diff_only": 0.0020000338554382324, + "tpp_threshold_100_unintended_diff_only": -0.001999989151954651, + "tpp_threshold_500_total_metric": 0.011250033974647522, + "tpp_threshold_500_intended_diff_only": 0.010000050067901611, + "tpp_threshold_500_unintended_diff_only": -0.0012499839067459106 }, "6": { - "tpp_threshold_2_total_metric": 0.004000052809715271, + "tpp_threshold_2_total_metric": 0.0017500370740890503, "tpp_threshold_2_intended_diff_only": 0.001000046730041504, - "tpp_threshold_2_unintended_diff_only": -0.003000006079673767, - "tpp_threshold_5_total_metric": -0.005249977111816406, - "tpp_threshold_5_intended_diff_only": -0.0029999613761901855, - "tpp_threshold_5_unintended_diff_only": 0.0022500157356262207, - "tpp_threshold_10_total_metric": 2.9802322387695312e-08, - "tpp_threshold_10_intended_diff_only": 0.001000046730041504, - "tpp_threshold_10_unintended_diff_only": 0.0010000169277191162 + "tpp_threshold_2_unintended_diff_only": -0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.0007500052452087402, + "tpp_threshold_5_intended_diff_only": -0.001999974250793457, + "tpp_threshold_5_unintended_diff_only": -0.0027499794960021973, + "tpp_threshold_10_total_metric": 0.003000035881996155, + "tpp_threshold_10_intended_diff_only": 0.0020000338554382324, + "tpp_threshold_10_unintended_diff_only": -0.0010000020265579224, + "tpp_threshold_20_total_metric": 0.003500029444694519, + "tpp_threshold_20_intended_diff_only": 0.001000046730041504, + "tpp_threshold_20_unintended_diff_only": -0.002499982714653015, + "tpp_threshold_50_total_metric": 0.0065000057220458984, + "tpp_threshold_50_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_50_unintended_diff_only": -0.002499997615814209, + "tpp_threshold_100_total_metric": 0.0007500201463699341, + "tpp_threshold_100_intended_diff_only": 0.0020000338554382324, + "tpp_threshold_100_unintended_diff_only": 0.0012500137090682983, + "tpp_threshold_500_total_metric": 0.006000012159347534, + "tpp_threshold_500_intended_diff_only": 0.00700002908706665, + "tpp_threshold_500_unintended_diff_only": 0.0010000169277191162 }, "9": { - "tpp_threshold_2_total_metric": -0.0010000020265579224, - "tpp_threshold_2_intended_diff_only": -0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 1.4901161193847656e-08, - "tpp_threshold_5_total_metric": -0.00599999725818634, - "tpp_threshold_5_intended_diff_only": -0.004999995231628418, - "tpp_threshold_5_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_10_total_metric": -0.0017499923706054688, - "tpp_threshold_10_intended_diff_only": -0.0009999871253967285, - "tpp_threshold_10_unintended_diff_only": 0.0007500052452087402 + "tpp_threshold_2_total_metric": -0.0004999935626983643, + "tpp_threshold_2_intended_diff_only": 0.0, + "tpp_threshold_2_unintended_diff_only": 0.0004999935626983643, + "tpp_threshold_5_total_metric": -0.000500023365020752, + "tpp_threshold_5_intended_diff_only": 0.0, + "tpp_threshold_5_unintended_diff_only": 0.000500023365020752, + "tpp_threshold_10_total_metric": -0.007499992847442627, + "tpp_threshold_10_intended_diff_only": -0.0059999823570251465, + "tpp_threshold_10_unintended_diff_only": 0.0015000104904174805, + "tpp_threshold_20_total_metric": -0.00024996697902679443, + "tpp_threshold_20_intended_diff_only": 0.001000046730041504, + "tpp_threshold_20_unintended_diff_only": 0.0012500137090682983, + "tpp_threshold_50_total_metric": 4.470348358154297e-08, + "tpp_threshold_50_intended_diff_only": 0.001000046730041504, + "tpp_threshold_50_unintended_diff_only": 0.0010000020265579224, + "tpp_threshold_100_total_metric": 0.0002500265836715698, + "tpp_threshold_100_intended_diff_only": 0.0020000338554382324, + "tpp_threshold_100_unintended_diff_only": 0.0017500072717666626, + "tpp_threshold_500_total_metric": 0.004500031471252441, + "tpp_threshold_500_intended_diff_only": 0.00700002908706665, + "tpp_threshold_500_unintended_diff_only": 0.002499997615814209 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.006999984383583069, - "tpp_threshold_2_intended_diff_only": 0.001999974250793457, - "tpp_threshold_2_unintended_diff_only": -0.005000010132789612, - "tpp_threshold_5_total_metric": 0.00899997353553772, - "tpp_threshold_5_intended_diff_only": 0.006999969482421875, - "tpp_threshold_5_unintended_diff_only": -0.0020000040531158447, - "tpp_threshold_10_total_metric": -0.007749974727630615, - "tpp_threshold_10_intended_diff_only": -0.004999995231628418, - "tpp_threshold_10_unintended_diff_only": 0.0027499794960021973 + "tpp_threshold_2_total_metric": 0.0007500052452087402, + "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_2_unintended_diff_only": 0.0032500028610229492, + "tpp_threshold_5_total_metric": 0.0012500137090682983, + "tpp_threshold_5_intended_diff_only": 0.003000020980834961, + "tpp_threshold_5_unintended_diff_only": 0.0017500072717666626, + "tpp_threshold_10_total_metric": -0.002249971032142639, + "tpp_threshold_10_intended_diff_only": -0.0009999871253967285, + "tpp_threshold_10_unintended_diff_only": 0.0012499839067459106, + "tpp_threshold_20_total_metric": -0.001999974250793457, + "tpp_threshold_20_intended_diff_only": -0.0009999871253967285, + "tpp_threshold_20_unintended_diff_only": 0.0009999871253967285, + "tpp_threshold_50_total_metric": 0.0017499923706054688, + "tpp_threshold_50_intended_diff_only": -0.0040000081062316895, + "tpp_threshold_50_unintended_diff_only": -0.005750000476837158, + "tpp_threshold_100_total_metric": -0.0010000020265579224, + "tpp_threshold_100_intended_diff_only": 0.0009999871253967285, + "tpp_threshold_100_unintended_diff_only": 0.001999989151954651, + "tpp_threshold_500_total_metric": 0.020249992609024048, + "tpp_threshold_500_intended_diff_only": 0.015999972820281982, + "tpp_threshold_500_unintended_diff_only": -0.004250019788742065 }, "2": { - "tpp_threshold_2_total_metric": -0.0057499706745147705, - "tpp_threshold_2_intended_diff_only": -0.001999974250793457, - "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135, - "tpp_threshold_5_total_metric": -0.007999926805496216, - "tpp_threshold_5_intended_diff_only": -0.003999948501586914, - "tpp_threshold_5_unintended_diff_only": 0.003999978303909302, - "tpp_threshold_10_total_metric": -0.0009999573230743408, - "tpp_threshold_10_intended_diff_only": -0.0029999613761901855, - "tpp_threshold_10_unintended_diff_only": -0.0020000040531158447 + "tpp_threshold_2_total_metric": 0.003000020980834961, + "tpp_threshold_2_intended_diff_only": -0.0009999871253967285, + "tpp_threshold_2_unintended_diff_only": -0.0040000081062316895, + "tpp_threshold_5_total_metric": -0.010749951004981995, + "tpp_threshold_5_intended_diff_only": -0.006999969482421875, + "tpp_threshold_5_unintended_diff_only": 0.0037499815225601196, + "tpp_threshold_10_total_metric": 0.003250017762184143, + "tpp_threshold_10_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_10_unintended_diff_only": 0.0007499903440475464, + "tpp_threshold_20_total_metric": -0.0019999444484710693, + "tpp_threshold_20_intended_diff_only": -0.003999948501586914, + "tpp_threshold_20_unintended_diff_only": -0.0020000040531158447, + "tpp_threshold_50_total_metric": -0.0004999637603759766, + "tpp_threshold_50_intended_diff_only": 0.0020000338554382324, + "tpp_threshold_50_unintended_diff_only": 0.002499997615814209, + "tpp_threshold_100_total_metric": -0.003749936819076538, + "tpp_threshold_100_intended_diff_only": 0.001000046730041504, + "tpp_threshold_100_unintended_diff_only": 0.004749983549118042, + "tpp_threshold_500_total_metric": 0.05375005304813385, + "tpp_threshold_500_intended_diff_only": 0.057000041007995605, + "tpp_threshold_500_unintended_diff_only": 0.0032499879598617554 }, "3": { - "tpp_threshold_2_total_metric": 0.010000035166740417, - "tpp_threshold_2_intended_diff_only": 0.003000020980834961, - "tpp_threshold_2_unintended_diff_only": -0.0070000141859054565, - "tpp_threshold_5_total_metric": 0.014250025153160095, - "tpp_threshold_5_intended_diff_only": 0.012000024318695068, - "tpp_threshold_5_unintended_diff_only": -0.002250000834465027, - "tpp_threshold_10_total_metric": 0.011250019073486328, - "tpp_threshold_10_intended_diff_only": 0.012000024318695068, - "tpp_threshold_10_unintended_diff_only": 0.0007500052452087402 + "tpp_threshold_2_total_metric": 0.0007500052452087402, + "tpp_threshold_2_intended_diff_only": 0.0, + "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402, + "tpp_threshold_5_total_metric": 0.006999999284744263, + "tpp_threshold_5_intended_diff_only": 0.0009999871253967285, + "tpp_threshold_5_unintended_diff_only": -0.006000012159347534, + "tpp_threshold_10_total_metric": 0.009000033140182495, + "tpp_threshold_10_intended_diff_only": 0.00700002908706665, + "tpp_threshold_10_unintended_diff_only": -0.0020000040531158447, + "tpp_threshold_20_total_metric": 0.0012499839067459106, + "tpp_threshold_20_intended_diff_only": 0.001999974250793457, + "tpp_threshold_20_unintended_diff_only": 0.0007499903440475464, + "tpp_threshold_50_total_metric": 0.013250023126602173, + "tpp_threshold_50_intended_diff_only": 0.00700002908706665, + "tpp_threshold_50_unintended_diff_only": -0.0062499940395355225, + "tpp_threshold_100_total_metric": 0.020500004291534424, + "tpp_threshold_100_intended_diff_only": 0.018999993801116943, + "tpp_threshold_100_unintended_diff_only": -0.0015000104904174805, + "tpp_threshold_500_total_metric": 0.03399999439716339, + "tpp_threshold_500_intended_diff_only": 0.03700000047683716, + "tpp_threshold_500_unintended_diff_only": 0.003000006079673767 }, "5": { - "tpp_threshold_2_total_metric": -0.006250053644180298, - "tpp_threshold_2_intended_diff_only": -0.001000046730041504, - "tpp_threshold_2_unintended_diff_only": 0.005250006914138794, - "tpp_threshold_5_total_metric": -0.005250066518783569, - "tpp_threshold_5_intended_diff_only": -0.005000054836273193, - "tpp_threshold_5_unintended_diff_only": 0.000250011682510376, - "tpp_threshold_10_total_metric": -0.005000010132789612, - "tpp_threshold_10_intended_diff_only": -0.0040000081062316895, - "tpp_threshold_10_unintended_diff_only": 0.0010000020265579224 + "tpp_threshold_2_total_metric": 0.0012499839067459106, + "tpp_threshold_2_intended_diff_only": 0.0, + "tpp_threshold_2_unintended_diff_only": -0.0012499839067459106, + "tpp_threshold_5_total_metric": 0.002249985933303833, + "tpp_threshold_5_intended_diff_only": 0.0009999871253967285, + "tpp_threshold_5_unintended_diff_only": -0.0012499988079071045, + "tpp_threshold_10_total_metric": -0.010500043630599976, + "tpp_threshold_10_intended_diff_only": -0.00700002908706665, + "tpp_threshold_10_unintended_diff_only": 0.003500014543533325, + "tpp_threshold_20_total_metric": -0.004750058054924011, + "tpp_threshold_20_intended_diff_only": -0.005000054836273193, + "tpp_threshold_20_unintended_diff_only": -0.00024999678134918213, + "tpp_threshold_50_total_metric": 0.0012499690055847168, + "tpp_threshold_50_intended_diff_only": 0.0029999613761901855, + "tpp_threshold_50_unintended_diff_only": 0.0017499923706054688, + "tpp_threshold_100_total_metric": -0.005250021815299988, + "tpp_threshold_100_intended_diff_only": -0.003000020980834961, + "tpp_threshold_100_unintended_diff_only": 0.002250000834465027, + "tpp_threshold_500_total_metric": 0.020999982953071594, + "tpp_threshold_500_intended_diff_only": 0.02799999713897705, + "tpp_threshold_500_unintended_diff_only": 0.0070000141859054565 }, "6": { - "tpp_threshold_2_total_metric": 0.007250010967254639, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": -0.0032500028610229492, - "tpp_threshold_5_total_metric": -0.004749983549118042, - "tpp_threshold_5_intended_diff_only": -0.0009999871253967285, - "tpp_threshold_5_unintended_diff_only": 0.0037499964237213135, - "tpp_threshold_10_total_metric": -0.003000006079673767, - "tpp_threshold_10_intended_diff_only": -0.0040000081062316895, - "tpp_threshold_10_unintended_diff_only": -0.0010000020265579224 + "tpp_threshold_2_total_metric": -0.0027499794960021973, + "tpp_threshold_2_intended_diff_only": -0.0009999871253967285, + "tpp_threshold_2_unintended_diff_only": 0.0017499923706054688, + "tpp_threshold_5_total_metric": 0.001749962568283081, + "tpp_threshold_5_intended_diff_only": -0.0020000338554382324, + "tpp_threshold_5_unintended_diff_only": -0.0037499964237213135, + "tpp_threshold_10_total_metric": 0.010749965906143188, + "tpp_threshold_10_intended_diff_only": 0.011999964714050293, + "tpp_threshold_10_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_20_total_metric": 0.001999974250793457, + "tpp_threshold_20_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_20_unintended_diff_only": 0.0040000081062316895, + "tpp_threshold_50_total_metric": 0.0057499706745147705, + "tpp_threshold_50_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_50_unintended_diff_only": 0.000250011682510376, + "tpp_threshold_100_total_metric": -0.0012499839067459106, + "tpp_threshold_100_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_100_unintended_diff_only": 0.0052499920129776, + "tpp_threshold_500_total_metric": 0.08650000393390656, + "tpp_threshold_500_intended_diff_only": 0.09200000762939453, + "tpp_threshold_500_unintended_diff_only": 0.005500003695487976 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-109863-tokens-450000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-109863-tokens-450000000/scr/saebench_scr_custom_sae_eval_results.json index 8294a5e6e636febd2b318d506020fc0e9dfc6c40..2c3b67b1f5f16e2da8f46876fbfa70bfb2d6b759 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-109863-tokens-450000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-109863-tokens-450000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "fe5d3fb3-3b9a-43dc-a36c-a365a99d6d04", - "datetime_epoch_millis": 1776883225997, + "eval_id": "89ef4954-029b-48d0-9a7a-282b42224e3d", + "datetime_epoch_millis": 1777315454905, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.24931224390936493, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.22760336536778464, "scr_metric_threshold_10": 0.22359674172327682, "scr_dir2_threshold_10": 0.22317059201096162, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.06267052552110919, + "scr_metric_threshold_20": 0.26391427202288076, + "scr_dir2_threshold_20": 0.2605021904039865, + "scr_dir1_threshold_50": 0.14509570183438503, + "scr_metric_threshold_50": 0.35441780922749, + "scr_dir2_threshold_50": 0.3490721145045288, + "scr_dir1_threshold_100": 0.1175602804273406, + "scr_metric_threshold_100": 0.3854469697379585, + "scr_dir2_threshold_100": 0.38118519632470216, + "scr_dir1_threshold_500": -0.7360395318955923, + "scr_metric_threshold_500": 0.20030516015854738, + "scr_dir2_threshold_500": 0.19811638530758433 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.4230775402847115, "scr_metric_threshold_10": 0.10103619422217953, "scr_dir2_threshold_10": 0.10103619422217953, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.47435892537422925, + "scr_metric_threshold_20": 0.173575039924299, + "scr_dir2_threshold_20": 0.173575039924299, + "scr_dir1_threshold_50": 0.5641028776049328, + "scr_metric_threshold_50": 0.27461138856264183, + "scr_dir2_threshold_50": 0.27461138856264183, + "scr_dir1_threshold_100": 0.39743608357792887, + "scr_metric_threshold_100": 0.3834196571158211, + "scr_dir2_threshold_100": 0.3834196571158211, + "scr_dir1_threshold_500": -2.730768907469913, + "scr_metric_threshold_500": 0.07253884570211949, + "scr_dir2_threshold_500": 0.07253884570211949 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.3484848005931564, "scr_metric_threshold_10": 0.19877673778514987, "scr_dir2_threshold_10": 0.19877673778514987, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.21212114370450913, + "scr_metric_threshold_20": 0.14373084950283896, + "scr_dir2_threshold_20": 0.14373084950283896, + "scr_dir1_threshold_50": 0.41666662903748003, + "scr_metric_threshold_50": 0.3944954111717689, + "scr_dir2_threshold_50": 0.3944954111717689, + "scr_dir1_threshold_100": 0.2045454853329709, + "scr_metric_threshold_100": 0.590214084558388, + "scr_dir2_threshold_100": 0.590214084558388, + "scr_dir1_threshold_500": 0.15151542518196345, + "scr_metric_threshold_500": 0.19571867338661902, + "scr_dir2_threshold_500": 0.19571867338661902 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.28125005820765137, "scr_metric_threshold_10": 0.11055282101401862, "scr_dir2_threshold_10": 0.11055282101401862, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.3906245634426147, + "scr_metric_threshold_20": 0.16080402160563206, + "scr_dir2_threshold_20": 0.16080402160563206, + "scr_dir1_threshold_50": -0.18750034924590825, + "scr_metric_threshold_50": 0.17085429167603294, + "scr_dir2_threshold_50": 0.17085429167603294, + "scr_dir1_threshold_100": 0.1093754365573853, + "scr_metric_threshold_100": 0.04522614043660845, + "scr_dir2_threshold_100": 0.04522614043660845, + "scr_dir1_threshold_500": -2.7812500582076516, + "scr_metric_threshold_500": -0.09547734102822189, + "scr_dir2_threshold_500": -0.09547734102822189 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": 0.04026834093059259, "scr_metric_threshold_10": 0.27388542165349355, "scr_dir2_threshold_10": 0.27388542165349355, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.6644298255263911, + "scr_metric_threshold_20": 0.32165607513677746, + "scr_dir2_threshold_20": 0.32165607513677746, + "scr_dir1_threshold_50": -0.38926196243306976, + "scr_metric_threshold_50": 0.5063695343586597, + "scr_dir2_threshold_50": 0.5063695343586597, + "scr_dir1_threshold_100": -0.6174497610813644, + "scr_metric_threshold_100": 0.47452243203647043, + "scr_dir2_threshold_100": 0.47452243203647043, + "scr_dir1_threshold_500": -1.5302017557369474, + "scr_metric_threshold_500": -0.4522293465167161, + "scr_dir2_threshold_500": -0.4522293465167161 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.07894729412163301, "scr_metric_threshold_10": 0.5396826147702655, "scr_dir2_threshold_10": 0.5396826147702655, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.11578955293692479, + "scr_metric_threshold_20": 0.5833333136228053, + "scr_dir2_threshold_20": 0.5833333136228053, + "scr_dir1_threshold_50": -0.28947372548798167, + "scr_metric_threshold_50": 0.654761901946115, + "scr_dir2_threshold_50": 0.654761901946115, + "scr_dir1_threshold_100": -0.29999990588740183, + "scr_metric_threshold_100": 0.7023808031440972, + "scr_dir2_threshold_100": 0.7023808031440972, + "scr_dir1_threshold_500": -0.19999993725826787, + "scr_metric_threshold_500": 0.7460317385229734, + "scr_dir2_threshold_500": 0.7460317385229734 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.3278689218723027, "scr_metric_threshold_10": 0.12595415766246074, "scr_dir2_threshold_10": 0.12595415766246074, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3169397732417441, + "scr_metric_threshold_20": 0.18702297804342538, + "scr_dir2_threshold_20": 0.18702297804342538, + "scr_dir1_threshold_50": 0.4043716594522235, + "scr_metric_threshold_50": 0.24045808212745792, + "scr_dir2_threshold_50": 0.24045808212745792, + "scr_dir1_threshold_100": 0.4535520140184937, + "scr_metric_threshold_100": 0.3015266750097995, + "scr_dir2_threshold_100": 0.3015266750097995, + "scr_dir1_threshold_500": 0.4535520140184937, + "scr_metric_threshold_500": 0.41221362757701907, + "scr_dir2_threshold_500": 0.41221362757701907 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.19534878692091323, "scr_metric_threshold_10": 0.31330480666733007, "scr_dir2_threshold_10": 0.31330480666733007, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2651161514146259, + "scr_metric_threshold_20": 0.36909885188142205, + "scr_dir2_threshold_20": 0.36909885188142205, + "scr_dir1_threshold_50": 0.40000005544617256, + "scr_metric_threshold_50": 0.35193143367601315, + "scr_dir2_threshold_50": 0.35193143367601315, + "scr_dir1_threshold_100": 0.4372091501709683, + "scr_metric_threshold_100": 0.330472224872739, + "scr_dir2_threshold_100": 0.330472224872739, + "scr_dir1_threshold_500": 0.4372091501709683, + "scr_metric_threshold_500": 0.41201726948797046, + "scr_dir2_threshold_500": 0.41201726948797046 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.12558118001131657, "scr_metric_threshold_10": 0.12558118001131657, "scr_dir2_threshold_10": 0.1221719823127949, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.1720930464658461, + "scr_metric_threshold_20": 0.1720930464658461, + "scr_dir2_threshold_20": 0.14479639351469145, + "scr_dir1_threshold_50": 0.24186043030123083, + "scr_metric_threshold_50": 0.24186043030123083, + "scr_dir2_threshold_50": 0.19909487251754088, + "scr_dir1_threshold_100": 0.2558137407297439, + "scr_metric_threshold_100": 0.2558137407297439, + "scr_dir2_threshold_100": 0.22171955342369326, + "scr_dir1_threshold_500": 0.3116278141366155, + "scr_metric_threshold_500": 0.3116278141366155, + "scr_dir2_threshold_500": 0.29411761532891106 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-109863-tokens-450000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-109863-tokens-450000000/tpp/saebench_tpp_custom_sae_eval_results.json index c48513a36b4e23a98811e23ee9805c21fbd6881b..29e7351797dc68cb2bc0eb32c8c039d3812758f9 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-109863-tokens-450000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-109863-tokens-450000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "ad3db8b5-38a5-4958-9dc1-a61ae0bfdebe", - "datetime_epoch_millis": 1776883393381, + "eval_id": "2eb8c3ea-654c-438a-b923-6016563c8d5b", + "datetime_epoch_millis": 1777315731228, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.014100004732608794, - "tpp_threshold_2_intended_diff_only": 0.015100008249282836, - "tpp_threshold_2_unintended_diff_only": 0.0010000035166740418, - "tpp_threshold_5_total_metric": 0.05454999953508377, - "tpp_threshold_5_intended_diff_only": 0.06330000162124634, - "tpp_threshold_5_unintended_diff_only": 0.008750002086162566, - "tpp_threshold_10_total_metric": 0.11117500215768814, - "tpp_threshold_10_intended_diff_only": 0.139000004529953, - "tpp_threshold_10_unintended_diff_only": 0.027825002372264863, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.012424995005130769, + "tpp_threshold_2_intended_diff_only": 0.014499998092651368, + "tpp_threshold_2_unintended_diff_only": 0.0020750030875205995, + "tpp_threshold_5_total_metric": 0.05477501600980759, + "tpp_threshold_5_intended_diff_only": 0.0626000165939331, + "tpp_threshold_5_unintended_diff_only": 0.00782500058412552, + "tpp_threshold_10_total_metric": 0.11235001981258393, + "tpp_threshold_10_intended_diff_only": 0.14010002017021178, + "tpp_threshold_10_unintended_diff_only": 0.02775000035762787, + "tpp_threshold_20_total_metric": 0.14235000908374787, + "tpp_threshold_20_intended_diff_only": 0.18160001039505005, + "tpp_threshold_20_unintended_diff_only": 0.039250001311302185, + "tpp_threshold_50_total_metric": 0.18815000653266908, + "tpp_threshold_50_intended_diff_only": 0.25790001153945924, + "tpp_threshold_50_unintended_diff_only": 0.06975000500679017, + "tpp_threshold_100_total_metric": 0.20410000830888747, + "tpp_threshold_100_intended_diff_only": 0.31500001549720763, + "tpp_threshold_100_unintended_diff_only": 0.11090000718832016, + "tpp_threshold_500_total_metric": 0.2460000142455101, + "tpp_threshold_500_intended_diff_only": 0.429500025510788, + "tpp_threshold_500_unintended_diff_only": 0.18350001126527787 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.02024998366832733, - "tpp_threshold_2_intended_diff_only": 0.021599996089935302, - "tpp_threshold_2_unintended_diff_only": 0.0013500124216079712, - "tpp_threshold_5_total_metric": 0.0921000063419342, - "tpp_threshold_5_intended_diff_only": 0.10720001459121704, - "tpp_threshold_5_unintended_diff_only": 0.015100008249282837, - "tpp_threshold_10_total_metric": 0.1965500146150589, - "tpp_threshold_10_intended_diff_only": 0.24900002479553224, + "tpp_threshold_2_total_metric": 0.01844998896121979, + "tpp_threshold_2_intended_diff_only": 0.02120000123977661, + "tpp_threshold_2_unintended_diff_only": 0.002750012278556824, + "tpp_threshold_5_total_metric": 0.09255001544952393, + "tpp_threshold_5_intended_diff_only": 0.10760002136230469, + "tpp_threshold_5_unintended_diff_only": 0.015050005912780762, + "tpp_threshold_10_total_metric": 0.1963500291109085, + "tpp_threshold_10_intended_diff_only": 0.24880003929138184, "tpp_threshold_10_unintended_diff_only": 0.05245001018047333, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_20_total_metric": 0.23575001060962678, + "tpp_threshold_20_intended_diff_only": 0.3086000204086304, + "tpp_threshold_20_unintended_diff_only": 0.0728500097990036, + "tpp_threshold_50_total_metric": 0.25990001261234286, + "tpp_threshold_50_intended_diff_only": 0.39240002632141113, + "tpp_threshold_50_unintended_diff_only": 0.1325000137090683, + "tpp_threshold_100_total_metric": 0.22480001151561738, + "tpp_threshold_100_intended_diff_only": 0.43480002880096436, + "tpp_threshold_100_unintended_diff_only": 0.21000001728534698, + "tpp_threshold_500_total_metric": 0.1297000229358673, + "tpp_threshold_500_intended_diff_only": 0.46680004596710206, + "tpp_threshold_500_unintended_diff_only": 0.33710002303123476 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.007950025796890258, - "tpp_threshold_2_intended_diff_only": 0.00860002040863037, - "tpp_threshold_2_unintended_diff_only": 0.0006499946117401123, - "tpp_threshold_5_total_metric": 0.016999992728233337, - "tpp_threshold_5_intended_diff_only": 0.019399988651275634, - "tpp_threshold_5_unintended_diff_only": 0.0023999959230422974, - "tpp_threshold_10_total_metric": 0.025799989700317383, - "tpp_threshold_10_intended_diff_only": 0.02899998426437378, - "tpp_threshold_10_unintended_diff_only": 0.0031999945640563965, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.006400001049041748, + "tpp_threshold_2_intended_diff_only": 0.007799994945526123, + "tpp_threshold_2_unintended_diff_only": 0.001399993896484375, + "tpp_threshold_5_total_metric": 0.017000016570091248, + "tpp_threshold_5_intended_diff_only": 0.017600011825561524, + "tpp_threshold_5_unintended_diff_only": 0.0005999952554702759, + "tpp_threshold_10_total_metric": 0.02835001051425934, + "tpp_threshold_10_intended_diff_only": 0.03140000104904175, + "tpp_threshold_10_unintended_diff_only": 0.0030499905347824096, + "tpp_threshold_20_total_metric": 0.048950007557868956, + "tpp_threshold_20_intended_diff_only": 0.05460000038146973, + "tpp_threshold_20_unintended_diff_only": 0.005649992823600769, + "tpp_threshold_50_total_metric": 0.1164000004529953, + "tpp_threshold_50_intended_diff_only": 0.12339999675750732, + "tpp_threshold_50_unintended_diff_only": 0.006999996304512024, + "tpp_threshold_100_total_metric": 0.1834000051021576, + "tpp_threshold_100_intended_diff_only": 0.19520000219345093, + "tpp_threshold_100_unintended_diff_only": 0.011799997091293335, + "tpp_threshold_500_total_metric": 0.3623000055551529, + "tpp_threshold_500_intended_diff_only": 0.3922000050544739, + "tpp_threshold_500_unintended_diff_only": 0.029899999499320984 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.0020000189542770386, "tpp_threshold_10_total_metric": 0.10624998807907104, "tpp_threshold_10_intended_diff_only": 0.12400001287460327, - "tpp_threshold_10_unintended_diff_only": 0.017750024795532227 + "tpp_threshold_10_unintended_diff_only": 0.017750024795532227, + "tpp_threshold_20_total_metric": 0.1942499577999115, + "tpp_threshold_20_intended_diff_only": 0.22899997234344482, + "tpp_threshold_20_unintended_diff_only": 0.034750014543533325, + "tpp_threshold_50_total_metric": 0.2789999842643738, + "tpp_threshold_50_intended_diff_only": 0.34700000286102295, + "tpp_threshold_50_unintended_diff_only": 0.06800001859664917, + "tpp_threshold_100_total_metric": 0.27274997532367706, + "tpp_threshold_100_intended_diff_only": 0.4179999828338623, + "tpp_threshold_100_unintended_diff_only": 0.14525000751018524, + "tpp_threshold_500_total_metric": 0.1419999897480011, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.31400004029273987 }, "1": { - "tpp_threshold_2_total_metric": 0.03899995982646942, + "tpp_threshold_2_total_metric": 0.04074995219707489, "tpp_threshold_2_intended_diff_only": 0.04799997806549072, - "tpp_threshold_2_unintended_diff_only": 0.009000018239021301, - "tpp_threshold_5_total_metric": 0.18300001323223114, + "tpp_threshold_2_unintended_diff_only": 0.0072500258684158325, + "tpp_threshold_5_total_metric": 0.18400001525878906, "tpp_threshold_5_intended_diff_only": 0.23000001907348633, - "tpp_threshold_5_unintended_diff_only": 0.04700000584125519, - "tpp_threshold_10_total_metric": 0.2045000046491623, - "tpp_threshold_10_intended_diff_only": 0.28200000524520874, - "tpp_threshold_10_unintended_diff_only": 0.07750000059604645 + "tpp_threshold_5_unintended_diff_only": 0.046000003814697266, + "tpp_threshold_10_total_metric": 0.20000001788139343, + "tpp_threshold_10_intended_diff_only": 0.281000018119812, + "tpp_threshold_10_unintended_diff_only": 0.08100000023841858, + "tpp_threshold_20_total_metric": 0.2292500138282776, + "tpp_threshold_20_intended_diff_only": 0.31800001859664917, + "tpp_threshold_20_unintended_diff_only": 0.08875000476837158, + "tpp_threshold_50_total_metric": 0.24375002086162567, + "tpp_threshold_50_intended_diff_only": 0.37300002574920654, + "tpp_threshold_50_unintended_diff_only": 0.12925000488758087, + "tpp_threshold_100_total_metric": 0.19874998927116394, + "tpp_threshold_100_intended_diff_only": 0.42000001668930054, + "tpp_threshold_100_unintended_diff_only": 0.2212500274181366, + "tpp_threshold_500_total_metric": 0.11475002765655518, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.34425002336502075 }, "2": { - "tpp_threshold_2_total_metric": 0.03800000250339508, - "tpp_threshold_2_intended_diff_only": 0.03600001335144043, - "tpp_threshold_2_unintended_diff_only": -0.001999989151954651, - "tpp_threshold_5_total_metric": 0.2017500251531601, - "tpp_threshold_5_intended_diff_only": 0.21100002527236938, - "tpp_threshold_5_unintended_diff_only": 0.00925000011920929, - "tpp_threshold_10_total_metric": 0.2410000115633011, - "tpp_threshold_10_intended_diff_only": 0.30400002002716064, - "tpp_threshold_10_unintended_diff_only": 0.06300000846385956 + "tpp_threshold_2_total_metric": 0.026750028133392334, + "tpp_threshold_2_intended_diff_only": 0.03100001811981201, + "tpp_threshold_2_unintended_diff_only": 0.004249989986419678, + "tpp_threshold_5_total_metric": 0.19725003838539124, + "tpp_threshold_5_intended_diff_only": 0.21000003814697266, + "tpp_threshold_5_unintended_diff_only": 0.012749999761581421, + "tpp_threshold_10_total_metric": 0.24250006675720215, + "tpp_threshold_10_intended_diff_only": 0.30500006675720215, + "tpp_threshold_10_unintended_diff_only": 0.0625, + "tpp_threshold_20_total_metric": 0.2447500079870224, + "tpp_threshold_20_intended_diff_only": 0.35500001907348633, + "tpp_threshold_20_unintended_diff_only": 0.11025001108646393, + "tpp_threshold_50_total_metric": 0.19425000250339508, + "tpp_threshold_50_intended_diff_only": 0.3830000162124634, + "tpp_threshold_50_unintended_diff_only": 0.1887500137090683, + "tpp_threshold_100_total_metric": 0.14525005221366882, + "tpp_threshold_100_intended_diff_only": 0.4160000681877136, + "tpp_threshold_100_unintended_diff_only": 0.2707500159740448, + "tpp_threshold_500_total_metric": 0.06650000810623169, + "tpp_threshold_500_intended_diff_only": 0.4520000219345093, + "tpp_threshold_500_unintended_diff_only": 0.3855000138282776 }, "6": { - "tpp_threshold_2_total_metric": 0.012749999761581421, + "tpp_threshold_2_total_metric": 0.010499969124794006, "tpp_threshold_2_intended_diff_only": 0.009999990463256836, - "tpp_threshold_2_unintended_diff_only": -0.002750009298324585, - "tpp_threshold_5_total_metric": 0.04575003683567047, - "tpp_threshold_5_intended_diff_only": 0.057000041007995605, - "tpp_threshold_5_unintended_diff_only": 0.011250004172325134, - "tpp_threshold_10_total_metric": 0.26075005531311035, + "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704, + "tpp_threshold_5_total_metric": 0.05250002443790436, + "tpp_threshold_5_intended_diff_only": 0.058000028133392334, + "tpp_threshold_5_unintended_diff_only": 0.005500003695487976, + "tpp_threshold_10_total_metric": 0.2630000412464142, "tpp_threshold_10_intended_diff_only": 0.315000057220459, - "tpp_threshold_10_unintended_diff_only": 0.05425000190734863 + "tpp_threshold_10_unintended_diff_only": 0.0520000159740448, + "tpp_threshold_20_total_metric": 0.2950000464916229, + "tpp_threshold_20_intended_diff_only": 0.3800000548362732, + "tpp_threshold_20_unintended_diff_only": 0.08500000834465027, + "tpp_threshold_50_total_metric": 0.30250002443790436, + "tpp_threshold_50_intended_diff_only": 0.4740000367164612, + "tpp_threshold_50_unintended_diff_only": 0.17150001227855682, + "tpp_threshold_100_total_metric": 0.2340000420808792, + "tpp_threshold_100_intended_diff_only": 0.48200005292892456, + "tpp_threshold_100_unintended_diff_only": 0.24800001084804535, + "tpp_threshold_500_total_metric": 0.16825003921985626, + "tpp_threshold_500_intended_diff_only": 0.49100005626678467, + "tpp_threshold_500_unintended_diff_only": 0.3227500170469284 }, "9": { - "tpp_threshold_2_total_metric": 0.003999978303909302, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162, - "tpp_threshold_5_total_metric": 0.0169999897480011, - "tpp_threshold_5_intended_diff_only": 0.023000001907348633, - "tpp_threshold_5_unintended_diff_only": 0.006000012159347534, - "tpp_threshold_10_total_metric": 0.17025001347064972, - "tpp_threshold_10_intended_diff_only": 0.2200000286102295, - "tpp_threshold_10_unintended_diff_only": 0.04975001513957977 + "tpp_threshold_2_total_metric": 0.006750017404556274, + "tpp_threshold_2_intended_diff_only": 0.008000016212463379, + "tpp_threshold_2_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_5_total_metric": 0.016000032424926758, + "tpp_threshold_5_intended_diff_only": 0.025000035762786865, + "tpp_threshold_5_unintended_diff_only": 0.009000003337860107, + "tpp_threshold_10_total_metric": 0.17000003159046173, + "tpp_threshold_10_intended_diff_only": 0.21900004148483276, + "tpp_threshold_10_unintended_diff_only": 0.04900000989437103, + "tpp_threshold_20_total_metric": 0.21550002694129944, + "tpp_threshold_20_intended_diff_only": 0.26100003719329834, + "tpp_threshold_20_unintended_diff_only": 0.0455000102519989, + "tpp_threshold_50_total_metric": 0.2800000309944153, + "tpp_threshold_50_intended_diff_only": 0.3850000500679016, + "tpp_threshold_50_unintended_diff_only": 0.10500001907348633, + "tpp_threshold_100_total_metric": 0.2732499986886978, + "tpp_threshold_100_intended_diff_only": 0.43800002336502075, + "tpp_threshold_100_unintended_diff_only": 0.16475002467632294, + "tpp_threshold_500_total_metric": 0.15700004994869232, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.3190000206232071 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.00849999487400055, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": -0.00449998676776886, - "tpp_threshold_5_total_metric": 0.011249974370002747, - "tpp_threshold_5_intended_diff_only": 0.009999990463256836, - "tpp_threshold_5_unintended_diff_only": -0.0012499839067459106, - "tpp_threshold_10_total_metric": 0.00024999678134918213, + "tpp_threshold_2_total_metric": 0.0029999762773513794, + "tpp_threshold_2_intended_diff_only": 0.006999969482421875, + "tpp_threshold_2_unintended_diff_only": 0.003999993205070496, + "tpp_threshold_5_total_metric": 0.006000027060508728, + "tpp_threshold_5_intended_diff_only": 0.008000016212463379, + "tpp_threshold_5_unintended_diff_only": 0.001999989151954651, + "tpp_threshold_10_total_metric": 0.002500012516975403, "tpp_threshold_10_intended_diff_only": 0.004999995231628418, - "tpp_threshold_10_unintended_diff_only": 0.004749998450279236 + "tpp_threshold_10_unintended_diff_only": 0.002499982714653015, + "tpp_threshold_20_total_metric": 0.009000033140182495, + "tpp_threshold_20_intended_diff_only": 0.012000024318695068, + "tpp_threshold_20_unintended_diff_only": 0.0029999911785125732, + "tpp_threshold_50_total_metric": 0.04874996840953827, + "tpp_threshold_50_intended_diff_only": 0.04799997806549072, + "tpp_threshold_50_unintended_diff_only": -0.0007499903440475464, + "tpp_threshold_100_total_metric": 0.09875001013278961, + "tpp_threshold_100_intended_diff_only": 0.10600000619888306, + "tpp_threshold_100_unintended_diff_only": 0.007249996066093445, + "tpp_threshold_500_total_metric": 0.3802500069141388, + "tpp_threshold_500_intended_diff_only": 0.3889999985694885, + "tpp_threshold_500_unintended_diff_only": 0.008749991655349731 }, "2": { - "tpp_threshold_2_total_metric": -0.0014999359846115112, - "tpp_threshold_2_intended_diff_only": 0.005000054836273193, - "tpp_threshold_2_unintended_diff_only": 0.006499990820884705, - "tpp_threshold_5_total_metric": 0.016500025987625122, - "tpp_threshold_5_intended_diff_only": 0.023000001907348633, - "tpp_threshold_5_unintended_diff_only": 0.006499975919723511, - "tpp_threshold_10_total_metric": 0.04575002193450928, - "tpp_threshold_10_intended_diff_only": 0.050999999046325684, - "tpp_threshold_10_unintended_diff_only": 0.005249977111816406 + "tpp_threshold_2_total_metric": 0.010000035166740417, + "tpp_threshold_2_intended_diff_only": 0.008000016212463379, + "tpp_threshold_2_unintended_diff_only": -0.0020000189542770386, + "tpp_threshold_5_total_metric": 0.011250078678131104, + "tpp_threshold_5_intended_diff_only": 0.01900005340576172, + "tpp_threshold_5_unintended_diff_only": 0.007749974727630615, + "tpp_threshold_10_total_metric": 0.04675005376338959, + "tpp_threshold_10_intended_diff_only": 0.053000032901763916, + "tpp_threshold_10_unintended_diff_only": 0.006249979138374329, + "tpp_threshold_20_total_metric": 0.08725002408027649, + "tpp_threshold_20_intended_diff_only": 0.09600001573562622, + "tpp_threshold_20_unintended_diff_only": 0.008749991655349731, + "tpp_threshold_50_total_metric": 0.18550004065036774, + "tpp_threshold_50_intended_diff_only": 0.20200002193450928, + "tpp_threshold_50_unintended_diff_only": 0.01649998128414154, + "tpp_threshold_100_total_metric": 0.30000004172325134, + "tpp_threshold_100_intended_diff_only": 0.32600003480911255, + "tpp_threshold_100_unintended_diff_only": 0.025999993085861206, + "tpp_threshold_500_total_metric": 0.40525004267692566, + "tpp_threshold_500_intended_diff_only": 0.437000036239624, + "tpp_threshold_500_unintended_diff_only": 0.031749993562698364 }, "3": { - "tpp_threshold_2_total_metric": 0.014000043272972107, - "tpp_threshold_2_intended_diff_only": 0.00700002908706665, - "tpp_threshold_2_unintended_diff_only": -0.0070000141859054565, - "tpp_threshold_5_total_metric": 0.015750005841255188, - "tpp_threshold_5_intended_diff_only": 0.013999998569488525, - "tpp_threshold_5_unintended_diff_only": -0.0017500072717666626, - "tpp_threshold_10_total_metric": 0.013499990105628967, - "tpp_threshold_10_intended_diff_only": 0.014999985694885254, - "tpp_threshold_10_unintended_diff_only": 0.0014999955892562866 + "tpp_threshold_2_total_metric": 0.00475001335144043, + "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402, + "tpp_threshold_5_total_metric": 0.01300002634525299, + "tpp_threshold_5_intended_diff_only": 0.00700002908706665, + "tpp_threshold_5_unintended_diff_only": -0.00599999725818634, + "tpp_threshold_10_total_metric": 0.015250027179718018, + "tpp_threshold_10_intended_diff_only": 0.012000024318695068, + "tpp_threshold_10_unintended_diff_only": -0.0032500028610229492, + "tpp_threshold_20_total_metric": 0.01400001347064972, + "tpp_threshold_20_intended_diff_only": 0.018000006675720215, + "tpp_threshold_20_unintended_diff_only": 0.003999993205070496, + "tpp_threshold_50_total_metric": 0.05400000512599945, + "tpp_threshold_50_intended_diff_only": 0.0559999942779541, + "tpp_threshold_50_unintended_diff_only": 0.001999989151954651, + "tpp_threshold_100_total_metric": 0.10975001752376556, + "tpp_threshold_100_intended_diff_only": 0.11500000953674316, + "tpp_threshold_100_unintended_diff_only": 0.0052499920129776, + "tpp_threshold_500_total_metric": 0.33400000631809235, + "tpp_threshold_500_intended_diff_only": 0.35199999809265137, + "tpp_threshold_500_unintended_diff_only": 0.01799999177455902 }, "5": { - "tpp_threshold_2_total_metric": -0.007749989628791809, - "tpp_threshold_2_intended_diff_only": 0.0, - "tpp_threshold_2_unintended_diff_only": 0.007749989628791809, - "tpp_threshold_5_total_metric": 0.0037499964237213135, - "tpp_threshold_5_intended_diff_only": 0.004999995231628418, - "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045, - "tpp_threshold_10_total_metric": 0.006749942898750305, - "tpp_threshold_10_intended_diff_only": 0.008999943733215332, - "tpp_threshold_10_unintended_diff_only": 0.002250000834465027 + "tpp_threshold_2_total_metric": 0.0004999786615371704, + "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, + "tpp_threshold_2_unintended_diff_only": 0.0005000084638595581, + "tpp_threshold_5_total_metric": 0.012999966740608215, + "tpp_threshold_5_intended_diff_only": 0.011999964714050293, + "tpp_threshold_5_unintended_diff_only": -0.0010000020265579224, + "tpp_threshold_10_total_metric": 0.0032499581575393677, + "tpp_threshold_10_intended_diff_only": 0.007999956607818604, + "tpp_threshold_10_unintended_diff_only": 0.004749998450279236, + "tpp_threshold_20_total_metric": 0.018749982118606567, + "tpp_threshold_20_intended_diff_only": 0.019999980926513672, + "tpp_threshold_20_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_50_total_metric": 0.09974999725818634, + "tpp_threshold_50_intended_diff_only": 0.10600000619888306, + "tpp_threshold_50_unintended_diff_only": 0.006250008940696716, + "tpp_threshold_100_total_metric": 0.15024995803833008, + "tpp_threshold_100_intended_diff_only": 0.15999996662139893, + "tpp_threshold_100_unintended_diff_only": 0.009750008583068848, + "tpp_threshold_500_total_metric": 0.3464999496936798, + "tpp_threshold_500_intended_diff_only": 0.4099999666213989, + "tpp_threshold_500_unintended_diff_only": 0.06350001692771912 }, "6": { - "tpp_threshold_2_total_metric": 0.026500016450881958, - "tpp_threshold_2_intended_diff_only": 0.027000010013580322, - "tpp_threshold_2_unintended_diff_only": 0.0004999935626983643, - "tpp_threshold_5_total_metric": 0.03774996101856232, - "tpp_threshold_5_intended_diff_only": 0.04499995708465576, - "tpp_threshold_5_unintended_diff_only": 0.007249996066093445, - "tpp_threshold_10_total_metric": 0.06274999678134918, - "tpp_threshold_10_intended_diff_only": 0.06499999761581421, - "tpp_threshold_10_unintended_diff_only": 0.002250000834465027 + "tpp_threshold_2_total_metric": 0.013750001788139343, + "tpp_threshold_2_intended_diff_only": 0.018999993801116943, + "tpp_threshold_2_unintended_diff_only": 0.0052499920129776, + "tpp_threshold_5_total_metric": 0.0417499840259552, + "tpp_threshold_5_intended_diff_only": 0.041999995708465576, + "tpp_threshold_5_unintended_diff_only": 0.000250011682510376, + "tpp_threshold_10_total_metric": 0.07400000095367432, + "tpp_threshold_10_intended_diff_only": 0.07899999618530273, + "tpp_threshold_10_unintended_diff_only": 0.004999995231628418, + "tpp_threshold_20_total_metric": 0.11574998497962952, + "tpp_threshold_20_intended_diff_only": 0.12699997425079346, + "tpp_threshold_20_unintended_diff_only": 0.01124998927116394, + "tpp_threshold_50_total_metric": 0.1939999908208847, + "tpp_threshold_50_intended_diff_only": 0.20499998331069946, + "tpp_threshold_50_unintended_diff_only": 0.010999992489814758, + "tpp_threshold_100_total_metric": 0.25824999809265137, + "tpp_threshold_100_intended_diff_only": 0.26899999380111694, + "tpp_threshold_100_unintended_diff_only": 0.010749995708465576, + "tpp_threshold_500_total_metric": 0.34550002217292786, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.027500003576278687 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-12207-tokens-50000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-12207-tokens-50000000/scr/saebench_scr_custom_sae_eval_results.json index 44bded113aab17771556886e8bf2880c325d28cb..21a9205a0d0b0a4590925849c42fd9972561bde0 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-12207-tokens-50000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-12207-tokens-50000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "06f0ebff-0b93-452d-8db9-accbec9aa577", - "datetime_epoch_millis": 1776883672083, + "eval_id": "f13a845e-4d8e-420f-a76e-c8300215f674", + "datetime_epoch_millis": 1777315454740, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.20930437004095767, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.25138475324743553, "scr_metric_threshold_10": 0.2690253055472029, "scr_dir2_threshold_10": 0.2691963281664894, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2509970209733557, + "scr_metric_threshold_20": 0.32767724537306003, + "scr_dir2_threshold_20": 0.32541218987999554, + "scr_dir1_threshold_50": 0.23604931453324968, + "scr_metric_threshold_50": 0.4076231162769153, + "scr_dir2_threshold_50": 0.40576581846881726, + "scr_dir1_threshold_100": 0.10870387574017482, + "scr_metric_threshold_100": 0.44131054126569613, + "scr_dir2_threshold_100": 0.4490844228433037, + "scr_dir1_threshold_500": 0.10266567723365157, + "scr_metric_threshold_500": 0.3333851707358609, + "scr_dir2_threshold_500": 0.3470808513435148 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.5384614208981502, "scr_metric_threshold_10": 0.11658034288417893, "scr_dir2_threshold_10": 0.11658034288417893, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.5769232238773122, + "scr_metric_threshold_20": 0.14248705143262683, + "scr_dir2_threshold_20": 0.14248705143262683, + "scr_dir1_threshold_50": 0.5769232238773122, + "scr_metric_threshold_50": 0.25906739431680575, + "scr_dir2_threshold_50": 0.25906739431680575, + "scr_dir1_threshold_100": 0.4487182328294704, + "scr_metric_threshold_100": 0.27720202853425396, + "scr_dir2_threshold_100": 0.27720202853425396, + "scr_dir1_threshold_500": 0.10256429850308302, + "scr_metric_threshold_500": 0.17616583431207444, + "scr_dir2_threshold_500": 0.17616583431207444 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.21212114370450913, "scr_metric_threshold_10": 0.34250758728798886, "scr_dir2_threshold_10": 0.34250758728798886, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.24999988711244006, + "scr_metric_threshold_20": 0.4128439798401429, + "scr_dir2_threshold_20": 0.4128439798401429, + "scr_dir1_threshold_50": 0.15151542518196345, + "scr_metric_threshold_50": 0.5321101318775462, + "scr_dir2_threshold_50": 0.5321101318775462, + "scr_dir1_threshold_100": -0.7272726862227055, + "scr_metric_threshold_100": 0.6116208999024817, + "scr_dir2_threshold_100": 0.6116208999024817, + "scr_dir1_threshold_500": 0.23484857036936357, + "scr_metric_threshold_500": 0.2599389371417114, + "scr_dir2_threshold_500": 0.2599389371417114 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.5312495925464404, "scr_metric_threshold_10": 0.10301508102112025, "scr_dir2_threshold_10": 0.10301508102112025, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3749997671693945, + "scr_metric_threshold_20": 0.1457286913802262, + "scr_dir2_threshold_20": 0.1457286913802262, + "scr_dir1_threshold_50": 0.32812537834973393, + "scr_metric_threshold_50": 0.22613070218304224, + "scr_dir2_threshold_50": 0.22613070218304224, + "scr_dir1_threshold_100": 0.250000465661211, + "scr_metric_threshold_100": 0.29145723300006154, + "scr_dir2_threshold_100": 0.29145723300006154, + "scr_dir1_threshold_500": 0.03125052386886235, + "scr_metric_threshold_500": 0.05276388042950681, + "scr_dir2_threshold_500": 0.05276388042950681 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": 0.24832216912919208, "scr_metric_threshold_10": 0.27388542165349355, "scr_dir2_threshold_10": 0.27388542165349355, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.04026834093059259, + "scr_metric_threshold_20": 0.37579626297872115, + "scr_dir2_threshold_20": 0.37579626297872115, + "scr_dir1_threshold_50": 0.08724800534441686, + "scr_metric_threshold_50": 0.4968153277325217, + "scr_dir2_threshold_50": 0.4968153277325217, + "scr_dir1_threshold_100": 0.13422806978944357, + "scr_metric_threshold_100": 0.5509555155744654, + "scr_dir2_threshold_100": 0.5509555155744654, + "scr_dir1_threshold_500": -0.1140940993397485, + "scr_metric_threshold_500": 0.1305732713799386, + "scr_dir2_threshold_500": 0.1305732713799386 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.0631580235225028, "scr_metric_threshold_10": 0.5198413665167169, "scr_dir2_threshold_10": 0.5198413665167169, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.12105264313663487, + "scr_metric_threshold_20": 0.5753969089319204, + "scr_dir2_threshold_20": 0.5753969089319204, + "scr_dir1_threshold_50": -0.026315764707211, + "scr_metric_threshold_50": 0.595238157185469, + "scr_dir2_threshold_50": 0.595238157185469, + "scr_dir1_threshold_100": -0.07368420392192293, + "scr_metric_threshold_100": 0.6269840124753451, + "scr_dir2_threshold_100": 0.6269840124753451, + "scr_dir1_threshold_500": -0.13157882353605502, + "scr_metric_threshold_500": 0.6190476077844602, + "scr_dir2_threshold_500": 0.6190476077844602 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.21311465264817295, "scr_metric_threshold_10": 0.15648856785294307, "scr_dir2_threshold_10": 0.15648856785294307, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3005465388586523, + "scr_metric_threshold_20": 0.2137404163361301, + "scr_dir2_threshold_20": 0.2137404163361301, + "scr_dir1_threshold_50": 0.2732241558450018, + "scr_metric_threshold_50": 0.29389318621149046, + "scr_dir2_threshold_50": 0.29389318621149046, + "scr_dir1_threshold_100": 0.2841529787670631, + "scr_metric_threshold_100": 0.3320610852002818, + "scr_dir2_threshold_100": 0.3320610852002818, + "scr_dir1_threshold_500": 0.3169397732417441, + "scr_metric_threshold_500": 0.5343510408403254, + "scr_dir2_threshold_500": 0.5343510408403254 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.08837205047154198, "scr_metric_threshold_10": 0.5236051041022068, "scr_dir2_threshold_10": 0.5236051041022068, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.17674410094308396, + "scr_metric_threshold_20": 0.587982986325977, + "scr_dir2_threshold_20": 0.587982986325977, + "scr_dir1_threshold_50": 0.28372083739245524, + "scr_metric_threshold_50": 0.6437767757261215, + "scr_dir2_threshold_50": 0.6437767757261215, + "scr_dir1_threshold_100": 0.3441859975126847, + "scr_metric_threshold_100": 0.6309014039325255, + "scr_dir2_threshold_100": 0.6309014039325255, + "scr_dir1_threshold_500": 0.08372094828480038, + "scr_metric_threshold_500": 0.5965665675217078, + "scr_dir2_threshold_500": 0.5965665675217078 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.1162789730589745, "scr_metric_threshold_10": 0.1162789730589745, "scr_dir2_threshold_10": 0.11764715401326675, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.16744166575873534, + "scr_metric_threshold_20": 0.16744166575873534, + "scr_dir2_threshold_20": 0.1493212218142196, + "scr_dir1_threshold_50": 0.21395325498232515, + "scr_metric_threshold_50": 0.21395325498232515, + "scr_dir2_threshold_50": 0.19909487251754088, + "scr_dir1_threshold_100": 0.2093021515061541, + "scr_metric_threshold_100": 0.2093021515061541, + "scr_dir2_threshold_100": 0.2714932041270145, + "scr_dir1_threshold_500": 0.2976742264771627, + "scr_metric_threshold_500": 0.2976742264771627, + "scr_dir2_threshold_500": 0.4072396713383939 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-12207-tokens-50000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-12207-tokens-50000000/tpp/saebench_tpp_custom_sae_eval_results.json index 3ef23ee7a08c237895a66c76a225a8d114b7d3b5..c1a3d3ff29ead80cfc726333d0641be7b739cf1a 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-12207-tokens-50000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-12207-tokens-50000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "b2714d67-9f92-442f-852f-7db2ff16de12", - "datetime_epoch_millis": 1776883839636, + "eval_id": "670be528-00c3-4655-b732-01db0612bc83", + "datetime_epoch_millis": 1777315727268, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.012475007772445678, - "tpp_threshold_2_intended_diff_only": 0.01220000982284546, - "tpp_threshold_2_unintended_diff_only": -0.0002749979496002197, - "tpp_threshold_5_total_metric": 0.015849997103214265, - "tpp_threshold_5_intended_diff_only": 0.01790000200271606, - "tpp_threshold_5_unintended_diff_only": 0.0020500048995018005, - "tpp_threshold_10_total_metric": 0.02477500140666962, - "tpp_threshold_10_intended_diff_only": 0.027300006151199343, - "tpp_threshold_10_unintended_diff_only": 0.0025250047445297243, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.010499992966651916, + "tpp_threshold_2_intended_diff_only": 0.011399996280670167, + "tpp_threshold_2_unintended_diff_only": 0.0009000033140182495, + "tpp_threshold_5_total_metric": 0.01495000422000885, + "tpp_threshold_5_intended_diff_only": 0.015800011157989503, + "tpp_threshold_5_unintended_diff_only": 0.0008500069379806519, + "tpp_threshold_10_total_metric": 0.02567498981952667, + "tpp_threshold_10_intended_diff_only": 0.028399997949600218, + "tpp_threshold_10_unintended_diff_only": 0.0027250081300735475, + "tpp_threshold_20_total_metric": 0.046775002777576444, + "tpp_threshold_20_intended_diff_only": 0.05060001015663147, + "tpp_threshold_20_unintended_diff_only": 0.003825007379055023, + "tpp_threshold_50_total_metric": 0.0995750069618225, + "tpp_threshold_50_intended_diff_only": 0.1049000084400177, + "tpp_threshold_50_unintended_diff_only": 0.005325001478195191, + "tpp_threshold_100_total_metric": 0.20667501986026765, + "tpp_threshold_100_intended_diff_only": 0.2150000214576721, + "tpp_threshold_100_unintended_diff_only": 0.00832500159740448, + "tpp_threshold_500_total_metric": 0.39562502056360244, + "tpp_threshold_500_intended_diff_only": 0.4092000246047973, + "tpp_threshold_500_unintended_diff_only": 0.013575004041194917 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.015350013971328735, - "tpp_threshold_2_intended_diff_only": 0.01500002145767212, - "tpp_threshold_2_unintended_diff_only": -0.0003499925136566162, - "tpp_threshold_5_total_metric": 0.016349998116493226, - "tpp_threshold_5_intended_diff_only": 0.018200016021728514, - "tpp_threshold_5_unintended_diff_only": 0.0018500179052352906, - "tpp_threshold_10_total_metric": 0.02030000984668732, - "tpp_threshold_10_intended_diff_only": 0.022600018978118898, - "tpp_threshold_10_unintended_diff_only": 0.0023000091314315796, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.013549983501434326, + "tpp_threshold_2_intended_diff_only": 0.01499999761581421, + "tpp_threshold_2_unintended_diff_only": 0.0014500141143798829, + "tpp_threshold_5_total_metric": 0.016849994659423828, + "tpp_threshold_5_intended_diff_only": 0.017400014400482177, + "tpp_threshold_5_unintended_diff_only": 0.0005500197410583496, + "tpp_threshold_10_total_metric": 0.020699986815452577, + "tpp_threshold_10_intended_diff_only": 0.022800004482269286, + "tpp_threshold_10_unintended_diff_only": 0.0021000176668167113, + "tpp_threshold_20_total_metric": 0.04675001204013825, + "tpp_threshold_20_intended_diff_only": 0.04900002479553223, + "tpp_threshold_20_unintended_diff_only": 0.0022500127553939818, + "tpp_threshold_50_total_metric": 0.09805000424385071, + "tpp_threshold_50_intended_diff_only": 0.10160001516342163, + "tpp_threshold_50_unintended_diff_only": 0.003550010919570923, + "tpp_threshold_100_total_metric": 0.22385002672672272, + "tpp_threshold_100_intended_diff_only": 0.2290000319480896, + "tpp_threshold_100_unintended_diff_only": 0.005150005221366882, + "tpp_threshold_500_total_metric": 0.43855002522468567, + "tpp_threshold_500_intended_diff_only": 0.44720003604888914, + "tpp_threshold_500_unintended_diff_only": 0.008650010824203492 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.009600001573562621, - "tpp_threshold_2_intended_diff_only": 0.009399998188018798, - "tpp_threshold_2_unintended_diff_only": -0.00020000338554382324, - "tpp_threshold_5_total_metric": 0.015349996089935303, - "tpp_threshold_5_intended_diff_only": 0.017599987983703613, - "tpp_threshold_5_unintended_diff_only": 0.0022499918937683105, - "tpp_threshold_10_total_metric": 0.029249992966651917, - "tpp_threshold_10_intended_diff_only": 0.03199999332427979, - "tpp_threshold_10_unintended_diff_only": 0.0027500003576278686, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.007450002431869507, + "tpp_threshold_2_intended_diff_only": 0.007799994945526123, + "tpp_threshold_2_unintended_diff_only": 0.0003499925136566162, + "tpp_threshold_5_total_metric": 0.013050013780593872, + "tpp_threshold_5_intended_diff_only": 0.014200007915496827, + "tpp_threshold_5_unintended_diff_only": 0.0011499941349029541, + "tpp_threshold_10_total_metric": 0.03064999282360077, + "tpp_threshold_10_intended_diff_only": 0.03399999141693115, + "tpp_threshold_10_unintended_diff_only": 0.0033499985933303834, + "tpp_threshold_20_total_metric": 0.04679999351501465, + "tpp_threshold_20_intended_diff_only": 0.05219999551773071, + "tpp_threshold_20_unintended_diff_only": 0.0054000020027160645, + "tpp_threshold_50_total_metric": 0.10110000967979431, + "tpp_threshold_50_intended_diff_only": 0.10820000171661377, + "tpp_threshold_50_unintended_diff_only": 0.007099992036819458, + "tpp_threshold_100_total_metric": 0.18950001299381256, + "tpp_threshold_100_intended_diff_only": 0.20100001096725464, + "tpp_threshold_100_unintended_diff_only": 0.011499997973442078, + "tpp_threshold_500_total_metric": 0.3527000159025192, + "tpp_threshold_500_intended_diff_only": 0.37120001316070556, + "tpp_threshold_500_unintended_diff_only": 0.01849999725818634 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.0015000253915786743, "tpp_threshold_10_total_metric": 0.00424996018409729, "tpp_threshold_10_intended_diff_only": 0.006999969482421875, - "tpp_threshold_10_unintended_diff_only": 0.002750009298324585 + "tpp_threshold_10_unintended_diff_only": 0.002750009298324585, + "tpp_threshold_20_total_metric": 0.03949996829032898, + "tpp_threshold_20_intended_diff_only": 0.041999995708465576, + "tpp_threshold_20_unintended_diff_only": 0.0025000274181365967, + "tpp_threshold_50_total_metric": 0.07549996674060822, + "tpp_threshold_50_intended_diff_only": 0.07999998331069946, + "tpp_threshold_50_unintended_diff_only": 0.0045000165700912476, + "tpp_threshold_100_total_metric": 0.24300000071525574, + "tpp_threshold_100_intended_diff_only": 0.24900001287460327, + "tpp_threshold_100_unintended_diff_only": 0.006000012159347534, + "tpp_threshold_500_total_metric": 0.4324999749660492, + "tpp_threshold_500_intended_diff_only": 0.4399999976158142, + "tpp_threshold_500_unintended_diff_only": 0.007500022649765015 }, "1": { - "tpp_threshold_2_total_metric": 0.00100003182888031, - "tpp_threshold_2_intended_diff_only": 0.0020000338554382324, - "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_5_total_metric": 0.0034999847412109375, - "tpp_threshold_5_intended_diff_only": 0.004999995231628418, - "tpp_threshold_5_unintended_diff_only": 0.0015000104904174805, - "tpp_threshold_10_total_metric": 0.004250004887580872, - "tpp_threshold_10_intended_diff_only": 0.008000016212463379, - "tpp_threshold_10_unintended_diff_only": 0.0037500113248825073 + "tpp_threshold_2_total_metric": 0.004749968647956848, + "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983, + "tpp_threshold_5_total_metric": -0.000250011682510376, + "tpp_threshold_5_intended_diff_only": 0.0, + "tpp_threshold_5_unintended_diff_only": 0.000250011682510376, + "tpp_threshold_10_total_metric": 0.011249944567680359, + "tpp_threshold_10_intended_diff_only": 0.010999977588653564, + "tpp_threshold_10_unintended_diff_only": -0.00024996697902679443, + "tpp_threshold_20_total_metric": 0.03499998152256012, + "tpp_threshold_20_intended_diff_only": 0.04100000858306885, + "tpp_threshold_20_unintended_diff_only": 0.006000027060508728, + "tpp_threshold_50_total_metric": 0.09724999964237213, + "tpp_threshold_50_intended_diff_only": 0.10600000619888306, + "tpp_threshold_50_unintended_diff_only": 0.008750006556510925, + "tpp_threshold_100_total_metric": 0.22550003230571747, + "tpp_threshold_100_intended_diff_only": 0.23400002717971802, + "tpp_threshold_100_unintended_diff_only": 0.00849999487400055, + "tpp_threshold_500_total_metric": 0.4415000081062317, + "tpp_threshold_500_intended_diff_only": 0.4570000171661377, + "tpp_threshold_500_unintended_diff_only": 0.015500009059906006 }, "2": { - "tpp_threshold_2_total_metric": 0.05475004017353058, - "tpp_threshold_2_intended_diff_only": 0.05200004577636719, - "tpp_threshold_2_unintended_diff_only": -0.002749994397163391, - "tpp_threshold_5_total_metric": 0.05849997699260712, - "tpp_threshold_5_intended_diff_only": 0.06000000238418579, - "tpp_threshold_5_unintended_diff_only": 0.0015000253915786743, - "tpp_threshold_10_total_metric": 0.06525003910064697, - "tpp_threshold_10_intended_diff_only": 0.06600004434585571, - "tpp_threshold_10_unintended_diff_only": 0.0007500052452087402 + "tpp_threshold_2_total_metric": 0.04249998927116394, + "tpp_threshold_2_intended_diff_only": 0.046000003814697266, + "tpp_threshold_2_unintended_diff_only": 0.003500014543533325, + "tpp_threshold_5_total_metric": 0.05324998497962952, + "tpp_threshold_5_intended_diff_only": 0.05500000715255737, + "tpp_threshold_5_unintended_diff_only": 0.0017500221729278564, + "tpp_threshold_10_total_metric": 0.0637500137090683, + "tpp_threshold_10_intended_diff_only": 0.06700003147125244, + "tpp_threshold_10_unintended_diff_only": 0.003250017762184143, + "tpp_threshold_20_total_metric": 0.10150006413459778, + "tpp_threshold_20_intended_diff_only": 0.10200005769729614, + "tpp_threshold_20_unintended_diff_only": 0.0004999935626983643, + "tpp_threshold_50_total_metric": 0.16050000488758087, + "tpp_threshold_50_intended_diff_only": 0.16100001335144043, + "tpp_threshold_50_unintended_diff_only": 0.0005000084638595581, + "tpp_threshold_100_total_metric": 0.2682500332593918, + "tpp_threshold_100_intended_diff_only": 0.27000004053115845, + "tpp_threshold_100_unintended_diff_only": 0.0017500072717666626, + "tpp_threshold_500_total_metric": 0.4240000396966934, + "tpp_threshold_500_intended_diff_only": 0.4320000410079956, + "tpp_threshold_500_unintended_diff_only": 0.008000001311302185 }, "6": { - "tpp_threshold_2_total_metric": 0.012999996542930603, + "tpp_threshold_2_total_metric": 0.010249972343444824, "tpp_threshold_2_intended_diff_only": 0.009999990463256836, - "tpp_threshold_2_unintended_diff_only": -0.003000006079673767, - "tpp_threshold_5_total_metric": 0.003000035881996155, - "tpp_threshold_5_intended_diff_only": 0.006000041961669922, - "tpp_threshold_5_unintended_diff_only": 0.003000006079673767, - "tpp_threshold_10_total_metric": 0.012750029563903809, - "tpp_threshold_10_intended_diff_only": 0.01500004529953003, - "tpp_threshold_10_unintended_diff_only": 0.0022500157356262207 + "tpp_threshold_2_unintended_diff_only": -0.0002499818801879883, + "tpp_threshold_5_total_metric": 0.009750008583068848, + "tpp_threshold_5_intended_diff_only": 0.00700002908706665, + "tpp_threshold_5_unintended_diff_only": -0.0027499794960021973, + "tpp_threshold_10_total_metric": 0.015750020742416382, + "tpp_threshold_10_intended_diff_only": 0.016000032424926758, + "tpp_threshold_10_unintended_diff_only": 0.000250011682510376, + "tpp_threshold_20_total_metric": 0.016750022768974304, + "tpp_threshold_20_intended_diff_only": 0.016000032424926758, + "tpp_threshold_20_unintended_diff_only": -0.0007499903440475464, + "tpp_threshold_50_total_metric": 0.022000014781951904, + "tpp_threshold_50_intended_diff_only": 0.021000027656555176, + "tpp_threshold_50_unintended_diff_only": -0.0009999871253967285, + "tpp_threshold_100_total_metric": 0.06149999797344208, + "tpp_threshold_100_intended_diff_only": 0.06400001049041748, + "tpp_threshold_100_unintended_diff_only": 0.002500012516975403, + "tpp_threshold_500_total_metric": 0.42900004982948303, + "tpp_threshold_500_intended_diff_only": 0.4310000538825989, + "tpp_threshold_500_unintended_diff_only": 0.0020000040531158447 }, "9": { - "tpp_threshold_2_total_metric": 0.0007500201463699341, - "tpp_threshold_2_intended_diff_only": 0.0020000338554382324, - "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983, - "tpp_threshold_5_total_metric": 0.004250019788742065, - "tpp_threshold_5_intended_diff_only": 0.006000041961669922, - "tpp_threshold_5_unintended_diff_only": 0.0017500221729278564, - "tpp_threshold_10_total_metric": 0.015000015497207642, - "tpp_threshold_10_intended_diff_only": 0.017000019550323486, - "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447 + "tpp_threshold_2_total_metric": 0.003000006079673767, + "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, + "tpp_threshold_5_total_metric": 0.009000018239021301, + "tpp_threshold_5_intended_diff_only": 0.01100003719329834, + "tpp_threshold_5_unintended_diff_only": 0.0020000189542770386, + "tpp_threshold_10_total_metric": 0.00849999487400055, + "tpp_threshold_10_intended_diff_only": 0.013000011444091797, + "tpp_threshold_10_unintended_diff_only": 0.0045000165700912476, + "tpp_threshold_20_total_metric": 0.04100002348423004, + "tpp_threshold_20_intended_diff_only": 0.04400002956390381, + "tpp_threshold_20_unintended_diff_only": 0.003000006079673767, + "tpp_threshold_50_total_metric": 0.13500003516674042, + "tpp_threshold_50_intended_diff_only": 0.14000004529953003, + "tpp_threshold_50_unintended_diff_only": 0.005000010132789612, + "tpp_threshold_100_total_metric": 0.3210000693798065, + "tpp_threshold_100_intended_diff_only": 0.3280000686645508, + "tpp_threshold_100_unintended_diff_only": 0.006999999284744263, + "tpp_threshold_500_total_metric": 0.465750053524971, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.010250017046928406 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.009000003337860107, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": -0.004999995231628418, - "tpp_threshold_5_total_metric": 0.008749991655349731, - "tpp_threshold_5_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_5_unintended_diff_only": -0.002750009298324585, - "tpp_threshold_10_total_metric": 0.0029999762773513794, + "tpp_threshold_2_total_metric": 0.0037499666213989258, + "tpp_threshold_2_intended_diff_only": 0.006999969482421875, + "tpp_threshold_2_unintended_diff_only": 0.0032500028610229492, + "tpp_threshold_5_total_metric": 0.0032500028610229492, + "tpp_threshold_5_intended_diff_only": 0.004999995231628418, + "tpp_threshold_5_unintended_diff_only": 0.0017499923706054688, + "tpp_threshold_10_total_metric": 0.004749968647956848, "tpp_threshold_10_intended_diff_only": 0.006999969482421875, - "tpp_threshold_10_unintended_diff_only": 0.003999993205070496 + "tpp_threshold_10_unintended_diff_only": 0.002250000834465027, + "tpp_threshold_20_total_metric": 0.01524999737739563, + "tpp_threshold_20_intended_diff_only": 0.018999993801116943, + "tpp_threshold_20_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_50_total_metric": 0.04849998652935028, + "tpp_threshold_50_intended_diff_only": 0.04799997806549072, + "tpp_threshold_50_unintended_diff_only": -0.0005000084638595581, + "tpp_threshold_100_total_metric": 0.10199999809265137, + "tpp_threshold_100_intended_diff_only": 0.10600000619888306, + "tpp_threshold_100_unintended_diff_only": 0.0040000081062316895, + "tpp_threshold_500_total_metric": 0.34375, + "tpp_threshold_500_intended_diff_only": 0.3479999899864197, + "tpp_threshold_500_unintended_diff_only": 0.004249989986419678 }, "2": { - "tpp_threshold_2_total_metric": 0.009500056505203247, - "tpp_threshold_2_intended_diff_only": 0.01500004529953003, - "tpp_threshold_2_unintended_diff_only": 0.005499988794326782, - "tpp_threshold_5_total_metric": 0.019000038504600525, - "tpp_threshold_5_intended_diff_only": 0.026000022888183594, - "tpp_threshold_5_unintended_diff_only": 0.006999984383583069, - "tpp_threshold_10_total_metric": 0.04950004816055298, - "tpp_threshold_10_intended_diff_only": 0.05200004577636719, - "tpp_threshold_10_unintended_diff_only": 0.002499997615814209 + "tpp_threshold_2_total_metric": 0.01775003969669342, + "tpp_threshold_2_intended_diff_only": 0.016000032424926758, + "tpp_threshold_2_unintended_diff_only": -0.0017500072717666626, + "tpp_threshold_5_total_metric": 0.014500036835670471, + "tpp_threshold_5_intended_diff_only": 0.022000014781951904, + "tpp_threshold_5_unintended_diff_only": 0.007499977946281433, + "tpp_threshold_10_total_metric": 0.05075004696846008, + "tpp_threshold_10_intended_diff_only": 0.058000028133392334, + "tpp_threshold_10_unintended_diff_only": 0.007249981164932251, + "tpp_threshold_20_total_metric": 0.07225000858306885, + "tpp_threshold_20_intended_diff_only": 0.07400000095367432, + "tpp_threshold_20_unintended_diff_only": 0.0017499923706054688, + "tpp_threshold_50_total_metric": 0.1297500729560852, + "tpp_threshold_50_intended_diff_only": 0.1390000581741333, + "tpp_threshold_50_unintended_diff_only": 0.009249985218048096, + "tpp_threshold_100_total_metric": 0.3250000476837158, + "tpp_threshold_100_intended_diff_only": 0.34400004148483276, + "tpp_threshold_100_unintended_diff_only": 0.018999993801116943, + "tpp_threshold_500_total_metric": 0.4045000523328781, + "tpp_threshold_500_intended_diff_only": 0.4360000491142273, + "tpp_threshold_500_unintended_diff_only": 0.03149999678134918 }, "3": { - "tpp_threshold_2_total_metric": 0.012250006198883057, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": -0.007250010967254639, - "tpp_threshold_5_total_metric": 0.015499994158744812, - "tpp_threshold_5_intended_diff_only": 0.013999998569488525, - "tpp_threshold_5_unintended_diff_only": -0.0014999955892562866, - "tpp_threshold_10_total_metric": 0.019250020384788513, - "tpp_threshold_10_intended_diff_only": 0.021000027656555176, - "tpp_threshold_10_unintended_diff_only": 0.0017500072717666626 + "tpp_threshold_2_total_metric": 0.002750009298324585, + "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, + "tpp_threshold_2_unintended_diff_only": -0.0017500221729278564, + "tpp_threshold_5_total_metric": 0.010500028729438782, + "tpp_threshold_5_intended_diff_only": 0.00700002908706665, + "tpp_threshold_5_unintended_diff_only": -0.0034999996423721313, + "tpp_threshold_10_total_metric": 0.0222499817609787, + "tpp_threshold_10_intended_diff_only": 0.019999980926513672, + "tpp_threshold_10_unintended_diff_only": -0.002250000834465027, + "tpp_threshold_20_total_metric": 0.01124998927116394, + "tpp_threshold_20_intended_diff_only": 0.013999998569488525, + "tpp_threshold_20_unintended_diff_only": 0.002750009298324585, + "tpp_threshold_50_total_metric": 0.05125004053115845, + "tpp_threshold_50_intended_diff_only": 0.053000032901763916, + "tpp_threshold_50_unintended_diff_only": 0.0017499923706054688, + "tpp_threshold_100_total_metric": 0.08875004947185516, + "tpp_threshold_100_intended_diff_only": 0.09500002861022949, + "tpp_threshold_100_unintended_diff_only": 0.006249979138374329, + "tpp_threshold_500_total_metric": 0.320000022649765, + "tpp_threshold_500_intended_diff_only": 0.3320000171661377, + "tpp_threshold_500_unintended_diff_only": 0.01199999451637268 }, "5": { - "tpp_threshold_2_total_metric": -0.004250019788742065, + "tpp_threshold_2_total_metric": 0.0029999762773513794, "tpp_threshold_2_intended_diff_only": 0.001999974250793457, - "tpp_threshold_2_unintended_diff_only": 0.0062499940395355225, - "tpp_threshold_5_total_metric": -0.004250019788742065, - "tpp_threshold_5_intended_diff_only": -0.0020000338554382324, - "tpp_threshold_5_unintended_diff_only": 0.002249985933303833, - "tpp_threshold_10_total_metric": 0.007999956607818604, - "tpp_threshold_10_intended_diff_only": 0.011999964714050293, - "tpp_threshold_10_unintended_diff_only": 0.0040000081062316895 + "tpp_threshold_2_unintended_diff_only": -0.0010000020265579224, + "tpp_threshold_5_total_metric": 0.0004999935626983643, + "tpp_threshold_5_intended_diff_only": 0.0, + "tpp_threshold_5_unintended_diff_only": -0.0004999935626983643, + "tpp_threshold_10_total_metric": -0.005000025033950806, + "tpp_threshold_10_intended_diff_only": 0.001999974250793457, + "tpp_threshold_10_unintended_diff_only": 0.006999999284744263, + "tpp_threshold_20_total_metric": 0.013999983668327332, + "tpp_threshold_20_intended_diff_only": 0.018999993801116943, + "tpp_threshold_20_unintended_diff_only": 0.005000010132789612, + "tpp_threshold_50_total_metric": 0.06074996292591095, + "tpp_threshold_50_intended_diff_only": 0.0679999589920044, + "tpp_threshold_50_unintended_diff_only": 0.007249996066093445, + "tpp_threshold_100_total_metric": 0.12399999797344208, + "tpp_threshold_100_intended_diff_only": 0.1340000033378601, + "tpp_threshold_100_unintended_diff_only": 0.01000000536441803, + "tpp_threshold_500_total_metric": 0.3507499694824219, + "tpp_threshold_500_intended_diff_only": 0.3669999837875366, + "tpp_threshold_500_unintended_diff_only": 0.016250014305114746 }, "6": { - "tpp_threshold_2_total_metric": 0.021499961614608765, - "tpp_threshold_2_intended_diff_only": 0.0209999680519104, - "tpp_threshold_2_unintended_diff_only": -0.0004999935626983643, - "tpp_threshold_5_total_metric": 0.03774997591972351, - "tpp_threshold_5_intended_diff_only": 0.04399996995925903, - "tpp_threshold_5_unintended_diff_only": 0.0062499940395355225, - "tpp_threshold_10_total_metric": 0.06649996340274811, - "tpp_threshold_10_intended_diff_only": 0.0679999589920044, - "tpp_threshold_10_unintended_diff_only": 0.0014999955892562866 + "tpp_threshold_2_total_metric": 0.010000020265579224, + "tpp_threshold_2_intended_diff_only": 0.013000011444091797, + "tpp_threshold_2_unintended_diff_only": 0.0029999911785125732, + "tpp_threshold_5_total_metric": 0.036500006914138794, + "tpp_threshold_5_intended_diff_only": 0.03700000047683716, + "tpp_threshold_5_unintended_diff_only": 0.0004999935626983643, + "tpp_threshold_10_total_metric": 0.08049999177455902, + "tpp_threshold_10_intended_diff_only": 0.08300000429153442, + "tpp_threshold_10_unintended_diff_only": 0.002500012516975403, + "tpp_threshold_20_total_metric": 0.12124998867511749, + "tpp_threshold_20_intended_diff_only": 0.13499999046325684, + "tpp_threshold_20_unintended_diff_only": 0.013750001788139343, + "tpp_threshold_50_total_metric": 0.21524998545646667, + "tpp_threshold_50_intended_diff_only": 0.2329999804496765, + "tpp_threshold_50_unintended_diff_only": 0.01774999499320984, + "tpp_threshold_100_total_metric": 0.3077499717473984, + "tpp_threshold_100_intended_diff_only": 0.3259999752044678, + "tpp_threshold_100_unintended_diff_only": 0.018250003457069397, + "tpp_threshold_500_total_metric": 0.34450003504753113, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.028499990701675415 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-122070-tokens-500000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-122070-tokens-500000000/scr/saebench_scr_custom_sae_eval_results.json index 26f8e6cab3694fc507920086d6673839455f151f..6d06e547b70e7ad7592f170440bfff6d572080d9 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-122070-tokens-500000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-122070-tokens-500000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "a8b883cd-cabb-41bb-875f-a774bc78f704", - "datetime_epoch_millis": 1776883807856, + "eval_id": "007d2088-7969-42f0-a21c-11e05d637da7", + "datetime_epoch_millis": 1777315503017, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.27700654085449283, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.21640243118517796, "scr_metric_threshold_10": 0.23223524384144018, "scr_dir2_threshold_10": 0.22135975708935063, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.04097264054927978, + "scr_metric_threshold_20": 0.29427585256810657, + "scr_dir2_threshold_20": 0.2820639250435518, + "scr_dir1_threshold_50": 0.25708903377113695, + "scr_metric_threshold_50": 0.3417567551129011, + "scr_dir2_threshold_50": 0.32764278327844015, + "scr_dir1_threshold_100": -0.03167553686039668, + "scr_metric_threshold_100": 0.3373470777000024, + "scr_dir2_threshold_100": 0.3179690100659854, + "scr_dir1_threshold_500": -0.9412289882116809, + "scr_metric_threshold_500": 0.19818225611848905, + "scr_dir2_threshold_500": 0.18342375782034986 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.4102564298503083, "scr_metric_threshold_10": 0.06994820573050735, "scr_dir2_threshold_10": 0.06994820573050735, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3589742805987668, + "scr_metric_threshold_20": 0.13730561707323927, + "scr_dir2_threshold_20": 0.13730561707323927, + "scr_dir1_threshold_50": 0.5256410746257707, + "scr_metric_threshold_50": 0.27461138856264183, + "scr_dir2_threshold_50": 0.27461138856264183, + "scr_dir1_threshold_100": 0.4487182328294704, + "scr_metric_threshold_100": 0.27461138856264183, + "scr_dir2_threshold_100": 0.27461138856264183, + "scr_dir1_threshold_500": -2.2820506746404425, + "scr_metric_threshold_500": 0.08808283994795557, + "scr_dir2_threshold_500": 0.08808283994795557 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.31060605718522544, "scr_metric_threshold_10": 0.3241590186196148, "scr_dir2_threshold_10": 0.3241590186196148, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.08333314518740012, + "scr_metric_threshold_20": 0.4311927307857058, + "scr_dir2_threshold_20": 0.4311927307857058, + "scr_dir1_threshold_50": 0.3181817155567637, + "scr_metric_threshold_50": 0.5657492048157633, + "scr_dir2_threshold_50": 0.5657492048157633, + "scr_dir1_threshold_100": -0.1287875469668693, + "scr_metric_threshold_100": 0.6513761016377605, + "scr_dir2_threshold_100": 0.6513761016377605, + "scr_dir1_threshold_500": -1.272726862227055, + "scr_metric_threshold_500": 0.321100954221084, + "scr_dir2_threshold_500": 0.321100954221084 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.17187462165026607, "scr_metric_threshold_10": 0.10804029093651613, "scr_dir2_threshold_10": 0.10804029093651613, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.42187508731147705, + "scr_metric_threshold_20": 0.17336682175353543, + "scr_dir2_threshold_20": 0.17336682175353543, + "scr_dir1_threshold_50": 0.3906245634426147, + "scr_metric_threshold_50": 0.22864323226054475, + "scr_dir2_threshold_50": 0.22864323226054475, + "scr_dir1_threshold_100": -0.6874994179234862, + "scr_metric_threshold_100": 0.06532668057741017, + "scr_dir2_threshold_100": 0.06532668057741017, + "scr_dir1_threshold_500": -3.484374272404358, + "scr_metric_threshold_500": -0.027638130373309224, + "scr_dir2_threshold_500": -0.027638130373309224 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": -0.2416108456459604, "scr_metric_threshold_10": 0.14649682254103324, "scr_dir2_threshold_10": 0.14649682254103324, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.7248325369378812, + "scr_metric_threshold_20": 0.16560504596960618, + "scr_dir2_threshold_20": 0.16560504596960618, + "scr_dir1_threshold_50": -0.36912759195217226, + "scr_metric_threshold_50": 0.06050953237690028, + "scr_dir2_threshold_50": 0.06050953237690028, + "scr_dir1_threshold_100": -1.0872488054068217, + "scr_metric_threshold_100": 0.00955420662613804, + "scr_dir2_threshold_100": 0.00955420662613804, + "scr_dir1_threshold_500": -1.5234904322537157, + "scr_metric_threshold_500": -0.5031846722674783, + "scr_dir2_threshold_500": -0.5031846722674783 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.08421069803000372, "scr_metric_threshold_10": 0.4999998817368318, "scr_dir2_threshold_10": 0.4999998817368318, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.11052646273721473, + "scr_metric_threshold_20": 0.5515872218065929, + "scr_dir2_threshold_20": 0.5515872218065929, + "scr_dir1_threshold_50": -0.11052614902855408, + "scr_metric_threshold_50": 0.6507936996006726, + "scr_dir2_threshold_50": 0.6507936996006726, + "scr_dir1_threshold_100": -0.22631570196547887, + "scr_metric_threshold_100": 0.7063492420158761, + "scr_dir2_threshold_100": 0.7063492420158761, + "scr_dir1_threshold_500": -0.20526302745797795, + "scr_metric_threshold_500": 0.6666667455087788, + "scr_dir2_threshold_500": 0.6666667455087788 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.4098360709132541, "scr_metric_threshold_10": 0.1183206688641517, "scr_dir2_threshold_10": 0.1183206688641517, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.4371584539269045, + "scr_metric_threshold_20": 0.17938926174649328, + "scr_dir2_threshold_20": 0.17938926174649328, + "scr_dir1_threshold_50": 0.497267631415236, + "scr_metric_threshold_50": 0.26335877602100816, + "scr_dir2_threshold_50": 0.26335877602100816, + "scr_dir1_threshold_100": 0.5300547515984144, + "scr_metric_threshold_100": 0.2671755204201627, + "scr_dir2_threshold_100": 0.2671755204201627, + "scr_dir1_threshold_500": 0.5683059575341262, + "scr_metric_threshold_500": 0.2709922648193172, + "scr_dir2_threshold_500": 0.2709922648193172 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.3813953694683433, "scr_metric_threshold_10": 0.3862660142728834, "scr_dir2_threshold_10": 0.3862660142728834, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3860464716550849, + "scr_metric_threshold_20": 0.4506438964966536, + "scr_dir2_threshold_20": 0.4506438964966536, + "scr_dir1_threshold_50": 0.47906962431336847, + "scr_metric_threshold_50": 0.3648068054696092, + "scr_dir2_threshold_50": 0.3648068054696092, + "scr_dir1_threshold_100": 0.5209300984557687, + "scr_metric_threshold_100": 0.3476393872642003, + "scr_dir2_threshold_100": 0.3476393872642003, + "scr_dir1_threshold_500": 0.3209302093481138, + "scr_metric_threshold_500": 0.4206008506837011, + "scr_dir2_threshold_500": 0.4206008506837011 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.20465104802998307, "scr_metric_threshold_10": 0.20465104802998307, "scr_dir2_threshold_10": 0.11764715401326675, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2651162249130257, + "scr_metric_threshold_20": 0.2651162249130257, + "scr_dir2_threshold_20": 0.16742080471658802, + "scr_dir1_threshold_50": 0.32558140179606837, + "scr_metric_threshold_50": 0.32558140179606837, + "scr_dir2_threshold_50": 0.21266962712038115, + "scr_dir1_threshold_100": 0.3767440944958292, + "scr_metric_threshold_100": 0.3767440944958292, + "scr_dir2_threshold_100": 0.22171955342369326, + "scr_dir1_threshold_500": 0.34883719640786326, + "scr_metric_threshold_500": 0.34883719640786326, + "scr_dir2_threshold_500": 0.23076921002274955 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-122070-tokens-500000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-122070-tokens-500000000/tpp/saebench_tpp_custom_sae_eval_results.json index c28c8298948fa3064448a999238d0115b45231f5..b62bff9fef3eba022310f4b857d9cbc8aa21d926 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-122070-tokens-500000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-122070-tokens-500000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "949d3d21-e9f7-4054-8410-d6325ea1d9d4", - "datetime_epoch_millis": 1776883959512, + "eval_id": "a9e83ab3-97ad-4e7b-9a39-78b6523f812c", + "datetime_epoch_millis": 1777315777509, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.022575005888938904, - "tpp_threshold_2_intended_diff_only": 0.02390000820159912, - "tpp_threshold_2_unintended_diff_only": 0.0013250023126602174, - "tpp_threshold_5_total_metric": 0.055699999630451205, - "tpp_threshold_5_intended_diff_only": 0.07050000429153443, - "tpp_threshold_5_unintended_diff_only": 0.014800004661083221, - "tpp_threshold_10_total_metric": 0.10322501361370087, - "tpp_threshold_10_intended_diff_only": 0.13190001845359803, - "tpp_threshold_10_unintended_diff_only": 0.028675004839897156, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.02144999951124191, + "tpp_threshold_2_intended_diff_only": 0.024100005626678467, + "tpp_threshold_2_unintended_diff_only": 0.002650006115436554, + "tpp_threshold_5_total_metric": 0.05662499815225601, + "tpp_threshold_5_intended_diff_only": 0.06930000185966492, + "tpp_threshold_5_unintended_diff_only": 0.012675003707408905, + "tpp_threshold_10_total_metric": 0.10545000731945038, + "tpp_threshold_10_intended_diff_only": 0.13310000896453858, + "tpp_threshold_10_unintended_diff_only": 0.027650001645088195, + "tpp_threshold_20_total_metric": 0.15019999593496322, + "tpp_threshold_20_intended_diff_only": 0.19020000100135803, + "tpp_threshold_20_unintended_diff_only": 0.04000000506639481, + "tpp_threshold_50_total_metric": 0.18925000429153443, + "tpp_threshold_50_intended_diff_only": 0.26040000915527345, + "tpp_threshold_50_unintended_diff_only": 0.07115000486373901, + "tpp_threshold_100_total_metric": 0.2069500148296356, + "tpp_threshold_100_intended_diff_only": 0.32150002717971804, + "tpp_threshold_100_unintended_diff_only": 0.11455001235008239, + "tpp_threshold_500_total_metric": 0.2484000191092491, + "tpp_threshold_500_intended_diff_only": 0.43350002765655515, + "tpp_threshold_500_unintended_diff_only": 0.18510000854730607 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.034050014615058896, - "tpp_threshold_2_intended_diff_only": 0.036400020122528076, - "tpp_threshold_2_unintended_diff_only": 0.0023500055074691774, - "tpp_threshold_5_total_metric": 0.0950499951839447, + "tpp_threshold_2_total_metric": 0.033350002765655515, + "tpp_threshold_2_intended_diff_only": 0.037600016593933104, + "tpp_threshold_2_unintended_diff_only": 0.0042500138282775875, + "tpp_threshold_5_total_metric": 0.09699999988079071, "tpp_threshold_5_intended_diff_only": 0.12220001220703125, - "tpp_threshold_5_unintended_diff_only": 0.02715001702308655, - "tpp_threshold_10_total_metric": 0.17530001401901246, + "tpp_threshold_5_unintended_diff_only": 0.02520001232624054, + "tpp_threshold_10_total_metric": 0.1768500179052353, "tpp_threshold_10_intended_diff_only": 0.23020002841949463, - "tpp_threshold_10_unintended_diff_only": 0.05490001440048218, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_10_unintended_diff_only": 0.05335001051425934, + "tpp_threshold_20_total_metric": 0.2408499985933304, + "tpp_threshold_20_intended_diff_only": 0.31660001277923583, + "tpp_threshold_20_unintended_diff_only": 0.07575001418590546, + "tpp_threshold_50_total_metric": 0.2548500090837479, + "tpp_threshold_50_intended_diff_only": 0.3928000211715698, + "tpp_threshold_50_unintended_diff_only": 0.13795001208782195, + "tpp_threshold_100_total_metric": 0.2173000156879425, + "tpp_threshold_100_intended_diff_only": 0.43680003881454466, + "tpp_threshold_100_unintended_diff_only": 0.21950002312660216, + "tpp_threshold_500_total_metric": 0.12845001816749574, + "tpp_threshold_500_intended_diff_only": 0.4674000382423401, + "tpp_threshold_500_unintended_diff_only": 0.33895002007484437 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.011099997162818908, - "tpp_threshold_2_intended_diff_only": 0.011399996280670167, - "tpp_threshold_2_unintended_diff_only": 0.0002999991178512573, - "tpp_threshold_5_total_metric": 0.016350004076957702, - "tpp_threshold_5_intended_diff_only": 0.018799996376037596, - "tpp_threshold_5_unintended_diff_only": 0.002449992299079895, - "tpp_threshold_10_total_metric": 0.031150013208389282, - "tpp_threshold_10_intended_diff_only": 0.033600008487701415, - "tpp_threshold_10_unintended_diff_only": 0.0024499952793121337, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.009549996256828308, + "tpp_threshold_2_intended_diff_only": 0.010599994659423828, + "tpp_threshold_2_unintended_diff_only": 0.00104999840259552, + "tpp_threshold_5_total_metric": 0.016249996423721314, + "tpp_threshold_5_intended_diff_only": 0.016399991512298585, + "tpp_threshold_5_unintended_diff_only": 0.0001499950885772705, + "tpp_threshold_10_total_metric": 0.03404999673366547, + "tpp_threshold_10_intended_diff_only": 0.03599998950958252, + "tpp_threshold_10_unintended_diff_only": 0.0019499927759170533, + "tpp_threshold_20_total_metric": 0.05954999327659607, + "tpp_threshold_20_intended_diff_only": 0.06379998922348022, + "tpp_threshold_20_unintended_diff_only": 0.004249995946884156, + "tpp_threshold_50_total_metric": 0.12364999949932098, + "tpp_threshold_50_intended_diff_only": 0.12799999713897706, + "tpp_threshold_50_unintended_diff_only": 0.004349997639656067, + "tpp_threshold_100_total_metric": 0.19660001397132873, + "tpp_threshold_100_intended_diff_only": 0.20620001554489137, + "tpp_threshold_100_unintended_diff_only": 0.009600001573562621, + "tpp_threshold_500_total_metric": 0.3683500200510025, + "tpp_threshold_500_intended_diff_only": 0.39960001707077025, + "tpp_threshold_500_unintended_diff_only": 0.031249997019767762 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.0020000189542770386, "tpp_threshold_10_total_metric": 0.006499990820884705, "tpp_threshold_10_intended_diff_only": 0.009000003337860107, - "tpp_threshold_10_unintended_diff_only": 0.002500012516975403 + "tpp_threshold_10_unintended_diff_only": 0.002500012516975403, + "tpp_threshold_20_total_metric": 0.2032499611377716, + "tpp_threshold_20_intended_diff_only": 0.23799997568130493, + "tpp_threshold_20_unintended_diff_only": 0.034750014543533325, + "tpp_threshold_50_total_metric": 0.2617499679327011, + "tpp_threshold_50_intended_diff_only": 0.35199999809265137, + "tpp_threshold_50_unintended_diff_only": 0.09025003015995026, + "tpp_threshold_100_total_metric": 0.24049995839595795, + "tpp_threshold_100_intended_diff_only": 0.4269999861717224, + "tpp_threshold_100_unintended_diff_only": 0.18650002777576447, + "tpp_threshold_500_total_metric": 0.1407500058412552, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.3152500241994858 }, "1": { - "tpp_threshold_2_total_metric": 0.0017500072717666626, - "tpp_threshold_2_intended_diff_only": 0.003000020980834961, - "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983, - "tpp_threshold_5_total_metric": 0.15999996662139893, + "tpp_threshold_2_total_metric": 0.0065000057220458984, + "tpp_threshold_2_intended_diff_only": 0.00700002908706665, + "tpp_threshold_2_unintended_diff_only": 0.000500023365020752, + "tpp_threshold_5_total_metric": 0.16374997794628143, "tpp_threshold_5_intended_diff_only": 0.18599998950958252, - "tpp_threshold_5_unintended_diff_only": 0.026000022888183594, - "tpp_threshold_10_total_metric": 0.20624999701976776, + "tpp_threshold_5_unintended_diff_only": 0.022250011563301086, + "tpp_threshold_10_total_metric": 0.20775000751018524, "tpp_threshold_10_intended_diff_only": 0.2670000195503235, - "tpp_threshold_10_unintended_diff_only": 0.060750022530555725 + "tpp_threshold_10_unintended_diff_only": 0.059250012040138245, + "tpp_threshold_20_total_metric": 0.23549999296665192, + "tpp_threshold_20_intended_diff_only": 0.30000001192092896, + "tpp_threshold_20_unintended_diff_only": 0.06450001895427704, + "tpp_threshold_50_total_metric": 0.24700000882148743, + "tpp_threshold_50_intended_diff_only": 0.3700000047683716, + "tpp_threshold_50_unintended_diff_only": 0.12299999594688416, + "tpp_threshold_100_total_metric": 0.2277500182390213, + "tpp_threshold_100_intended_diff_only": 0.42200005054473877, + "tpp_threshold_100_unintended_diff_only": 0.19425003230571747, + "tpp_threshold_500_total_metric": 0.11350002884864807, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.34550002217292786 }, "2": { - "tpp_threshold_2_total_metric": 0.1430000364780426, - "tpp_threshold_2_intended_diff_only": 0.15400004386901855, - "tpp_threshold_2_unintended_diff_only": 0.011000007390975952, - "tpp_threshold_5_total_metric": 0.20775003731250763, - "tpp_threshold_5_intended_diff_only": 0.2930000424385071, - "tpp_threshold_5_unintended_diff_only": 0.08525000512599945, - "tpp_threshold_10_total_metric": 0.21100002527236938, - "tpp_threshold_10_intended_diff_only": 0.3160000443458557, - "tpp_threshold_10_unintended_diff_only": 0.10500001907348633 + "tpp_threshold_2_total_metric": 0.1337500363588333, + "tpp_threshold_2_intended_diff_only": 0.15300005674362183, + "tpp_threshold_2_unintended_diff_only": 0.019250020384788513, + "tpp_threshold_5_total_metric": 0.21225003898143768, + "tpp_threshold_5_intended_diff_only": 0.29200005531311035, + "tpp_threshold_5_unintended_diff_only": 0.07975001633167267, + "tpp_threshold_10_total_metric": 0.21400001645088196, + "tpp_threshold_10_intended_diff_only": 0.31700003147125244, + "tpp_threshold_10_unintended_diff_only": 0.10300001502037048, + "tpp_threshold_20_total_metric": 0.1940000057220459, + "tpp_threshold_20_intended_diff_only": 0.35500001907348633, + "tpp_threshold_20_unintended_diff_only": 0.16100001335144043, + "tpp_threshold_50_total_metric": 0.16449999809265137, + "tpp_threshold_50_intended_diff_only": 0.37400001287460327, + "tpp_threshold_50_unintended_diff_only": 0.2095000147819519, + "tpp_threshold_100_total_metric": 0.11375002562999725, + "tpp_threshold_100_intended_diff_only": 0.40400004386901855, + "tpp_threshold_100_unintended_diff_only": 0.2902500182390213, + "tpp_threshold_500_total_metric": 0.06400001049041748, + "tpp_threshold_500_intended_diff_only": 0.4520000219345093, + "tpp_threshold_500_unintended_diff_only": 0.3880000114440918 }, "6": { - "tpp_threshold_2_total_metric": 0.011750012636184692, + "tpp_threshold_2_total_metric": 0.009499981999397278, "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": -0.002750009298324585, - "tpp_threshold_5_total_metric": 0.078249990940094, - "tpp_threshold_5_intended_diff_only": 0.09200000762939453, - "tpp_threshold_5_unintended_diff_only": 0.013750016689300537, - "tpp_threshold_10_total_metric": 0.27650004625320435, + "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704, + "tpp_threshold_5_total_metric": 0.08399999141693115, + "tpp_threshold_5_intended_diff_only": 0.09299999475479126, + "tpp_threshold_5_unintended_diff_only": 0.009000003337860107, + "tpp_threshold_10_total_metric": 0.2797500491142273, "tpp_threshold_10_intended_diff_only": 0.3290000557899475, - "tpp_threshold_10_unintended_diff_only": 0.052500009536743164 + "tpp_threshold_10_unintended_diff_only": 0.049250006675720215, + "tpp_threshold_20_total_metric": 0.33900001645088196, + "tpp_threshold_20_intended_diff_only": 0.409000039100647, + "tpp_threshold_20_unintended_diff_only": 0.07000002264976501, + "tpp_threshold_50_total_metric": 0.3007500171661377, + "tpp_threshold_50_intended_diff_only": 0.4790000319480896, + "tpp_threshold_50_unintended_diff_only": 0.1782500147819519, + "tpp_threshold_100_total_metric": 0.24150002002716064, + "tpp_threshold_100_intended_diff_only": 0.4830000400543213, + "tpp_threshold_100_unintended_diff_only": 0.24150002002716064, + "tpp_threshold_500_total_metric": 0.16449999809265137, + "tpp_threshold_500_intended_diff_only": 0.49400001764297485, + "tpp_threshold_500_unintended_diff_only": 0.3295000195503235 }, "9": { - "tpp_threshold_2_total_metric": 0.0050000399351119995, - "tpp_threshold_2_intended_diff_only": 0.006000041961669922, - "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_5_total_metric": 0.016250014305114746, + "tpp_threshold_2_total_metric": 0.008250012993812561, + "tpp_threshold_2_intended_diff_only": 0.009000003337860107, + "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.012000024318695068, "tpp_threshold_5_intended_diff_only": 0.025000035762786865, - "tpp_threshold_5_unintended_diff_only": 0.00875002145767212, - "tpp_threshold_10_total_metric": 0.17625001072883606, - "tpp_threshold_10_intended_diff_only": 0.23000001907348633, - "tpp_threshold_10_unintended_diff_only": 0.05375000834465027 + "tpp_threshold_5_unintended_diff_only": 0.013000011444091797, + "tpp_threshold_10_total_metric": 0.17625002562999725, + "tpp_threshold_10_intended_diff_only": 0.2290000319480896, + "tpp_threshold_10_unintended_diff_only": 0.052750006318092346, + "tpp_threshold_20_total_metric": 0.23250001668930054, + "tpp_threshold_20_intended_diff_only": 0.281000018119812, + "tpp_threshold_20_unintended_diff_only": 0.048500001430511475, + "tpp_threshold_50_total_metric": 0.3002500534057617, + "tpp_threshold_50_intended_diff_only": 0.3890000581741333, + "tpp_threshold_50_unintended_diff_only": 0.08875000476837158, + "tpp_threshold_100_total_metric": 0.2630000561475754, + "tpp_threshold_100_intended_diff_only": 0.44800007343292236, + "tpp_threshold_100_unintended_diff_only": 0.18500001728534698, + "tpp_threshold_500_total_metric": 0.15950004756450653, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.3165000230073929 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.008750006556510925, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": -0.004749998450279236, - "tpp_threshold_5_total_metric": 0.01374998688697815, - "tpp_threshold_5_intended_diff_only": 0.010999977588653564, - "tpp_threshold_5_unintended_diff_only": -0.002750009298324585, - "tpp_threshold_10_total_metric": 0.003500014543533325, - "tpp_threshold_10_intended_diff_only": 0.008000016212463379, - "tpp_threshold_10_unintended_diff_only": 0.004500001668930054 + "tpp_threshold_2_total_metric": 0.0012499988079071045, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_5_total_metric": 0.004749983549118042, + "tpp_threshold_5_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_10_total_metric": 0.004749983549118042, + "tpp_threshold_10_intended_diff_only": 0.006999969482421875, + "tpp_threshold_10_unintended_diff_only": 0.002249985933303833, + "tpp_threshold_20_total_metric": 0.013249978423118591, + "tpp_threshold_20_intended_diff_only": 0.015999972820281982, + "tpp_threshold_20_unintended_diff_only": 0.002749994397163391, + "tpp_threshold_50_total_metric": 0.052750006318092346, + "tpp_threshold_50_intended_diff_only": 0.050999999046325684, + "tpp_threshold_50_unintended_diff_only": -0.0017500072717666626, + "tpp_threshold_100_total_metric": 0.10175003111362457, + "tpp_threshold_100_intended_diff_only": 0.10900002717971802, + "tpp_threshold_100_unintended_diff_only": 0.007249996066093445, + "tpp_threshold_500_total_metric": 0.3980000168085098, + "tpp_threshold_500_intended_diff_only": 0.40700000524520874, + "tpp_threshold_500_unintended_diff_only": 0.008999988436698914 }, "2": { - "tpp_threshold_2_total_metric": 0.00575004518032074, - "tpp_threshold_2_intended_diff_only": 0.01100003719329834, - "tpp_threshold_2_unintended_diff_only": 0.0052499920129776, - "tpp_threshold_5_total_metric": 0.004750043153762817, - "tpp_threshold_5_intended_diff_only": 0.012000024318695068, - "tpp_threshold_5_unintended_diff_only": 0.007249981164932251, - "tpp_threshold_10_total_metric": 0.044000014662742615, - "tpp_threshold_10_intended_diff_only": 0.046000003814697266, - "tpp_threshold_10_unintended_diff_only": 0.001999989151954651 + "tpp_threshold_2_total_metric": 0.015000015497207642, + "tpp_threshold_2_intended_diff_only": 0.013000011444091797, + "tpp_threshold_2_unintended_diff_only": -0.0020000040531158447, + "tpp_threshold_5_total_metric": 0.003000020980834961, + "tpp_threshold_5_intended_diff_only": 0.009000003337860107, + "tpp_threshold_5_unintended_diff_only": 0.0059999823570251465, + "tpp_threshold_10_total_metric": 0.043750062584877014, + "tpp_threshold_10_intended_diff_only": 0.04700005054473877, + "tpp_threshold_10_unintended_diff_only": 0.0032499879598617554, + "tpp_threshold_20_total_metric": 0.07650002837181091, + "tpp_threshold_20_intended_diff_only": 0.08300000429153442, + "tpp_threshold_20_unintended_diff_only": 0.006499975919723511, + "tpp_threshold_50_total_metric": 0.14325003325939178, + "tpp_threshold_50_intended_diff_only": 0.156000018119812, + "tpp_threshold_50_unintended_diff_only": 0.012749984860420227, + "tpp_threshold_100_total_metric": 0.25325004756450653, + "tpp_threshold_100_intended_diff_only": 0.27000004053115845, + "tpp_threshold_100_unintended_diff_only": 0.016749992966651917, + "tpp_threshold_500_total_metric": 0.4085000604391098, + "tpp_threshold_500_intended_diff_only": 0.4360000491142273, + "tpp_threshold_500_unintended_diff_only": 0.027499988675117493 }, "3": { - "tpp_threshold_2_total_metric": 0.011749997735023499, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": -0.006750002503395081, - "tpp_threshold_5_total_metric": 0.014750003814697266, - "tpp_threshold_5_intended_diff_only": 0.013999998569488525, - "tpp_threshold_5_unintended_diff_only": -0.0007500052452087402, - "tpp_threshold_10_total_metric": 0.015000015497207642, - "tpp_threshold_10_intended_diff_only": 0.018000006675720215, - "tpp_threshold_10_unintended_diff_only": 0.0029999911785125732 + "tpp_threshold_2_total_metric": 0.003500014543533325, + "tpp_threshold_2_intended_diff_only": 0.003000020980834961, + "tpp_threshold_2_unintended_diff_only": -0.0004999935626983643, + "tpp_threshold_5_total_metric": 0.01000000536441803, + "tpp_threshold_5_intended_diff_only": 0.004999995231628418, + "tpp_threshold_5_unintended_diff_only": -0.005000010132789612, + "tpp_threshold_10_total_metric": 0.017749980092048645, + "tpp_threshold_10_intended_diff_only": 0.015999972820281982, + "tpp_threshold_10_unintended_diff_only": -0.0017500072717666626, + "tpp_threshold_20_total_metric": 0.013500019907951355, + "tpp_threshold_20_intended_diff_only": 0.017000019550323486, + "tpp_threshold_20_unintended_diff_only": 0.0034999996423721313, + "tpp_threshold_50_total_metric": 0.07150000333786011, + "tpp_threshold_50_intended_diff_only": 0.0690000057220459, + "tpp_threshold_50_unintended_diff_only": -0.002499997615814209, + "tpp_threshold_100_total_metric": 0.13300000131130219, + "tpp_threshold_100_intended_diff_only": 0.1380000114440918, + "tpp_threshold_100_unintended_diff_only": 0.005000010132789612, + "tpp_threshold_500_total_metric": 0.34925003349781036, + "tpp_threshold_500_intended_diff_only": 0.3680000305175781, + "tpp_threshold_500_unintended_diff_only": 0.01874999701976776 }, "5": { - "tpp_threshold_2_total_metric": -0.005750030279159546, - "tpp_threshold_2_intended_diff_only": 0.001999974250793457, - "tpp_threshold_2_unintended_diff_only": 0.007750004529953003, - "tpp_threshold_5_total_metric": 0.003999978303909302, - "tpp_threshold_5_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_5_unintended_diff_only": 0.0020000040531158447, - "tpp_threshold_10_total_metric": 0.011500000953674316, - "tpp_threshold_10_intended_diff_only": 0.013999998569488525, - "tpp_threshold_10_unintended_diff_only": 0.002499997615814209 + "tpp_threshold_2_total_metric": 0.0032499581575393677, + "tpp_threshold_2_intended_diff_only": 0.0029999613761901855, + "tpp_threshold_2_unintended_diff_only": -0.00024999678134918213, + "tpp_threshold_5_total_metric": 0.015999972820281982, + "tpp_threshold_5_intended_diff_only": 0.014999985694885254, + "tpp_threshold_5_unintended_diff_only": -0.0009999871253967285, + "tpp_threshold_10_total_metric": 0.011999964714050293, + "tpp_threshold_10_intended_diff_only": 0.015999972820281982, + "tpp_threshold_10_unintended_diff_only": 0.0040000081062316895, + "tpp_threshold_20_total_metric": 0.04399995505809784, + "tpp_threshold_20_intended_diff_only": 0.04399996995925903, + "tpp_threshold_20_unintended_diff_only": 1.4901161193847656e-08, + "tpp_threshold_50_total_metric": 0.12949998676776886, + "tpp_threshold_50_intended_diff_only": 0.13499999046325684, + "tpp_threshold_50_unintended_diff_only": 0.005500003695487976, + "tpp_threshold_100_total_metric": 0.21249999105930328, + "tpp_threshold_100_intended_diff_only": 0.22200000286102295, + "tpp_threshold_100_unintended_diff_only": 0.009500011801719666, + "tpp_threshold_500_total_metric": 0.34349997341632843, + "tpp_threshold_500_intended_diff_only": 0.4139999747276306, + "tpp_threshold_500_unintended_diff_only": 0.07050000131130219 }, "6": { - "tpp_threshold_2_total_metric": 0.034999966621398926, - "tpp_threshold_2_intended_diff_only": 0.034999966621398926, - "tpp_threshold_2_unintended_diff_only": 0.0, - "tpp_threshold_5_total_metric": 0.04450000822544098, - "tpp_threshold_5_intended_diff_only": 0.050999999046325684, - "tpp_threshold_5_unintended_diff_only": 0.006499990820884705, - "tpp_threshold_10_total_metric": 0.08175002038478851, - "tpp_threshold_10_intended_diff_only": 0.0820000171661377, - "tpp_threshold_10_unintended_diff_only": 0.00024999678134918213 + "tpp_threshold_2_total_metric": 0.0247499942779541, + "tpp_threshold_2_intended_diff_only": 0.02899998426437378, + "tpp_threshold_2_unintended_diff_only": 0.004249989986419678, + "tpp_threshold_5_total_metric": 0.04749999940395355, + "tpp_threshold_5_intended_diff_only": 0.046999990940093994, + "tpp_threshold_5_unintended_diff_only": -0.0005000084638595581, + "tpp_threshold_10_total_metric": 0.09199999272823334, + "tpp_threshold_10_intended_diff_only": 0.09399998188018799, + "tpp_threshold_10_unintended_diff_only": 0.001999989151954651, + "tpp_threshold_20_total_metric": 0.15049998462200165, + "tpp_threshold_20_intended_diff_only": 0.1589999794960022, + "tpp_threshold_20_unintended_diff_only": 0.00849999487400055, + "tpp_threshold_50_total_metric": 0.22124996781349182, + "tpp_threshold_50_intended_diff_only": 0.22899997234344482, + "tpp_threshold_50_unintended_diff_only": 0.007750004529953003, + "tpp_threshold_100_total_metric": 0.2824999988079071, + "tpp_threshold_100_intended_diff_only": 0.2919999957084656, + "tpp_threshold_100_unintended_diff_only": 0.009499996900558472, + "tpp_threshold_500_total_metric": 0.3425000160932541, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.030500009655952454 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-134277-tokens-550000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-134277-tokens-550000000/scr/saebench_scr_custom_sae_eval_results.json index dd2bb4784230389927aa5fe4ca832c2a9f7cf420..4bf39ada799657ebc716e49e97eca01059a467c3 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-134277-tokens-550000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-134277-tokens-550000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,31 +69,31 @@ ] } }, - "eval_id": "4a893800-28cc-4140-a764-87637efe1b4a", - "datetime_epoch_millis": 1776883995505, + "eval_id": "52d0a3c8-3d5e-4e60-9281-3c3ff4323bb1", + "datetime_epoch_millis": 1777315626029, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.27679038925488475, "scr_metric_threshold_2": 0.11448470285338772, "scr_dir2_threshold_2": 0.11011504535000179, "scr_dir1_threshold_5": 0.2516291575551625, - "scr_metric_threshold_5": 0.18619278477797802, - "scr_dir2_threshold_5": 0.1800000058909232, + "scr_metric_threshold_5": 0.1857947007445432, + "scr_dir2_threshold_5": 0.17960192185748838, "scr_dir1_threshold_10": 0.25503683173522657, "scr_metric_threshold_10": 0.23262178467385908, "scr_dir2_threshold_10": 0.2263658672576473, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.022267688790582715, + "scr_metric_threshold_20": 0.2600485209905197, + "scr_dir2_threshold_20": 0.254184610575226, + "scr_dir1_threshold_50": 0.22170874833388707, + "scr_metric_threshold_50": 0.32764610137984584, + "scr_dir2_threshold_50": 0.3239814665851593, + "scr_dir1_threshold_100": 0.0005200397136160759, + "scr_metric_threshold_100": 0.33646740091203153, + "scr_dir2_threshold_100": 0.3285593847314374, + "scr_dir1_threshold_500": -0.730662973340079, + "scr_metric_threshold_500": 0.14416851198599903, + "scr_dir2_threshold_500": 0.11886596932476784 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.5641028776049328, "scr_metric_threshold_10": 0.08290156000473131, "scr_dir2_threshold_10": 0.08290156000473131, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.08974395223070357, + "scr_metric_threshold_20": 0.1398964114610147, + "scr_dir2_threshold_20": 0.1398964114610147, + "scr_dir1_threshold_50": 0.3205132417816285, + "scr_metric_threshold_50": 0.27461138856264183, + "scr_dir2_threshold_50": 0.27461138856264183, + "scr_dir1_threshold_100": 0.4487182328294704, + "scr_metric_threshold_100": 0.33160624001892525, + "scr_dir2_threshold_100": 0.33160624001892525, + "scr_dir1_threshold_500": -2.0769228417963004, + "scr_metric_threshold_500": 0.09585491427895527, + "scr_dir2_threshold_500": 0.09585491427895527 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.23484857036936357, "scr_metric_threshold_10": 0.33639145849092716, "scr_dir2_threshold_10": 0.33639145849092716, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.022727426664854444, + "scr_metric_threshold_20": 0.20795111325793134, + "scr_dir2_threshold_20": 0.20795111325793134, + "scr_dir1_threshold_50": -0.08333314518740012, + "scr_metric_threshold_50": 0.5198776920062339, + "scr_dir2_threshold_50": 0.5198776920062339, + "scr_dir1_threshold_100": 0.04545485332970889, + "scr_metric_threshold_100": 0.5137615632091722, + "scr_dir2_threshold_100": 0.5137615632091722, + "scr_dir1_threshold_500": -0.3939392023726256, + "scr_metric_threshold_500": -0.24464825059467932, + "scr_dir2_threshold_500": -0.24464825059467932 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.31249965075409175, "scr_metric_threshold_10": 0.1482412214577287, "scr_dir2_threshold_10": 0.1482412214577287, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.5, + "scr_metric_threshold_20": 0.16582923152102794, + "scr_dir2_threshold_20": 0.16582923152102794, + "scr_dir1_threshold_50": 0.3906245634426147, + "scr_metric_threshold_50": 0.13065336115482035, + "scr_dir2_threshold_50": 0.13065336115482035, + "scr_dir1_threshold_100": -1.2656243306120092, + "scr_metric_threshold_100": 0.037688550204100946, + "scr_dir2_threshold_100": 0.037688550204100946, + "scr_dir1_threshold_500": -2.7187499417923484, + "scr_metric_threshold_500": -0.13316574147193197, + "scr_dir2_threshold_500": -0.13316574147193197 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -171,23 +175,23 @@ "scr_metric_threshold_2": 0.03184710232218929, "scr_dir2_threshold_2": 0.03184710232218929, "scr_dir1_threshold_5": 0.04697966441382428, - "scr_metric_threshold_5": 0.10509551359270591, - "scr_dir2_threshold_5": 0.10509551359270591, + "scr_metric_threshold_5": 0.10191084132522761, + "scr_dir2_threshold_5": 0.10191084132522761, "scr_dir1_threshold_10": -0.10067145237328512, "scr_metric_threshold_10": 0.1305732713799386, "scr_dir2_threshold_10": 0.1305732713799386, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.8791949772082223, + "scr_metric_threshold_20": 0.14968149480851153, + "scr_dir2_threshold_20": 0.14968149480851153, + "scr_dir1_threshold_50": 0.12080502279177775, + "scr_metric_threshold_50": 0.14968149480851153, + "scr_dir2_threshold_50": 0.14968149480851153, + "scr_dir1_threshold_100": -0.36912759195217226, + "scr_metric_threshold_100": 0.1624203737021279, + "scr_dir2_threshold_100": 0.1624203737021279, + "scr_dir1_threshold_500": -1.738255583935547, + "scr_metric_threshold_500": -0.509554016802435, + "scr_dir2_threshold_500": -0.509554016802435 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.08421069803000372, "scr_metric_threshold_10": 0.4166666863771947, "scr_dir2_threshold_10": 0.4166666863771947, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.08421069803000372, + "scr_metric_threshold_20": 0.5357141758984867, + "scr_dir2_threshold_20": 0.5357141758984867, + "scr_dir1_threshold_50": -0.19999993725826787, + "scr_metric_threshold_50": 0.6230158101299026, + "scr_dir2_threshold_50": 0.6230158101299026, + "scr_dir1_threshold_100": -0.21578952156605874, + "scr_metric_threshold_100": 0.6349206536925663, + "scr_dir2_threshold_100": 0.6349206536925663, + "scr_dir1_threshold_500": -0.16842108235134678, + "scr_metric_threshold_500": 0.6190476077844602, + "scr_dir2_threshold_500": 0.6190476077844602 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.3825136878996036, "scr_metric_threshold_10": 0.14885507905463402, "scr_dir2_threshold_10": 0.14885507905463402, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.4043716594522235, + "scr_metric_threshold_20": 0.19847321124088896, + "scr_dir2_threshold_20": 0.19847321124088896, + "scr_dir1_threshold_50": 0.4808743970321442, + "scr_metric_threshold_50": 0.28625946991455836, + "scr_dir2_threshold_50": 0.28625946991455836, + "scr_dir1_threshold_100": 0.4535520140184937, + "scr_metric_threshold_100": 0.28625946991455836, + "scr_dir2_threshold_100": 0.28625946991455836, + "scr_dir1_threshold_500": 0.5300547515984144, + "scr_metric_threshold_500": 0.38931293368346886, + "scr_dir2_threshold_500": 0.38931293368346886 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.3860464716550849, "scr_metric_threshold_10": 0.4206008506837011, "scr_dir2_threshold_10": 0.4206008506837011, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3720928878639972, + "scr_metric_threshold_20": 0.4549356870945189, + "scr_dir2_threshold_20": 0.4549356870945189, + "scr_dir1_threshold_50": 0.49767431029119774, + "scr_metric_threshold_50": 0.3905580606846963, + "scr_dir2_threshold_50": 0.3905580606846963, + "scr_dir1_threshold_100": 0.5767441563892566, + "scr_metric_threshold_100": 0.3948498512825616, + "scr_dir2_threshold_100": 0.3948498512825616, + "scr_dir1_threshold_500": 0.2651161514146259, + "scr_metric_threshold_500": 0.48068668649565843, + "scr_dir2_threshold_500": 0.48068668649565843 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.17674414994201715, "scr_metric_threshold_10": 0.17674414994201715, "scr_dir2_threshold_10": 0.12669681061232305, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.227906842641778, + "scr_metric_threshold_20": 0.227906842641778, + "scr_dir2_threshold_20": 0.1809955593194283, + "scr_dir1_threshold_50": 0.24651153377740184, + "scr_metric_threshold_50": 0.24651153377740184, + "scr_dir2_threshold_50": 0.2171944554199093, + "scr_dir1_threshold_100": 0.3302325052722394, + "scr_metric_threshold_100": 0.3302325052722394, + "scr_dir2_threshold_100": 0.2669683758274864, + "scr_dir1_threshold_500": 0.4558139625144957, + "scr_metric_threshold_500": 0.4558139625144957, + "scr_dir2_threshold_500": 0.2533936212246461 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-134277-tokens-550000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-134277-tokens-550000000/tpp/saebench_tpp_custom_sae_eval_results.json index 9f3c45fcf1169cdc29199f8e0992add731d93645..d2c0d1ec911a465bca4e9febbf179e4bce05ef56 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-134277-tokens-550000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-134277-tokens-550000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "a32b1258-f85a-4ba5-9120-84849dd3f983", - "datetime_epoch_millis": 1776884158771, + "eval_id": "607f6535-ca12-468e-9875-eb95b41b0228", + "datetime_epoch_millis": 1777315903998, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.021700002253055573, - "tpp_threshold_2_intended_diff_only": 0.022700005769729616, - "tpp_threshold_2_unintended_diff_only": 0.0010000035166740418, - "tpp_threshold_5_total_metric": 0.061674997210502625, - "tpp_threshold_5_intended_diff_only": 0.07070000171661377, - "tpp_threshold_5_unintended_diff_only": 0.009025004506111146, - "tpp_threshold_10_total_metric": 0.10635001063346863, - "tpp_threshold_10_intended_diff_only": 0.1358000159263611, - "tpp_threshold_10_unintended_diff_only": 0.029450005292892455, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.020499995350837706, + "tpp_threshold_2_intended_diff_only": 0.02290000319480896, + "tpp_threshold_2_unintended_diff_only": 0.0024000078439712523, + "tpp_threshold_5_total_metric": 0.06154999881982803, + "tpp_threshold_5_intended_diff_only": 0.06920000314712524, + "tpp_threshold_5_unintended_diff_only": 0.00765000432729721, + "tpp_threshold_10_total_metric": 0.10792499929666519, + "tpp_threshold_10_intended_diff_only": 0.13700000643730165, + "tpp_threshold_10_unintended_diff_only": 0.029075007140636447, + "tpp_threshold_20_total_metric": 0.14760000854730607, + "tpp_threshold_20_intended_diff_only": 0.1952000141143799, + "tpp_threshold_20_unintended_diff_only": 0.04760000556707382, + "tpp_threshold_50_total_metric": 0.17302500754594802, + "tpp_threshold_50_intended_diff_only": 0.254500013589859, + "tpp_threshold_50_unintended_diff_only": 0.08147500604391099, + "tpp_threshold_100_total_metric": 0.1921500101685524, + "tpp_threshold_100_intended_diff_only": 0.3089000165462494, + "tpp_threshold_100_unintended_diff_only": 0.11675000637769699, + "tpp_threshold_500_total_metric": 0.24855002611875535, + "tpp_threshold_500_intended_diff_only": 0.4323000371456146, + "tpp_threshold_500_unintended_diff_only": 0.18375001102685928 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.03174999058246612, - "tpp_threshold_2_intended_diff_only": 0.033399999141693115, - "tpp_threshold_2_unintended_diff_only": 0.0016500085592269897, - "tpp_threshold_5_total_metric": 0.10250000357627868, + "tpp_threshold_2_total_metric": 0.030799999833106995, + "tpp_threshold_2_intended_diff_only": 0.03420001268386841, + "tpp_threshold_2_unintended_diff_only": 0.0034000128507614136, + "tpp_threshold_5_total_metric": 0.10330000221729278, "tpp_threshold_5_intended_diff_only": 0.11820001602172851, - "tpp_threshold_5_unintended_diff_only": 0.01570001244544983, - "tpp_threshold_10_total_metric": 0.18215001821517945, + "tpp_threshold_5_unintended_diff_only": 0.01490001380443573, + "tpp_threshold_10_total_metric": 0.1831500142812729, "tpp_threshold_10_intended_diff_only": 0.23820003271102905, - "tpp_threshold_10_unintended_diff_only": 0.05605001449584961, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_10_unintended_diff_only": 0.05505001842975617, + "tpp_threshold_20_total_metric": 0.24570001363754274, + "tpp_threshold_20_intended_diff_only": 0.3366000294685364, + "tpp_threshold_20_unintended_diff_only": 0.09090001583099365, + "tpp_threshold_50_total_metric": 0.24230002462863923, + "tpp_threshold_50_intended_diff_only": 0.4008000373840332, + "tpp_threshold_50_unintended_diff_only": 0.158500012755394, + "tpp_threshold_100_total_metric": 0.2114500105381012, + "tpp_threshold_100_intended_diff_only": 0.43480002880096436, + "tpp_threshold_100_unintended_diff_only": 0.22335001826286316, + "tpp_threshold_500_total_metric": 0.12215001881122589, + "tpp_threshold_500_intended_diff_only": 0.4670000433921814, + "tpp_threshold_500_unintended_diff_only": 0.3448500245809555 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.011650013923645019, - "tpp_threshold_2_intended_diff_only": 0.012000012397766113, - "tpp_threshold_2_unintended_diff_only": 0.00034999847412109375, - "tpp_threshold_5_total_metric": 0.020849990844726562, - "tpp_threshold_5_intended_diff_only": 0.023199987411499024, - "tpp_threshold_5_unintended_diff_only": 0.002349996566772461, - "tpp_threshold_10_total_metric": 0.030550003051757812, - "tpp_threshold_10_intended_diff_only": 0.033399999141693115, - "tpp_threshold_10_unintended_diff_only": 0.0028499960899353027, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.010199990868568421, + "tpp_threshold_2_intended_diff_only": 0.011599993705749512, + "tpp_threshold_2_unintended_diff_only": 0.0014000028371810914, + "tpp_threshold_5_total_metric": 0.01979999542236328, + "tpp_threshold_5_intended_diff_only": 0.02019999027252197, + "tpp_threshold_5_unintended_diff_only": 0.0003999948501586914, + "tpp_threshold_10_total_metric": 0.03269998431205749, + "tpp_threshold_10_intended_diff_only": 0.03579998016357422, + "tpp_threshold_10_unintended_diff_only": 0.003099995851516724, + "tpp_threshold_20_total_metric": 0.0495000034570694, + "tpp_threshold_20_intended_diff_only": 0.053799998760223386, + "tpp_threshold_20_unintended_diff_only": 0.0042999953031539915, + "tpp_threshold_50_total_metric": 0.10374999046325684, + "tpp_threshold_50_intended_diff_only": 0.10819998979568482, + "tpp_threshold_50_unintended_diff_only": 0.004449999332427979, + "tpp_threshold_100_total_metric": 0.1728500097990036, + "tpp_threshold_100_intended_diff_only": 0.18300000429153443, + "tpp_threshold_100_unintended_diff_only": 0.010149994492530822, + "tpp_threshold_500_total_metric": 0.3749500334262848, + "tpp_threshold_500_intended_diff_only": 0.39760003089904783, + "tpp_threshold_500_unintended_diff_only": 0.022649997472763063 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.0020000189542770386, "tpp_threshold_10_total_metric": 0.013249963521957397, "tpp_threshold_10_intended_diff_only": 0.015999972820281982, - "tpp_threshold_10_unintended_diff_only": 0.002750009298324585 + "tpp_threshold_10_unintended_diff_only": 0.002750009298324585, + "tpp_threshold_20_total_metric": 0.23674999177455902, + "tpp_threshold_20_intended_diff_only": 0.30800002813339233, + "tpp_threshold_20_unintended_diff_only": 0.07125003635883331, + "tpp_threshold_50_total_metric": 0.2720000147819519, + "tpp_threshold_50_intended_diff_only": 0.3970000147819519, + "tpp_threshold_50_unintended_diff_only": 0.125, + "tpp_threshold_100_total_metric": 0.2347499430179596, + "tpp_threshold_100_intended_diff_only": 0.43699997663497925, + "tpp_threshold_100_unintended_diff_only": 0.20225003361701965, + "tpp_threshold_500_total_metric": 0.14925000071525574, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.30675002932548523 }, "1": { - "tpp_threshold_2_total_metric": -2.9802322387695312e-08, - "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162, - "tpp_threshold_5_total_metric": 0.15024997293949127, + "tpp_threshold_2_total_metric": 0.004499971866607666, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 0.000500023365020752, + "tpp_threshold_5_total_metric": 0.15549997985363007, "tpp_threshold_5_intended_diff_only": 0.17599999904632568, - "tpp_threshold_5_unintended_diff_only": 0.02575002610683441, - "tpp_threshold_10_total_metric": 0.200500026345253, + "tpp_threshold_5_unintended_diff_only": 0.020500019192695618, + "tpp_threshold_10_total_metric": 0.20125000178813934, "tpp_threshold_10_intended_diff_only": 0.26100003719329834, - "tpp_threshold_10_unintended_diff_only": 0.06050001084804535 + "tpp_threshold_10_unintended_diff_only": 0.059750035405159, + "tpp_threshold_20_total_metric": 0.23274998366832733, + "tpp_threshold_20_intended_diff_only": 0.296999990940094, + "tpp_threshold_20_unintended_diff_only": 0.06425000727176666, + "tpp_threshold_50_total_metric": 0.2394999861717224, + "tpp_threshold_50_intended_diff_only": 0.3700000047683716, + "tpp_threshold_50_unintended_diff_only": 0.13050001859664917, + "tpp_threshold_100_total_metric": 0.216000035405159, + "tpp_threshold_100_intended_diff_only": 0.40800005197525024, + "tpp_threshold_100_unintended_diff_only": 0.19200001657009125, + "tpp_threshold_500_total_metric": 0.10525001585483551, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.3537500351667404 }, "2": { - "tpp_threshold_2_total_metric": 0.13449999690055847, - "tpp_threshold_2_intended_diff_only": 0.14300000667572021, - "tpp_threshold_2_unintended_diff_only": 0.008500009775161743, - "tpp_threshold_5_total_metric": 0.22200004756450653, - "tpp_threshold_5_intended_diff_only": 0.2510000467300415, - "tpp_threshold_5_unintended_diff_only": 0.028999999165534973, - "tpp_threshold_10_total_metric": 0.22675003111362457, - "tpp_threshold_10_intended_diff_only": 0.315000057220459, - "tpp_threshold_10_unintended_diff_only": 0.08825002610683441 + "tpp_threshold_2_total_metric": 0.12450003623962402, + "tpp_threshold_2_intended_diff_only": 0.14000004529953003, + "tpp_threshold_2_unintended_diff_only": 0.015500009059906006, + "tpp_threshold_5_total_metric": 0.21875004470348358, + "tpp_threshold_5_intended_diff_only": 0.2500000596046448, + "tpp_threshold_5_unintended_diff_only": 0.031250014901161194, + "tpp_threshold_10_total_metric": 0.22950002551078796, + "tpp_threshold_10_intended_diff_only": 0.3160000443458557, + "tpp_threshold_10_unintended_diff_only": 0.08650001883506775, + "tpp_threshold_20_total_metric": 0.2030000388622284, + "tpp_threshold_20_intended_diff_only": 0.35700005292892456, + "tpp_threshold_20_unintended_diff_only": 0.15400001406669617, + "tpp_threshold_50_total_metric": 0.15275003015995026, + "tpp_threshold_50_intended_diff_only": 0.3760000467300415, + "tpp_threshold_50_unintended_diff_only": 0.22325001657009125, + "tpp_threshold_100_total_metric": 0.12000003457069397, + "tpp_threshold_100_intended_diff_only": 0.41200006008148193, + "tpp_threshold_100_unintended_diff_only": 0.29200002551078796, + "tpp_threshold_500_total_metric": 0.06074999272823334, + "tpp_threshold_500_intended_diff_only": 0.4520000219345093, + "tpp_threshold_500_unintended_diff_only": 0.39125002920627594 }, "6": { - "tpp_threshold_2_total_metric": 0.011000022292137146, + "tpp_threshold_2_total_metric": 0.008750006556510925, "tpp_threshold_2_intended_diff_only": 0.008000016212463379, - "tpp_threshold_2_unintended_diff_only": -0.003000006079673767, - "tpp_threshold_5_total_metric": 0.10750003159046173, - "tpp_threshold_5_intended_diff_only": 0.12200003862380981, - "tpp_threshold_5_unintended_diff_only": 0.014500007033348083, - "tpp_threshold_10_total_metric": 0.29575005173683167, + "tpp_threshold_2_unintended_diff_only": -0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.11275002360343933, + "tpp_threshold_5_intended_diff_only": 0.12300002574920654, + "tpp_threshold_5_unintended_diff_only": 0.010250002145767212, + "tpp_threshold_10_total_metric": 0.2980000525712967, "tpp_threshold_10_intended_diff_only": 0.37000006437301636, - "tpp_threshold_10_unintended_diff_only": 0.07425001263618469 + "tpp_threshold_10_unintended_diff_only": 0.07200001180171967, + "tpp_threshold_20_total_metric": 0.32750000059604645, + "tpp_threshold_20_intended_diff_only": 0.44300001859664917, + "tpp_threshold_20_unintended_diff_only": 0.11550001800060272, + "tpp_threshold_50_total_metric": 0.28025004267692566, + "tpp_threshold_50_intended_diff_only": 0.47700005769729614, + "tpp_threshold_50_unintended_diff_only": 0.19675001502037048, + "tpp_threshold_100_total_metric": 0.2265000343322754, + "tpp_threshold_100_intended_diff_only": 0.4830000400543213, + "tpp_threshold_100_unintended_diff_only": 0.2565000057220459, + "tpp_threshold_500_total_metric": 0.14800003170967102, + "tpp_threshold_500_intended_diff_only": 0.4920000433921814, + "tpp_threshold_500_unintended_diff_only": 0.3440000116825104 }, "9": { - "tpp_threshold_2_total_metric": 0.003999993205070496, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_total_metric": 0.0070000141859054565, + "tpp_threshold_2_intended_diff_only": 0.008000016212463379, "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_5_total_metric": 0.017750024795532227, + "tpp_threshold_5_total_metric": 0.014500021934509277, "tpp_threshold_5_intended_diff_only": 0.025000035762786865, - "tpp_threshold_5_unintended_diff_only": 0.007250010967254639, - "tpp_threshold_10_total_metric": 0.1745000183582306, - "tpp_threshold_10_intended_diff_only": 0.2290000319480896, - "tpp_threshold_10_unintended_diff_only": 0.05450001358985901 + "tpp_threshold_5_unintended_diff_only": 0.010500013828277588, + "tpp_threshold_10_total_metric": 0.17375002801418304, + "tpp_threshold_10_intended_diff_only": 0.22800004482269287, + "tpp_threshold_10_unintended_diff_only": 0.05425001680850983, + "tpp_threshold_20_total_metric": 0.22850005328655243, + "tpp_threshold_20_intended_diff_only": 0.2780000567436218, + "tpp_threshold_20_unintended_diff_only": 0.0495000034570694, + "tpp_threshold_50_total_metric": 0.2670000493526459, + "tpp_threshold_50_intended_diff_only": 0.3840000629425049, + "tpp_threshold_50_unintended_diff_only": 0.11700001358985901, + "tpp_threshold_100_total_metric": 0.26000000536441803, + "tpp_threshold_100_intended_diff_only": 0.43400001525878906, + "tpp_threshold_100_unintended_diff_only": 0.17400000989437103, + "tpp_threshold_500_total_metric": 0.14750005304813385, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.32850001752376556 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.007750019431114197, - "tpp_threshold_2_intended_diff_only": 0.003000020980834961, - "tpp_threshold_2_unintended_diff_only": -0.004749998450279236, - "tpp_threshold_5_total_metric": 0.013499975204467773, - "tpp_threshold_5_intended_diff_only": 0.010999977588653564, - "tpp_threshold_5_unintended_diff_only": -0.002499997615814209, - "tpp_threshold_10_total_metric": 0.0024999678134918213, + "tpp_threshold_2_total_metric": 0.002249985933303833, + "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_5_total_metric": 0.0059999823570251465, + "tpp_threshold_5_intended_diff_only": 0.006999969482421875, + "tpp_threshold_5_unintended_diff_only": 0.0009999871253967285, + "tpp_threshold_10_total_metric": 0.005249977111816406, "tpp_threshold_10_intended_diff_only": 0.006999969482421875, - "tpp_threshold_10_unintended_diff_only": 0.004500001668930054 + "tpp_threshold_10_unintended_diff_only": 0.0017499923706054688, + "tpp_threshold_20_total_metric": 0.014000028371810913, + "tpp_threshold_20_intended_diff_only": 0.017000019550323486, + "tpp_threshold_20_unintended_diff_only": 0.0029999911785125732, + "tpp_threshold_50_total_metric": 0.03949999809265137, + "tpp_threshold_50_intended_diff_only": 0.03799998760223389, + "tpp_threshold_50_unintended_diff_only": -0.0015000104904174805, + "tpp_threshold_100_total_metric": 0.10574999451637268, + "tpp_threshold_100_intended_diff_only": 0.1119999885559082, + "tpp_threshold_100_unintended_diff_only": 0.0062499940395355225, + "tpp_threshold_500_total_metric": 0.3827500343322754, + "tpp_threshold_500_intended_diff_only": 0.39500004053115845, + "tpp_threshold_500_unintended_diff_only": 0.012250006198883057 }, "2": { - "tpp_threshold_2_total_metric": 0.0020000338554382324, - "tpp_threshold_2_intended_diff_only": 0.00700002908706665, - "tpp_threshold_2_unintended_diff_only": 0.004999995231628418, - "tpp_threshold_5_total_metric": 0.008250057697296143, - "tpp_threshold_5_intended_diff_only": 0.016000032424926758, - "tpp_threshold_5_unintended_diff_only": 0.007749974727630615, - "tpp_threshold_10_total_metric": 0.04075002670288086, - "tpp_threshold_10_intended_diff_only": 0.04400002956390381, - "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492 + "tpp_threshold_2_total_metric": 0.011250004172325134, + "tpp_threshold_2_intended_diff_only": 0.009000003337860107, + "tpp_threshold_2_unintended_diff_only": -0.002250000834465027, + "tpp_threshold_5_total_metric": 0.006500020623207092, + "tpp_threshold_5_intended_diff_only": 0.013000011444091797, + "tpp_threshold_5_unintended_diff_only": 0.006499990820884705, + "tpp_threshold_10_total_metric": 0.041750043630599976, + "tpp_threshold_10_intended_diff_only": 0.0480000376701355, + "tpp_threshold_10_unintended_diff_only": 0.0062499940395355225, + "tpp_threshold_20_total_metric": 0.06575003266334534, + "tpp_threshold_20_intended_diff_only": 0.0690000057220459, + "tpp_threshold_20_unintended_diff_only": 0.0032499730587005615, + "tpp_threshold_50_total_metric": 0.10825006663799286, + "tpp_threshold_50_intended_diff_only": 0.11600005626678467, + "tpp_threshold_50_unintended_diff_only": 0.007749989628791809, + "tpp_threshold_100_total_metric": 0.18625007569789886, + "tpp_threshold_100_intended_diff_only": 0.20000004768371582, + "tpp_threshold_100_unintended_diff_only": 0.013749971985816956, + "tpp_threshold_500_total_metric": 0.40400007367134094, + "tpp_threshold_500_intended_diff_only": 0.43000006675720215, + "tpp_threshold_500_unintended_diff_only": 0.025999993085861206 }, "3": { - "tpp_threshold_2_total_metric": 0.013750031590461731, - "tpp_threshold_2_intended_diff_only": 0.00700002908706665, - "tpp_threshold_2_unintended_diff_only": -0.006750002503395081, - "tpp_threshold_5_total_metric": 0.01825001835823059, - "tpp_threshold_5_intended_diff_only": 0.017000019550323486, - "tpp_threshold_5_unintended_diff_only": -0.0012499988079071045, - "tpp_threshold_10_total_metric": 0.015000015497207642, - "tpp_threshold_10_intended_diff_only": 0.018000006675720215, - "tpp_threshold_10_unintended_diff_only": 0.0029999911785125732 + "tpp_threshold_2_total_metric": 0.004500001668930054, + "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_2_unintended_diff_only": -0.0004999935626983643, + "tpp_threshold_5_total_metric": 0.013000011444091797, + "tpp_threshold_5_intended_diff_only": 0.008000016212463379, + "tpp_threshold_5_unintended_diff_only": -0.004999995231628418, + "tpp_threshold_10_total_metric": 0.015749990940093994, + "tpp_threshold_10_intended_diff_only": 0.014999985694885254, + "tpp_threshold_10_unintended_diff_only": -0.0007500052452087402, + "tpp_threshold_20_total_metric": 0.013750031590461731, + "tpp_threshold_20_intended_diff_only": 0.017000019550323486, + "tpp_threshold_20_unintended_diff_only": 0.0032499879598617554, + "tpp_threshold_50_total_metric": 0.04874996840953827, + "tpp_threshold_50_intended_diff_only": 0.04799997806549072, + "tpp_threshold_50_unintended_diff_only": -0.0007499903440475464, + "tpp_threshold_100_total_metric": 0.11499997973442078, + "tpp_threshold_100_intended_diff_only": 0.12199997901916504, + "tpp_threshold_100_unintended_diff_only": 0.006999999284744263, + "tpp_threshold_500_total_metric": 0.3657500296831131, + "tpp_threshold_500_intended_diff_only": 0.38600003719329834, + "tpp_threshold_500_unintended_diff_only": 0.02025000751018524 }, "5": { - "tpp_threshold_2_total_metric": -0.006750017404556274, - "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 0.007750004529953003, - "tpp_threshold_5_total_metric": 0.0029999464750289917, - "tpp_threshold_5_intended_diff_only": 0.003999948501586914, - "tpp_threshold_5_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_10_total_metric": 0.007750004529953003, - "tpp_threshold_10_intended_diff_only": 0.009999990463256836, - "tpp_threshold_10_unintended_diff_only": 0.002249985933303833 + "tpp_threshold_2_total_metric": 0.00449998676776886, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 0.0005000084638595581, + "tpp_threshold_5_total_metric": 0.010249942541122437, + "tpp_threshold_5_intended_diff_only": 0.008999943733215332, + "tpp_threshold_5_unintended_diff_only": -0.0012499988079071045, + "tpp_threshold_10_total_metric": 0.003999948501586914, + "tpp_threshold_10_intended_diff_only": 0.008999943733215332, + "tpp_threshold_10_unintended_diff_only": 0.004999995231628418, + "tpp_threshold_20_total_metric": 0.01949995756149292, + "tpp_threshold_20_intended_diff_only": 0.0209999680519104, + "tpp_threshold_20_unintended_diff_only": 0.0015000104904174805, + "tpp_threshold_50_total_metric": 0.0887499451637268, + "tpp_threshold_50_intended_diff_only": 0.09599995613098145, + "tpp_threshold_50_unintended_diff_only": 0.007250010967254639, + "tpp_threshold_100_total_metric": 0.16875000298023224, + "tpp_threshold_100_intended_diff_only": 0.18000000715255737, + "tpp_threshold_100_unintended_diff_only": 0.011250004172325134, + "tpp_threshold_500_total_metric": 0.37824998795986176, + "tpp_threshold_500_intended_diff_only": 0.4039999842643738, + "tpp_threshold_500_unintended_diff_only": 0.025749996304512024 }, "6": { - "tpp_threshold_2_total_metric": 0.04150000214576721, - "tpp_threshold_2_intended_diff_only": 0.041999995708465576, - "tpp_threshold_2_unintended_diff_only": 0.0004999935626983643, - "tpp_threshold_5_total_metric": 0.061249956488609314, - "tpp_threshold_5_intended_diff_only": 0.0679999589920044, - "tpp_threshold_5_unintended_diff_only": 0.006750002503395081, - "tpp_threshold_10_total_metric": 0.08675000071525574, - "tpp_threshold_10_intended_diff_only": 0.08799999952316284, - "tpp_threshold_10_unintended_diff_only": 0.0012499988079071045 + "tpp_threshold_2_total_metric": 0.02849997580051422, + "tpp_threshold_2_intended_diff_only": 0.0339999794960022, + "tpp_threshold_2_unintended_diff_only": 0.005500003695487976, + "tpp_threshold_5_total_metric": 0.06325002014636993, + "tpp_threshold_5_intended_diff_only": 0.06400001049041748, + "tpp_threshold_5_unintended_diff_only": 0.0007499903440475464, + "tpp_threshold_10_total_metric": 0.09674996137619019, + "tpp_threshold_10_intended_diff_only": 0.09999996423721313, + "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492, + "tpp_threshold_20_total_metric": 0.13449996709823608, + "tpp_threshold_20_intended_diff_only": 0.14499998092651367, + "tpp_threshold_20_unintended_diff_only": 0.010500013828277588, + "tpp_threshold_50_total_metric": 0.23349997401237488, + "tpp_threshold_50_intended_diff_only": 0.24299997091293335, + "tpp_threshold_50_unintended_diff_only": 0.009499996900558472, + "tpp_threshold_100_total_metric": 0.28849999606609344, + "tpp_threshold_100_intended_diff_only": 0.3009999990463257, + "tpp_threshold_100_unintended_diff_only": 0.012500002980232239, + "tpp_threshold_500_total_metric": 0.34400004148483276, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.02899998426437378 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-146484-tokens-600000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-146484-tokens-600000000/scr/saebench_scr_custom_sae_eval_results.json index 8aeb05c75612c186f568044d810c473a95100cd3..a03c7f5bee9067425221bad60f376c83abe4ca34 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-146484-tokens-600000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-146484-tokens-600000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "56c595ad-e2bf-4f48-aade-2c8de2be3566", - "datetime_epoch_millis": 1776884036103, + "eval_id": "e46c9f21-46ba-403d-b877-38fe589d355a", + "datetime_epoch_millis": 1777315968594, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.27738809689751404, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.3321123269633328, "scr_metric_threshold_10": 0.23550300413495637, "scr_dir2_threshold_10": 0.2263243279951899, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2469655041244911, + "scr_metric_threshold_20": 0.2674926661046679, + "scr_dir2_threshold_20": 0.25760632497206615, + "scr_dir1_threshold_50": 0.18479817640625043, + "scr_metric_threshold_50": 0.3553871309435371, + "scr_dir2_threshold_50": 0.3500099011394471, + "scr_dir1_threshold_100": 0.10319157981943705, + "scr_metric_threshold_100": 0.39137358639136266, + "scr_dir2_threshold_100": 0.3857753722056411, + "scr_dir1_threshold_500": -0.5537493792316611, + "scr_metric_threshold_500": 0.15692243971978725, + "scr_dir2_threshold_500": 0.12993883713028137 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.5769232238773122, "scr_metric_threshold_10": 0.06217613139950766, "scr_dir2_threshold_10": 0.06217613139950766, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.43589788655709094, + "scr_metric_threshold_20": 0.1398964114610147, + "scr_dir2_threshold_20": 0.1398964114610147, + "scr_dir1_threshold_50": 0.43589788655709094, + "scr_metric_threshold_50": 0.2124352571631342, + "scr_dir2_threshold_50": 0.2124352571631342, + "scr_dir1_threshold_100": 0.4615385791018498, + "scr_metric_threshold_100": 0.32383416568792556, + "scr_dir2_threshold_100": 0.32383416568792556, + "scr_dir1_threshold_500": -1.794871402993834, + "scr_metric_threshold_500": 0.020725428605223653, + "scr_dir2_threshold_500": 0.020725428605223653 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.42424228740901826, "scr_metric_threshold_10": 0.34862389836223945, "scr_dir2_threshold_10": 0.34862389836223945, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.16666674192503994, + "scr_metric_threshold_20": 0.23853212179761762, + "scr_dir2_threshold_20": 0.23853212179761762, + "scr_dir1_threshold_50": 0.5681820542194435, + "scr_metric_threshold_50": 0.5840979557613263, + "scr_dir2_threshold_50": 0.5840979557613263, + "scr_dir1_threshold_100": 0.26515165540575625, + "scr_metric_threshold_100": 0.5321101318775462, + "scr_dir2_threshold_100": 0.5321101318775462, + "scr_dir1_threshold_500": -0.7499996613373202, + "scr_metric_threshold_500": -0.23241581072336703, + "scr_dir2_threshold_500": -0.23241581072336703 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.3749997671693945, "scr_metric_threshold_10": 0.10050255094361776, "scr_dir2_threshold_10": 0.10050255094361776, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.015624796273220196, + "scr_metric_threshold_20": 0.15577896145062706, + "scr_dir2_threshold_20": 0.15577896145062706, + "scr_dir1_threshold_50": -0.8124996507540918, + "scr_metric_threshold_50": 0.18341709182393628, + "scr_dir2_threshold_50": 0.18341709182393628, + "scr_dir1_threshold_100": -0.5312495925464404, + "scr_metric_threshold_100": 0.13819095138732784, + "scr_dir2_threshold_100": 0.13819095138732784, + "scr_dir1_threshold_500": -1.8593749708961742, + "scr_metric_threshold_500": -0.025125600295806724, + "scr_dir2_threshold_500": -0.025125600295806724 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": 0.2617448160956555, "scr_metric_threshold_10": 0.1687899080607876, "scr_dir2_threshold_10": 0.1687899080607876, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.22818779864829455, + "scr_metric_threshold_20": 0.18152878695440397, + "scr_dir2_threshold_20": 0.18152878695440397, + "scr_dir1_threshold_50": 0.2617448160956555, + "scr_metric_threshold_50": 0.3407642985653504, + "scr_dir2_threshold_50": 0.3407642985653504, + "scr_dir1_threshold_100": -0.5838927436340035, + "scr_metric_threshold_100": 0.47452243203647043, + "scr_dir2_threshold_100": 0.47452243203647043, + "scr_dir1_threshold_500": -1.5234904322537157, + "scr_metric_threshold_500": -0.5127388788936164, + "scr_dir2_threshold_500": -0.5127388788936164 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.07368420392192293, "scr_metric_threshold_10": 0.4365079346307434, "scr_dir2_threshold_10": 0.4365079346307434, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.11052646273721473, + "scr_metric_threshold_20": 0.5238095688621593, + "scr_dir2_threshold_20": 0.5238095688621593, + "scr_dir1_threshold_50": -0.19999993725826787, + "scr_metric_threshold_50": 0.6111109665672388, + "scr_dir2_threshold_50": 0.6111109665672388, + "scr_dir1_threshold_100": -0.184210352950477, + "scr_metric_threshold_100": 0.6468252607288938, + "scr_dir2_threshold_100": 0.6468252607288938, + "scr_dir1_threshold_500": -0.15789458824326602, + "scr_metric_threshold_500": 0.7023808031440972, + "scr_dir2_threshold_500": 0.7023808031440972 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.3825136878996036, "scr_metric_threshold_10": 0.13358787395939287, "scr_dir2_threshold_10": 0.13358787395939287, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3989072479911928, + "scr_metric_threshold_20": 0.19465646684173443, + "scr_dir2_threshold_20": 0.19465646684173443, + "scr_dir1_threshold_50": 0.4808743970321442, + "scr_metric_threshold_50": 0.24809157092576697, + "scr_dir2_threshold_50": 0.24809157092576697, + "scr_dir1_threshold_100": 0.5191256029678558, + "scr_metric_threshold_100": 0.2786259811162493, + "scr_dir2_threshold_100": 0.2786259811162493, + "scr_dir1_threshold_500": 0.5300547515984144, + "scr_metric_threshold_500": 0.36641223978991866, + "scr_dir2_threshold_500": 0.36641223978991866 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.3581395813037724, "scr_metric_threshold_10": 0.42918468769337936, "scr_dir2_threshold_10": 0.42918468769337936, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.4046511576329142, + "scr_metric_threshold_20": 0.45922747769238426, + "scr_dir2_threshold_20": 0.45922747769238426, + "scr_dir1_threshold_50": 0.49302320810445616, + "scr_metric_threshold_50": 0.41201726948797046, + "scr_dir2_threshold_50": 0.41201726948797046, + "scr_dir1_threshold_100": 0.5627905725981688, + "scr_metric_threshold_100": 0.4206008506837011, + "scr_dir2_threshold_100": 0.4206008506837011, + "scr_dir1_threshold_500": 0.6744184112342817, + "scr_metric_threshold_500": 0.48497847709352376, + "scr_dir2_threshold_500": 0.48497847709352376 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.20465104802998307, "scr_metric_threshold_10": 0.20465104802998307, "scr_dir2_threshold_10": 0.1312216389118512, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.24651153377740184, + "scr_metric_threshold_20": 0.24651153377740184, + "scr_dir2_threshold_20": 0.16742080471658802, + "scr_dir1_threshold_50": 0.2511626372535729, + "scr_metric_threshold_50": 0.2511626372535729, + "scr_dir2_threshold_50": 0.208144798820853, + "scr_dir1_threshold_100": 0.31627891761278654, + "scr_metric_threshold_100": 0.31627891761278654, + "scr_dir2_threshold_100": 0.2714932041270145, + "scr_dir1_threshold_500": 0.4511628590383247, + "scr_metric_threshold_500": 0.4511628590383247, + "scr_dir2_threshold_500": 0.23529403832227772 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-146484-tokens-600000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-146484-tokens-600000000/tpp/saebench_tpp_custom_sae_eval_results.json index 57ed054c29486f48288caa1c73ce2b05b6b313da..a532f54e3a6284275c2eac3761ae974ec2f05752 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-146484-tokens-600000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-146484-tokens-600000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "cb259007-e24c-44ba-8006-1acf43c62162", - "datetime_epoch_millis": 1776884205111, + "eval_id": "482df7fa-eeb3-40f9-bb54-02dce228797d", + "datetime_epoch_millis": 1777316251617, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.027574996650218966, - "tpp_threshold_2_intended_diff_only": 0.02940000295639038, - "tpp_threshold_2_unintended_diff_only": 0.001825006306171417, - "tpp_threshold_5_total_metric": 0.05815000534057617, - "tpp_threshold_5_intended_diff_only": 0.06840000748634338, - "tpp_threshold_5_unintended_diff_only": 0.010250002145767212, - "tpp_threshold_10_total_metric": 0.11262500584125519, - "tpp_threshold_10_intended_diff_only": 0.14600000977516173, - "tpp_threshold_10_unintended_diff_only": 0.03337500393390656, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.02634999305009842, + "tpp_threshold_2_intended_diff_only": 0.029500001668930055, + "tpp_threshold_2_unintended_diff_only": 0.0031500086188316345, + "tpp_threshold_5_total_metric": 0.058674997091293334, + "tpp_threshold_5_intended_diff_only": 0.06740000247955322, + "tpp_threshold_5_unintended_diff_only": 0.008725005388259889, + "tpp_threshold_10_total_metric": 0.11385000199079515, + "tpp_threshold_10_intended_diff_only": 0.14680001139640808, + "tpp_threshold_10_unintended_diff_only": 0.03295000940561294, + "tpp_threshold_20_total_metric": 0.14422501027584075, + "tpp_threshold_20_intended_diff_only": 0.19700001478195192, + "tpp_threshold_20_unintended_diff_only": 0.052775004506111146, + "tpp_threshold_50_total_metric": 0.17449999749660491, + "tpp_threshold_50_intended_diff_only": 0.262800008058548, + "tpp_threshold_50_unintended_diff_only": 0.08830001056194305, + "tpp_threshold_100_total_metric": 0.1912250116467476, + "tpp_threshold_100_intended_diff_only": 0.3198000192642212, + "tpp_threshold_100_unintended_diff_only": 0.1285750076174736, + "tpp_threshold_500_total_metric": 0.24270001649856565, + "tpp_threshold_500_intended_diff_only": 0.4316000282764435, + "tpp_threshold_500_unintended_diff_only": 0.18890001177787782 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.044399994611740115, - "tpp_threshold_2_intended_diff_only": 0.04780000448226929, - "tpp_threshold_2_unintended_diff_only": 0.0034000098705291746, - "tpp_threshold_5_total_metric": 0.09930000901222229, - "tpp_threshold_5_intended_diff_only": 0.11780002117156982, - "tpp_threshold_5_unintended_diff_only": 0.018500012159347535, - "tpp_threshold_10_total_metric": 0.19675001502037048, - "tpp_threshold_10_intended_diff_only": 0.2610000252723694, - "tpp_threshold_10_unintended_diff_only": 0.0642500102519989, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.043049997091293334, + "tpp_threshold_2_intended_diff_only": 0.048200011253356934, + "tpp_threshold_2_unintended_diff_only": 0.005150014162063598, + "tpp_threshold_5_total_metric": 0.10069999992847442, + "tpp_threshold_5_intended_diff_only": 0.11760001182556153, + "tpp_threshold_5_unintended_diff_only": 0.0169000118970871, + "tpp_threshold_10_total_metric": 0.19780000746250154, + "tpp_threshold_10_intended_diff_only": 0.26080002784729006, + "tpp_threshold_10_unintended_diff_only": 0.06300002038478851, + "tpp_threshold_20_total_metric": 0.23650000393390655, + "tpp_threshold_20_intended_diff_only": 0.3376000165939331, + "tpp_threshold_20_unintended_diff_only": 0.10110001266002655, + "tpp_threshold_50_total_metric": 0.23419999480247497, + "tpp_threshold_50_intended_diff_only": 0.4052000164985657, + "tpp_threshold_50_unintended_diff_only": 0.1710000216960907, + "tpp_threshold_100_total_metric": 0.19340000450611114, + "tpp_threshold_100_intended_diff_only": 0.4388000249862671, + "tpp_threshold_100_unintended_diff_only": 0.24540002048015594, + "tpp_threshold_500_total_metric": 0.11355001628398895, + "tpp_threshold_500_intended_diff_only": 0.4670000433921814, + "tpp_threshold_500_unintended_diff_only": 0.35345002710819245 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.010749998688697814, - "tpp_threshold_2_intended_diff_only": 0.011000001430511474, - "tpp_threshold_2_unintended_diff_only": 0.0002500027418136597, - "tpp_threshold_5_total_metric": 0.017000001668930054, - "tpp_threshold_5_intended_diff_only": 0.018999993801116943, - "tpp_threshold_5_unintended_diff_only": 0.00199999213218689, - "tpp_threshold_10_total_metric": 0.02849999666213989, - "tpp_threshold_10_intended_diff_only": 0.0309999942779541, - "tpp_threshold_10_unintended_diff_only": 0.002499997615814209, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.009649989008903504, + "tpp_threshold_2_intended_diff_only": 0.010799992084503173, + "tpp_threshold_2_unintended_diff_only": 0.0011500030755996703, + "tpp_threshold_5_total_metric": 0.016649994254112243, + "tpp_threshold_5_intended_diff_only": 0.017199993133544922, + "tpp_threshold_5_unintended_diff_only": 0.0005499988794326783, + "tpp_threshold_10_total_metric": 0.029899996519088746, + "tpp_threshold_10_intended_diff_only": 0.03279999494552612, + "tpp_threshold_10_unintended_diff_only": 0.002899998426437378, + "tpp_threshold_20_total_metric": 0.051950016617774965, + "tpp_threshold_20_intended_diff_only": 0.0564000129699707, + "tpp_threshold_20_unintended_diff_only": 0.00444999635219574, + "tpp_threshold_50_total_metric": 0.11480000019073486, + "tpp_threshold_50_intended_diff_only": 0.12039999961853028, + "tpp_threshold_50_unintended_diff_only": 0.00559999942779541, + "tpp_threshold_100_total_metric": 0.18905001878738403, + "tpp_threshold_100_intended_diff_only": 0.2008000135421753, + "tpp_threshold_100_unintended_diff_only": 0.01174999475479126, + "tpp_threshold_500_total_metric": 0.3718500167131424, + "tpp_threshold_500_intended_diff_only": 0.3962000131607056, + "tpp_threshold_500_unintended_diff_only": 0.024349996447563173 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.011250019073486328, "tpp_threshold_10_total_metric": 0.1004999577999115, "tpp_threshold_10_intended_diff_only": 0.11299997568130493, - "tpp_threshold_10_unintended_diff_only": 0.012500017881393433 + "tpp_threshold_10_unintended_diff_only": 0.012500017881393433, + "tpp_threshold_20_total_metric": 0.205999955534935, + "tpp_threshold_20_intended_diff_only": 0.3259999752044678, + "tpp_threshold_20_unintended_diff_only": 0.12000001966953278, + "tpp_threshold_50_total_metric": 0.23149995505809784, + "tpp_threshold_50_intended_diff_only": 0.39499998092651367, + "tpp_threshold_50_unintended_diff_only": 0.16350002586841583, + "tpp_threshold_100_total_metric": 0.18024997413158417, + "tpp_threshold_100_intended_diff_only": 0.4390000104904175, + "tpp_threshold_100_unintended_diff_only": 0.2587500363588333, + "tpp_threshold_500_total_metric": 0.11225000023841858, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.3437500298023224 }, "1": { - "tpp_threshold_2_total_metric": 0.0017500072717666626, - "tpp_threshold_2_intended_diff_only": 0.003000020980834961, - "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983, - "tpp_threshold_5_total_metric": 0.15300002694129944, + "tpp_threshold_2_total_metric": 0.0054999589920043945, + "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_2_unintended_diff_only": 0.000500023365020752, + "tpp_threshold_5_total_metric": 0.15725000202655792, "tpp_threshold_5_intended_diff_only": 0.17800003290176392, - "tpp_threshold_5_unintended_diff_only": 0.025000005960464478, - "tpp_threshold_10_total_metric": 0.20575003325939178, - "tpp_threshold_10_intended_diff_only": 0.26600003242492676, - "tpp_threshold_10_unintended_diff_only": 0.06024999916553497 + "tpp_threshold_5_unintended_diff_only": 0.020750030875205994, + "tpp_threshold_10_total_metric": 0.20500002801418304, + "tpp_threshold_10_intended_diff_only": 0.26500004529953003, + "tpp_threshold_10_unintended_diff_only": 0.060000017285346985, + "tpp_threshold_20_total_metric": 0.2487499862909317, + "tpp_threshold_20_intended_diff_only": 0.3140000104904175, + "tpp_threshold_20_unintended_diff_only": 0.06525002419948578, + "tpp_threshold_50_total_metric": 0.25425001978874207, + "tpp_threshold_50_intended_diff_only": 0.3850000500679016, + "tpp_threshold_50_unintended_diff_only": 0.13075003027915955, + "tpp_threshold_100_total_metric": 0.22874997556209564, + "tpp_threshold_100_intended_diff_only": 0.4259999990463257, + "tpp_threshold_100_unintended_diff_only": 0.19725002348423004, + "tpp_threshold_500_total_metric": 0.10125000774860382, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.3577500432729721 }, "2": { - "tpp_threshold_2_total_metric": 0.1367499977350235, - "tpp_threshold_2_intended_diff_only": 0.1470000147819519, - "tpp_threshold_2_unintended_diff_only": 0.010250017046928406, - "tpp_threshold_5_total_metric": 0.22850003838539124, - "tpp_threshold_5_intended_diff_only": 0.2690000534057617, - "tpp_threshold_5_unintended_diff_only": 0.04050001502037048, - "tpp_threshold_10_total_metric": 0.21875004470348358, - "tpp_threshold_10_intended_diff_only": 0.3340000510215759, - "tpp_threshold_10_unintended_diff_only": 0.11525000631809235 + "tpp_threshold_2_total_metric": 0.12625004351139069, + "tpp_threshold_2_intended_diff_only": 0.14400005340576172, + "tpp_threshold_2_unintended_diff_only": 0.017750009894371033, + "tpp_threshold_5_total_metric": 0.22750000655651093, + "tpp_threshold_5_intended_diff_only": 0.2680000066757202, + "tpp_threshold_5_unintended_diff_only": 0.04050000011920929, + "tpp_threshold_10_total_metric": 0.22300001978874207, + "tpp_threshold_10_intended_diff_only": 0.33500003814697266, + "tpp_threshold_10_unintended_diff_only": 0.11200001835823059, + "tpp_threshold_20_total_metric": 0.1977500468492508, + "tpp_threshold_20_intended_diff_only": 0.36100006103515625, + "tpp_threshold_20_unintended_diff_only": 0.16325001418590546, + "tpp_threshold_50_total_metric": 0.14774999022483826, + "tpp_threshold_50_intended_diff_only": 0.3830000162124634, + "tpp_threshold_50_unintended_diff_only": 0.23525002598762512, + "tpp_threshold_100_total_metric": 0.11075001955032349, + "tpp_threshold_100_intended_diff_only": 0.4140000343322754, + "tpp_threshold_100_unintended_diff_only": 0.3032500147819519, + "tpp_threshold_500_total_metric": 0.05900000035762787, + "tpp_threshold_500_intended_diff_only": 0.4520000219345093, + "tpp_threshold_500_unintended_diff_only": 0.3930000215768814 }, "6": { - "tpp_threshold_2_total_metric": 0.011750012636184692, + "tpp_threshold_2_total_metric": 0.009499981999397278, "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": -0.002750009298324585, - "tpp_threshold_5_total_metric": 0.005500003695487976, - "tpp_threshold_5_intended_diff_only": 0.013000011444091797, - "tpp_threshold_5_unintended_diff_only": 0.007500007748603821, - "tpp_threshold_10_total_metric": 0.2847500443458557, + "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704, + "tpp_threshold_5_total_metric": 0.013499990105628967, + "tpp_threshold_5_intended_diff_only": 0.013999998569488525, + "tpp_threshold_5_unintended_diff_only": 0.0005000084638595581, + "tpp_threshold_10_total_metric": 0.28725002706050873, "tpp_threshold_10_intended_diff_only": 0.362000048160553, - "tpp_threshold_10_unintended_diff_only": 0.07725000381469727 + "tpp_threshold_10_unintended_diff_only": 0.07475002110004425, + "tpp_threshold_20_total_metric": 0.30925001204013824, + "tpp_threshold_20_intended_diff_only": 0.41600000858306885, + "tpp_threshold_20_unintended_diff_only": 0.1067499965429306, + "tpp_threshold_50_total_metric": 0.27524998784065247, + "tpp_threshold_50_intended_diff_only": 0.48000001907348633, + "tpp_threshold_50_unintended_diff_only": 0.20475003123283386, + "tpp_threshold_100_total_metric": 0.21175001561641693, + "tpp_threshold_100_intended_diff_only": 0.4830000400543213, + "tpp_threshold_100_unintended_diff_only": 0.27125002443790436, + "tpp_threshold_500_total_metric": 0.14725002646446228, + "tpp_threshold_500_intended_diff_only": 0.4920000433921814, + "tpp_threshold_500_unintended_diff_only": 0.3447500169277191 }, "9": { - "tpp_threshold_2_total_metric": 0.003999993205070496, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_5_total_metric": 0.02275000512599945, - "tpp_threshold_5_intended_diff_only": 0.03100001811981201, - "tpp_threshold_5_unintended_diff_only": 0.008250012993812561, - "tpp_threshold_10_total_metric": 0.17399999499320984, - "tpp_threshold_10_intended_diff_only": 0.23000001907348633, - "tpp_threshold_10_unintended_diff_only": 0.05600002408027649 + "tpp_threshold_2_total_metric": 0.006250038743019104, + "tpp_threshold_2_intended_diff_only": 0.00700002908706665, + "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.018500030040740967, + "tpp_threshold_5_intended_diff_only": 0.030000030994415283, + "tpp_threshold_5_unintended_diff_only": 0.011500000953674316, + "tpp_threshold_10_total_metric": 0.1732500046491623, + "tpp_threshold_10_intended_diff_only": 0.2290000319480896, + "tpp_threshold_10_unintended_diff_only": 0.05575002729892731, + "tpp_threshold_20_total_metric": 0.22075001895427704, + "tpp_threshold_20_intended_diff_only": 0.2710000276565552, + "tpp_threshold_20_unintended_diff_only": 0.05025000870227814, + "tpp_threshold_50_total_metric": 0.26225002110004425, + "tpp_threshold_50_intended_diff_only": 0.3830000162124634, + "tpp_threshold_50_unintended_diff_only": 0.12074999511241913, + "tpp_threshold_100_total_metric": 0.2355000376701355, + "tpp_threshold_100_intended_diff_only": 0.4320000410079956, + "tpp_threshold_100_unintended_diff_only": 0.1965000033378601, + "tpp_threshold_500_total_metric": 0.14800004661083221, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.3280000239610672 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.009249985218048096, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": -0.004249989986419678, - "tpp_threshold_5_total_metric": 0.012250006198883057, - "tpp_threshold_5_intended_diff_only": 0.009999990463256836, - "tpp_threshold_5_unintended_diff_only": -0.0022500157356262207, - "tpp_threshold_10_total_metric": 0.003250017762184143, - "tpp_threshold_10_intended_diff_only": 0.008000016212463379, - "tpp_threshold_10_unintended_diff_only": 0.004749998450279236 + "tpp_threshold_2_total_metric": 0.0029999762773513794, + "tpp_threshold_2_intended_diff_only": 0.006999969482421875, + "tpp_threshold_2_unintended_diff_only": 0.003999993205070496, + "tpp_threshold_5_total_metric": 0.007250010967254639, + "tpp_threshold_5_intended_diff_only": 0.009000003337860107, + "tpp_threshold_5_unintended_diff_only": 0.0017499923706054688, + "tpp_threshold_10_total_metric": 0.004249975085258484, + "tpp_threshold_10_intended_diff_only": 0.006999969482421875, + "tpp_threshold_10_unintended_diff_only": 0.002749994397163391, + "tpp_threshold_20_total_metric": 0.019250020384788513, + "tpp_threshold_20_intended_diff_only": 0.022000014781951904, + "tpp_threshold_20_unintended_diff_only": 0.002749994397163391, + "tpp_threshold_50_total_metric": 0.06050001084804535, + "tpp_threshold_50_intended_diff_only": 0.06000000238418579, + "tpp_threshold_50_unintended_diff_only": -0.0005000084638595581, + "tpp_threshold_100_total_metric": 0.11225004494190216, + "tpp_threshold_100_intended_diff_only": 0.11800003051757812, + "tpp_threshold_100_unintended_diff_only": 0.005749985575675964, + "tpp_threshold_500_total_metric": 0.390500009059906, + "tpp_threshold_500_intended_diff_only": 0.40299999713897705, + "tpp_threshold_500_unintended_diff_only": 0.012499988079071045 }, "2": { - "tpp_threshold_2_total_metric": 0.002750024199485779, - "tpp_threshold_2_intended_diff_only": 0.008000016212463379, - "tpp_threshold_2_unintended_diff_only": 0.0052499920129776, - "tpp_threshold_5_total_metric": 0.012000024318695068, - "tpp_threshold_5_intended_diff_only": 0.018000006675720215, - "tpp_threshold_5_unintended_diff_only": 0.0059999823570251465, - "tpp_threshold_10_total_metric": 0.04950001835823059, - "tpp_threshold_10_intended_diff_only": 0.050999999046325684, - "tpp_threshold_10_unintended_diff_only": 0.0014999806880950928 + "tpp_threshold_2_total_metric": 0.013000041246414185, + "tpp_threshold_2_intended_diff_only": 0.01100003719329834, + "tpp_threshold_2_unintended_diff_only": -0.0020000040531158447, + "tpp_threshold_5_total_metric": 0.007750019431114197, + "tpp_threshold_5_intended_diff_only": 0.013999998569488525, + "tpp_threshold_5_unintended_diff_only": 0.006249979138374329, + "tpp_threshold_10_total_metric": 0.048250049352645874, + "tpp_threshold_10_intended_diff_only": 0.05200004577636719, + "tpp_threshold_10_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_20_total_metric": 0.09850004315376282, + "tpp_threshold_20_intended_diff_only": 0.10300004482269287, + "tpp_threshold_20_unintended_diff_only": 0.004500001668930054, + "tpp_threshold_50_total_metric": 0.18325001001358032, + "tpp_threshold_50_intended_diff_only": 0.1940000057220459, + "tpp_threshold_50_unintended_diff_only": 0.010749995708465576, + "tpp_threshold_100_total_metric": 0.285750076174736, + "tpp_threshold_100_intended_diff_only": 0.30500006675720215, + "tpp_threshold_100_unintended_diff_only": 0.019249990582466125, + "tpp_threshold_500_total_metric": 0.41200003027915955, + "tpp_threshold_500_intended_diff_only": 0.43800002336502075, + "tpp_threshold_500_unintended_diff_only": 0.025999993085861206 }, "3": { - "tpp_threshold_2_total_metric": 0.013500019907951355, - "tpp_threshold_2_intended_diff_only": 0.00700002908706665, - "tpp_threshold_2_unintended_diff_only": -0.006499990820884705, - "tpp_threshold_5_total_metric": 0.01900002360343933, - "tpp_threshold_5_intended_diff_only": 0.017000019550323486, - "tpp_threshold_5_unintended_diff_only": -0.0020000040531158447, - "tpp_threshold_10_total_metric": 0.017250001430511475, - "tpp_threshold_10_intended_diff_only": 0.018999993801116943, - "tpp_threshold_10_unintended_diff_only": 0.0017499923706054688 + "tpp_threshold_2_total_metric": 0.005499988794326782, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": -0.0004999935626983643, + "tpp_threshold_5_total_metric": 0.012999996542930603, + "tpp_threshold_5_intended_diff_only": 0.008000016212463379, + "tpp_threshold_5_unintended_diff_only": -0.004999980330467224, + "tpp_threshold_10_total_metric": 0.016749992966651917, + "tpp_threshold_10_intended_diff_only": 0.014999985694885254, + "tpp_threshold_10_unintended_diff_only": -0.0017500072717666626, + "tpp_threshold_20_total_metric": 0.01524999737739563, + "tpp_threshold_20_intended_diff_only": 0.018999993801116943, + "tpp_threshold_20_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_50_total_metric": 0.04850001633167267, + "tpp_threshold_50_intended_diff_only": 0.04900002479553223, + "tpp_threshold_50_unintended_diff_only": 0.0005000084638595581, + "tpp_threshold_100_total_metric": 0.109250009059906, + "tpp_threshold_100_intended_diff_only": 0.12000000476837158, + "tpp_threshold_100_unintended_diff_only": 0.010749995708465576, + "tpp_threshold_500_total_metric": 0.34800003468990326, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.024999991059303284 }, "5": { - "tpp_threshold_2_total_metric": -0.006000012159347534, - "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 0.006999999284744263, - "tpp_threshold_5_total_metric": 0.0022499561309814453, - "tpp_threshold_5_intended_diff_only": 0.003999948501586914, - "tpp_threshold_5_unintended_diff_only": 0.0017499923706054688, - "tpp_threshold_10_total_metric": 0.004499942064285278, - "tpp_threshold_10_intended_diff_only": 0.007999956607818604, - "tpp_threshold_10_unintended_diff_only": 0.003500014543533325 + "tpp_threshold_2_total_metric": 0.001999959349632263, + "tpp_threshold_2_intended_diff_only": 0.001999974250793457, + "tpp_threshold_2_unintended_diff_only": 1.4901161193847656e-08, + "tpp_threshold_5_total_metric": 0.012249961495399475, + "tpp_threshold_5_intended_diff_only": 0.011999964714050293, + "tpp_threshold_5_unintended_diff_only": -0.00024999678134918213, + "tpp_threshold_10_total_metric": 0.0007499605417251587, + "tpp_threshold_10_intended_diff_only": 0.006999969482421875, + "tpp_threshold_10_unintended_diff_only": 0.006250008940696716, + "tpp_threshold_20_total_metric": 0.012749999761581421, + "tpp_threshold_20_intended_diff_only": 0.013999998569488525, + "tpp_threshold_20_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_50_total_metric": 0.0782499760389328, + "tpp_threshold_50_intended_diff_only": 0.08499997854232788, + "tpp_threshold_50_unintended_diff_only": 0.006750002503395081, + "tpp_threshold_100_total_metric": 0.16099996864795685, + "tpp_threshold_100_intended_diff_only": 0.17299997806549072, + "tpp_threshold_100_unintended_diff_only": 0.012000009417533875, + "tpp_threshold_500_total_metric": 0.3669999837875366, + "tpp_threshold_500_intended_diff_only": 0.39399999380111694, + "tpp_threshold_500_unintended_diff_only": 0.027000010013580322 }, "6": { - "tpp_threshold_2_total_metric": 0.03424997627735138, - "tpp_threshold_2_intended_diff_only": 0.0339999794960022, - "tpp_threshold_2_unintended_diff_only": -0.00024999678134918213, - "tpp_threshold_5_total_metric": 0.03949999809265137, - "tpp_threshold_5_intended_diff_only": 0.046000003814697266, - "tpp_threshold_5_unintended_diff_only": 0.0065000057220458984, - "tpp_threshold_10_total_metric": 0.06800000369548798, - "tpp_threshold_10_intended_diff_only": 0.0690000057220459, - "tpp_threshold_10_unintended_diff_only": 0.0010000020265579224 + "tpp_threshold_2_total_metric": 0.024749979376792908, + "tpp_threshold_2_intended_diff_only": 0.02899998426437378, + "tpp_threshold_2_unintended_diff_only": 0.004250004887580872, + "tpp_threshold_5_total_metric": 0.042999982833862305, + "tpp_threshold_5_intended_diff_only": 0.042999982833862305, + "tpp_threshold_5_unintended_diff_only": 0.0, + "tpp_threshold_10_total_metric": 0.07950000464916229, + "tpp_threshold_10_intended_diff_only": 0.08300000429153442, + "tpp_threshold_10_unintended_diff_only": 0.0034999996423721313, + "tpp_threshold_20_total_metric": 0.11400002241134644, + "tpp_threshold_20_intended_diff_only": 0.12400001287460327, + "tpp_threshold_20_unintended_diff_only": 0.009999990463256836, + "tpp_threshold_50_total_metric": 0.20349998772144318, + "tpp_threshold_50_intended_diff_only": 0.21399998664855957, + "tpp_threshold_50_unintended_diff_only": 0.010499998927116394, + "tpp_threshold_100_total_metric": 0.27699999511241913, + "tpp_threshold_100_intended_diff_only": 0.2879999876022339, + "tpp_threshold_100_unintended_diff_only": 0.010999992489814758, + "tpp_threshold_500_total_metric": 0.34175002574920654, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.03125 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-158691-tokens-650000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-158691-tokens-650000000/scr/saebench_scr_custom_sae_eval_results.json index 1c9ecdf4d0ef92891fd327871d61e7a1a1de629a..f91c0bcd2d6924eca11b7456397f165a738ea9bc 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-158691-tokens-650000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-158691-tokens-650000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "98c6e8bd-337e-442b-83a2-41a114ad753c", - "datetime_epoch_millis": 1776884250692, + "eval_id": "d0b86c77-9df7-491a-b204-93d13ef5b918", + "datetime_epoch_millis": 1777316081630, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.2936601455339623, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.29843206934547256, "scr_metric_threshold_10": 0.25342062417805467, "scr_dir2_threshold_10": 0.23900938731986085, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.13807877076721764, + "scr_metric_threshold_20": 0.3056230421071651, + "scr_dir2_threshold_20": 0.2893886838653024, + "scr_dir1_threshold_50": 0.29747191129756534, + "scr_metric_threshold_50": 0.3545211424365467, + "scr_dir2_threshold_50": 0.339873136799641, + "scr_dir1_threshold_100": -0.14174953070081284, + "scr_metric_threshold_100": 0.37074884019934473, + "scr_dir2_threshold_100": 0.3525809025664471, + "scr_dir1_threshold_500": -1.0172816718021795, + "scr_metric_threshold_500": 0.2074486256017326, + "scr_dir2_threshold_500": 0.16098171357221602 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.38461573730554943, "scr_metric_threshold_10": 0.08290156000473131, "scr_dir2_threshold_10": 0.08290156000473131, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2820514388024664, + "scr_metric_threshold_20": 0.13471497710162714, + "scr_dir2_threshold_20": 0.13471497710162714, + "scr_dir1_threshold_50": 0.5384614208981502, + "scr_metric_threshold_50": 0.26683931423164214, + "scr_dir2_threshold_50": 0.26683931423164214, + "scr_dir1_threshold_100": 0.5256410746257707, + "scr_metric_threshold_100": 0.3264248056595377, + "scr_dir2_threshold_100": 0.3264248056595377, + "scr_dir1_threshold_500": -2.16666602986498, + "scr_metric_threshold_500": -0.0492227771252837, + "scr_dir2_threshold_500": -0.0492227771252837 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.41666662903748003, "scr_metric_threshold_10": 0.31192657874830254, "scr_dir2_threshold_10": 0.31192657874830254, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.12121234014557077, + "scr_metric_threshold_20": 0.31804288982255313, + "scr_dir2_threshold_20": 0.31804288982255313, + "scr_dir1_threshold_50": 0.5681820542194435, + "scr_metric_threshold_50": 0.47706424359523525, + "scr_dir2_threshold_50": 0.47706424359523525, + "scr_dir1_threshold_100": 0.10606057185225456, + "scr_metric_threshold_100": 0.5290520674790153, + "scr_dir2_threshold_100": 0.5290520674790153, + "scr_dir1_threshold_500": -1.9772721217849059, + "scr_metric_threshold_500": 0.09785933669330951, + "scr_dir2_threshold_500": 0.09785933669330951 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.31249965075409175, "scr_metric_threshold_10": 0.11306535109152112, "scr_dir2_threshold_10": 0.11306535109152112, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.0, + "scr_metric_threshold_20": 0.2060301620422405, + "scr_dir2_threshold_20": 0.2060301620422405, + "scr_dir1_threshold_50": 0.28125005820765137, + "scr_metric_threshold_50": 0.1859296219014388, + "scr_dir2_threshold_50": 0.1859296219014388, + "scr_dir1_threshold_100": -1.8124996507540918, + "scr_metric_threshold_100": 0.16080402160563206, + "scr_dir2_threshold_100": 0.16080402160563206, + "scr_dir1_threshold_500": -3.343749243300532, + "scr_metric_threshold_500": -0.10301508102112025, + "scr_dir2_threshold_500": -0.10301508102112025 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": 0.1543624402703411, "scr_metric_threshold_10": 0.1337579436474169, "scr_dir2_threshold_10": 0.1337579436474169, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.5637587731843083, + "scr_metric_threshold_20": 0.1687899080607876, + "scr_dir2_threshold_20": 0.1687899080607876, + "scr_dir1_threshold_50": -0.0604027114114901, + "scr_metric_threshold_50": 0.21974523381154987, + "scr_dir2_threshold_50": 0.21974523381154987, + "scr_dir1_threshold_100": -1.1140940993397486, + "scr_metric_threshold_100": 0.124203926844982, + "scr_dir2_threshold_100": 0.124203926844982, + "scr_dir1_threshold_500": -1.912751994655583, + "scr_metric_threshold_500": -0.528662430054711, + "scr_dir2_threshold_500": -0.528662430054711 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.08947378822971379, "scr_metric_threshold_10": 0.5277777712076017, "scr_dir2_threshold_10": 0.5277777712076017, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.08947378822971379, + "scr_metric_threshold_20": 0.5992063595309115, + "scr_dir2_threshold_20": 0.5992063595309115, + "scr_dir1_threshold_50": -0.2578945568723999, + "scr_metric_threshold_50": 0.6944443984532124, + "scr_dir2_threshold_50": 0.6944443984532124, + "scr_dir1_threshold_100": -0.23684219607355966, + "scr_metric_threshold_100": 0.7222222879239824, + "scr_dir2_threshold_100": 0.7222222879239824, + "scr_dir1_threshold_500": -0.15263149804355594, + "scr_metric_threshold_500": 0.8055554832836195, + "scr_dir2_threshold_500": 0.8055554832836195 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.3879780993606342, "scr_metric_threshold_10": 0.15648856785294307, "scr_dir2_threshold_10": 0.15648856785294307, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3989072479911928, + "scr_metric_threshold_20": 0.20229018313866656, + "scr_dir2_threshold_20": 0.20229018313866656, + "scr_dir1_threshold_50": 0.4590164254795243, + "scr_metric_threshold_50": 0.29389318621149046, + "scr_dir2_threshold_50": 0.29389318621149046, + "scr_dir1_threshold_100": 0.4535520140184937, + "scr_metric_threshold_100": 0.3206106245041952, + "scr_dir2_threshold_100": 0.3206106245041952, + "scr_dir1_threshold_500": 0.5683059575341262, + "scr_metric_threshold_500": 0.4007633943795555, + "scr_dir2_threshold_500": 0.4007633943795555 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.3953486760285681, "scr_metric_threshold_10": 0.4549356870945189, "scr_dir2_threshold_10": 0.4549356870945189, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.49302320810445616, + "scr_metric_threshold_20": 0.5321889411118851, + "scr_dir2_threshold_20": 0.5321889411118851, + "scr_dir1_threshold_50": 0.5348836822468563, + "scr_metric_threshold_50": 0.38197422367501804, + "scr_dir2_threshold_50": 0.38197422367501804, + "scr_dir1_threshold_100": 0.5906977401803443, + "scr_metric_threshold_100": 0.42918468769337936, + "scr_dir2_threshold_100": 0.42918468769337936, + "scr_dir1_threshold_500": 0.3209302093481138, + "scr_metric_threshold_500": 0.5107297323086109, + "scr_dir2_threshold_500": 0.5107297323086109 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.24651153377740184, "scr_metric_threshold_10": 0.24651153377740184, "scr_dir2_threshold_10": 0.1312216389118512, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2837209160486496, + "scr_metric_threshold_20": 0.2837209160486496, + "scr_dir2_threshold_20": 0.15384605011374777, + "scr_dir1_threshold_50": 0.31627891761278654, + "scr_metric_threshold_50": 0.31627891761278654, + "scr_dir2_threshold_50": 0.19909487251754088, + "scr_dir1_threshold_100": 0.3534882998840343, + "scr_metric_threshold_100": 0.3534882998840343, + "scr_dir2_threshold_100": 0.208144798820853, + "scr_dir1_threshold_500": 0.5255813463498804, + "scr_metric_threshold_500": 0.5255813463498804, + "scr_dir2_threshold_500": 0.15384605011374777 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-158691-tokens-650000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-158691-tokens-650000000/tpp/saebench_tpp_custom_sae_eval_results.json index 136e5088e2678e2508bf8238ad284d5896e3eb43..20c42bbac6e10bad66826794d88377016f4d6927 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-158691-tokens-650000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-158691-tokens-650000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "6a3f4e41-9281-4ef1-a4a0-7a787ea04fa5", - "datetime_epoch_millis": 1776884414187, + "eval_id": "17179781-3ba6-4cf6-9aed-554cc99614ea", + "datetime_epoch_millis": 1777316353206, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.019324998557567596, - "tpp_threshold_2_intended_diff_only": 0.020399999618530274, - "tpp_threshold_2_unintended_diff_only": 0.001075001060962677, - "tpp_threshold_5_total_metric": 0.06357499957084656, - "tpp_threshold_5_intended_diff_only": 0.07430000305175781, - "tpp_threshold_5_unintended_diff_only": 0.010725003480911254, - "tpp_threshold_10_total_metric": 0.10377499908208848, - "tpp_threshold_10_intended_diff_only": 0.13830000758171082, - "tpp_threshold_10_unintended_diff_only": 0.034525008499622346, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.018099991977214812, + "tpp_threshold_2_intended_diff_only": 0.020499998331069948, + "tpp_threshold_2_unintended_diff_only": 0.0024000063538551332, + "tpp_threshold_5_total_metric": 0.06372499167919159, + "tpp_threshold_5_intended_diff_only": 0.07319999933242798, + "tpp_threshold_5_unintended_diff_only": 0.00947500765323639, + "tpp_threshold_10_total_metric": 0.10470000058412553, + "tpp_threshold_10_intended_diff_only": 0.139000004529953, + "tpp_threshold_10_unintended_diff_only": 0.03430000394582748, + "tpp_threshold_20_total_metric": 0.14527500718832015, + "tpp_threshold_20_intended_diff_only": 0.1982000172138214, + "tpp_threshold_20_unintended_diff_only": 0.05292501002550125, + "tpp_threshold_50_total_metric": 0.1659500002861023, + "tpp_threshold_50_intended_diff_only": 0.2554000079631805, + "tpp_threshold_50_unintended_diff_only": 0.08945000767707825, + "tpp_threshold_100_total_metric": 0.18922500908374784, + "tpp_threshold_100_intended_diff_only": 0.31510002017021177, + "tpp_threshold_100_unintended_diff_only": 0.12587501108646393, + "tpp_threshold_500_total_metric": 0.24060002863407134, + "tpp_threshold_500_intended_diff_only": 0.4327000379562378, + "tpp_threshold_500_unintended_diff_only": 0.19210000932216645 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.02819998562335968, - "tpp_threshold_2_intended_diff_only": 0.02999999523162842, - "tpp_threshold_2_unintended_diff_only": 0.0018000096082687379, - "tpp_threshold_5_total_metric": 0.1095499962568283, + "tpp_threshold_2_total_metric": 0.027099993824958802, + "tpp_threshold_2_intended_diff_only": 0.030800008773803712, + "tpp_threshold_2_unintended_diff_only": 0.0037000149488449095, + "tpp_threshold_5_total_metric": 0.11039999425411225, "tpp_threshold_5_intended_diff_only": 0.1290000081062317, - "tpp_threshold_5_unintended_diff_only": 0.01945001184940338, - "tpp_threshold_10_total_metric": 0.18080000281333924, - "tpp_threshold_10_intended_diff_only": 0.24680001735687257, - "tpp_threshold_10_unintended_diff_only": 0.06600001454353333, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_5_unintended_diff_only": 0.018600013852119446, + "tpp_threshold_10_total_metric": 0.18130000531673432, + "tpp_threshold_10_intended_diff_only": 0.2466000199317932, + "tpp_threshold_10_unintended_diff_only": 0.0653000146150589, + "tpp_threshold_20_total_metric": 0.24280000627040862, + "tpp_threshold_20_intended_diff_only": 0.3440000295639038, + "tpp_threshold_20_unintended_diff_only": 0.10120002329349517, + "tpp_threshold_50_total_metric": 0.2319500058889389, + "tpp_threshold_50_intended_diff_only": 0.40540002584457396, + "tpp_threshold_50_unintended_diff_only": 0.17345001995563508, + "tpp_threshold_100_total_metric": 0.19920000731945037, + "tpp_threshold_100_intended_diff_only": 0.4392000317573547, + "tpp_threshold_100_unintended_diff_only": 0.24000002443790436, + "tpp_threshold_500_total_metric": 0.10850001871585846, + "tpp_threshold_500_intended_diff_only": 0.46620004177093505, + "tpp_threshold_500_unintended_diff_only": 0.3577000230550766 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.010450011491775513, - "tpp_threshold_2_intended_diff_only": 0.010800004005432129, - "tpp_threshold_2_unintended_diff_only": 0.0003499925136566162, - "tpp_threshold_5_total_metric": 0.017600002884864806, - "tpp_threshold_5_intended_diff_only": 0.019599997997283937, - "tpp_threshold_5_unintended_diff_only": 0.0019999951124191283, - "tpp_threshold_10_total_metric": 0.026749995350837708, - "tpp_threshold_10_intended_diff_only": 0.029799997806549072, - "tpp_threshold_10_unintended_diff_only": 0.003050002455711365, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.009099990129470825, + "tpp_threshold_2_intended_diff_only": 0.010199987888336181, + "tpp_threshold_2_unintended_diff_only": 0.0010999977588653565, + "tpp_threshold_5_total_metric": 0.017049989104270934, + "tpp_threshold_5_intended_diff_only": 0.01739999055862427, + "tpp_threshold_5_unintended_diff_only": 0.00035000145435333253, + "tpp_threshold_10_total_metric": 0.028099995851516724, + "tpp_threshold_10_intended_diff_only": 0.031399989128112794, + "tpp_threshold_10_unintended_diff_only": 0.003299993276596069, + "tpp_threshold_20_total_metric": 0.04775000810623169, + "tpp_threshold_20_intended_diff_only": 0.05240000486373901, + "tpp_threshold_20_unintended_diff_only": 0.004649996757507324, + "tpp_threshold_50_total_metric": 0.09994999468326568, + "tpp_threshold_50_intended_diff_only": 0.1053999900817871, + "tpp_threshold_50_unintended_diff_only": 0.005449995398521423, + "tpp_threshold_100_total_metric": 0.17925001084804534, + "tpp_threshold_100_intended_diff_only": 0.19100000858306884, + "tpp_threshold_100_unintended_diff_only": 0.011749997735023499, + "tpp_threshold_500_total_metric": 0.37270003855228423, + "tpp_threshold_500_intended_diff_only": 0.39920003414154054, + "tpp_threshold_500_unintended_diff_only": 0.026499995589256288 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.003250017762184143, "tpp_threshold_10_total_metric": 0.019999966025352478, "tpp_threshold_10_intended_diff_only": 0.02399998903274536, - "tpp_threshold_10_unintended_diff_only": 0.004000023007392883 + "tpp_threshold_10_unintended_diff_only": 0.004000023007392883, + "tpp_threshold_20_total_metric": 0.22499997913837433, + "tpp_threshold_20_intended_diff_only": 0.30000001192092896, + "tpp_threshold_20_unintended_diff_only": 0.07500003278255463, + "tpp_threshold_50_total_metric": 0.25224997103214264, + "tpp_threshold_50_intended_diff_only": 0.39899998903274536, + "tpp_threshold_50_unintended_diff_only": 0.14675001800060272, + "tpp_threshold_100_total_metric": 0.22524996101856232, + "tpp_threshold_100_intended_diff_only": 0.44099998474121094, + "tpp_threshold_100_unintended_diff_only": 0.21575002372264862, + "tpp_threshold_500_total_metric": 0.125, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.33100003004074097 }, "1": { - "tpp_threshold_2_total_metric": -2.9802322387695312e-08, - "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162, - "tpp_threshold_5_total_metric": 0.15574999153614044, + "tpp_threshold_2_total_metric": 0.004499971866607666, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 0.000500023365020752, + "tpp_threshold_5_total_metric": 0.16050000488758087, "tpp_threshold_5_intended_diff_only": 0.18400001525878906, - "tpp_threshold_5_unintended_diff_only": 0.02825002372264862, - "tpp_threshold_10_total_metric": 0.20650000870227814, - "tpp_threshold_10_intended_diff_only": 0.2720000147819519, - "tpp_threshold_10_unintended_diff_only": 0.06550000607967377 + "tpp_threshold_5_unintended_diff_only": 0.02350001037120819, + "tpp_threshold_10_total_metric": 0.20725001394748688, + "tpp_threshold_10_intended_diff_only": 0.2710000276565552, + "tpp_threshold_10_unintended_diff_only": 0.0637500137090683, + "tpp_threshold_20_total_metric": 0.23374997079372406, + "tpp_threshold_20_intended_diff_only": 0.3149999976158142, + "tpp_threshold_20_unintended_diff_only": 0.08125002682209015, + "tpp_threshold_50_total_metric": 0.22200000286102295, + "tpp_threshold_50_intended_diff_only": 0.36500000953674316, + "tpp_threshold_50_unintended_diff_only": 0.14300000667572021, + "tpp_threshold_100_total_metric": 0.20224998891353607, + "tpp_threshold_100_intended_diff_only": 0.406000018119812, + "tpp_threshold_100_unintended_diff_only": 0.20375002920627594, + "tpp_threshold_500_total_metric": 0.09350000321865082, + "tpp_threshold_500_intended_diff_only": 0.4570000171661377, + "tpp_threshold_500_unintended_diff_only": 0.3635000139474869 }, "2": { - "tpp_threshold_2_total_metric": 0.11574999988079071, - "tpp_threshold_2_intended_diff_only": 0.12400001287460327, - "tpp_threshold_2_unintended_diff_only": 0.008250012993812561, - "tpp_threshold_5_total_metric": 0.242000013589859, - "tpp_threshold_5_intended_diff_only": 0.281000018119812, - "tpp_threshold_5_unintended_diff_only": 0.039000004529953, - "tpp_threshold_10_total_metric": 0.2175000160932541, - "tpp_threshold_10_intended_diff_only": 0.3360000252723694, - "tpp_threshold_10_unintended_diff_only": 0.1185000091791153 + "tpp_threshold_2_total_metric": 0.10475003719329834, + "tpp_threshold_2_intended_diff_only": 0.12100005149841309, + "tpp_threshold_2_unintended_diff_only": 0.016250014305114746, + "tpp_threshold_5_total_metric": 0.23900000751018524, + "tpp_threshold_5_intended_diff_only": 0.2800000309944153, + "tpp_threshold_5_unintended_diff_only": 0.04100002348423004, + "tpp_threshold_10_total_metric": 0.2199999988079071, + "tpp_threshold_10_intended_diff_only": 0.3370000123977661, + "tpp_threshold_10_unintended_diff_only": 0.11700001358985901, + "tpp_threshold_20_total_metric": 0.19850000739097595, + "tpp_threshold_20_intended_diff_only": 0.3630000352859497, + "tpp_threshold_20_unintended_diff_only": 0.16450002789497375, + "tpp_threshold_50_total_metric": 0.15650004148483276, + "tpp_threshold_50_intended_diff_only": 0.3840000629425049, + "tpp_threshold_50_unintended_diff_only": 0.22750002145767212, + "tpp_threshold_100_total_metric": 0.11699999868869781, + "tpp_threshold_100_intended_diff_only": 0.4230000376701355, + "tpp_threshold_100_unintended_diff_only": 0.3060000389814377, + "tpp_threshold_500_total_metric": 0.056249991059303284, + "tpp_threshold_500_intended_diff_only": 0.4520000219345093, + "tpp_threshold_500_unintended_diff_only": 0.395750030875206 }, "6": { - "tpp_threshold_2_total_metric": 0.010750025510787964, + "tpp_threshold_2_total_metric": 0.00849999487400055, "tpp_threshold_2_intended_diff_only": 0.008000016212463379, - "tpp_threshold_2_unintended_diff_only": -0.002750009298324585, - "tpp_threshold_5_total_metric": 0.1027500331401825, - "tpp_threshold_5_intended_diff_only": 0.1170000433921814, - "tpp_threshold_5_unintended_diff_only": 0.014250010251998901, - "tpp_threshold_10_total_metric": 0.27524998784065247, + "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704, + "tpp_threshold_5_total_metric": 0.10775002837181091, + "tpp_threshold_5_intended_diff_only": 0.11800003051757812, + "tpp_threshold_5_unintended_diff_only": 0.010250002145767212, + "tpp_threshold_10_total_metric": 0.2772499918937683, "tpp_threshold_10_intended_diff_only": 0.35100001096725464, - "tpp_threshold_10_unintended_diff_only": 0.07575002312660217 + "tpp_threshold_10_unintended_diff_only": 0.07375001907348633, + "tpp_threshold_20_total_metric": 0.3137500435113907, + "tpp_threshold_20_intended_diff_only": 0.4310000538825989, + "tpp_threshold_20_unintended_diff_only": 0.11725001037120819, + "tpp_threshold_50_total_metric": 0.27125000953674316, + "tpp_threshold_50_intended_diff_only": 0.47800004482269287, + "tpp_threshold_50_unintended_diff_only": 0.2067500352859497, + "tpp_threshold_100_total_metric": 0.22150003910064697, + "tpp_threshold_100_intended_diff_only": 0.48200005292892456, + "tpp_threshold_100_unintended_diff_only": 0.2605000138282776, + "tpp_threshold_500_total_metric": 0.13700005412101746, + "tpp_threshold_500_intended_diff_only": 0.49000006914138794, + "tpp_threshold_500_unintended_diff_only": 0.3530000150203705 }, "9": { - "tpp_threshold_2_total_metric": 0.003999993205070496, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_5_total_metric": 0.02449999749660492, + "tpp_threshold_2_total_metric": 0.0072500258684158325, + "tpp_threshold_2_intended_diff_only": 0.008000016212463379, + "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.021999984979629517, "tpp_threshold_5_intended_diff_only": 0.03700000047683716, - "tpp_threshold_5_unintended_diff_only": 0.012500002980232239, - "tpp_threshold_10_total_metric": 0.184750035405159, - "tpp_threshold_10_intended_diff_only": 0.2510000467300415, - "tpp_threshold_10_unintended_diff_only": 0.06625001132488251 + "tpp_threshold_5_unintended_diff_only": 0.015000015497207642, + "tpp_threshold_10_total_metric": 0.1820000559091568, + "tpp_threshold_10_intended_diff_only": 0.2500000596046448, + "tpp_threshold_10_unintended_diff_only": 0.06800000369548798, + "tpp_threshold_20_total_metric": 0.24300003051757812, + "tpp_threshold_20_intended_diff_only": 0.3110000491142273, + "tpp_threshold_20_unintended_diff_only": 0.06800001859664917, + "tpp_threshold_50_total_metric": 0.257750004529953, + "tpp_threshold_50_intended_diff_only": 0.4010000228881836, + "tpp_threshold_50_unintended_diff_only": 0.1432500183582306, + "tpp_threshold_100_total_metric": 0.23000004887580872, + "tpp_threshold_100_intended_diff_only": 0.4440000653266907, + "tpp_threshold_100_unintended_diff_only": 0.21400001645088196, + "tpp_threshold_500_total_metric": 0.13075004518032074, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.3452500253915787 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.009499996900558472, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": -0.004500001668930054, - "tpp_threshold_5_total_metric": 0.014000028371810913, - "tpp_threshold_5_intended_diff_only": 0.012000024318695068, - "tpp_threshold_5_unintended_diff_only": -0.0020000040531158447, - "tpp_threshold_10_total_metric": -0.000250011682510376, + "tpp_threshold_2_total_metric": 0.0029999762773513794, + "tpp_threshold_2_intended_diff_only": 0.006999969482421875, + "tpp_threshold_2_unintended_diff_only": 0.003999993205070496, + "tpp_threshold_5_total_metric": 0.006999999284744263, + "tpp_threshold_5_intended_diff_only": 0.009000003337860107, + "tpp_threshold_5_unintended_diff_only": 0.0020000040531158447, + "tpp_threshold_10_total_metric": 0.0017500072717666626, "tpp_threshold_10_intended_diff_only": 0.004999995231628418, - "tpp_threshold_10_unintended_diff_only": 0.005250006914138794 + "tpp_threshold_10_unintended_diff_only": 0.0032499879598617554, + "tpp_threshold_20_total_metric": 0.010000020265579224, + "tpp_threshold_20_intended_diff_only": 0.013000011444091797, + "tpp_threshold_20_unintended_diff_only": 0.0029999911785125732, + "tpp_threshold_50_total_metric": 0.05274999141693115, + "tpp_threshold_50_intended_diff_only": 0.05199998617172241, + "tpp_threshold_50_unintended_diff_only": -0.0007500052452087402, + "tpp_threshold_100_total_metric": 0.10674998164176941, + "tpp_threshold_100_intended_diff_only": 0.11299997568130493, + "tpp_threshold_100_unintended_diff_only": 0.0062499940395355225, + "tpp_threshold_500_total_metric": 0.39225003123283386, + "tpp_threshold_500_intended_diff_only": 0.406000018119812, + "tpp_threshold_500_unintended_diff_only": 0.01374998688697815 }, "2": { - "tpp_threshold_2_total_metric": 0.0015000402927398682, - "tpp_threshold_2_intended_diff_only": 0.00700002908706665, - "tpp_threshold_2_unintended_diff_only": 0.005499988794326782, - "tpp_threshold_5_total_metric": 0.007750019431114197, - "tpp_threshold_5_intended_diff_only": 0.013999998569488525, - "tpp_threshold_5_unintended_diff_only": 0.006249979138374329, - "tpp_threshold_10_total_metric": 0.03450004756450653, - "tpp_threshold_10_intended_diff_only": 0.03800004720687866, - "tpp_threshold_10_unintended_diff_only": 0.0034999996423721313 + "tpp_threshold_2_total_metric": 0.011000007390975952, + "tpp_threshold_2_intended_diff_only": 0.009000003337860107, + "tpp_threshold_2_unintended_diff_only": -0.0020000040531158447, + "tpp_threshold_5_total_metric": 0.0022500157356262207, + "tpp_threshold_5_intended_diff_only": 0.009000003337860107, + "tpp_threshold_5_unintended_diff_only": 0.006749987602233887, + "tpp_threshold_10_total_metric": 0.034750014543533325, + "tpp_threshold_10_intended_diff_only": 0.04100000858306885, + "tpp_threshold_10_unintended_diff_only": 0.0062499940395355225, + "tpp_threshold_20_total_metric": 0.07700005173683167, + "tpp_threshold_20_intended_diff_only": 0.07900005578994751, + "tpp_threshold_20_unintended_diff_only": 0.0020000040531158447, + "tpp_threshold_50_total_metric": 0.13475003838539124, + "tpp_threshold_50_intended_diff_only": 0.14600002765655518, + "tpp_threshold_50_unintended_diff_only": 0.01124998927116394, + "tpp_threshold_100_total_metric": 0.22825007140636444, + "tpp_threshold_100_intended_diff_only": 0.24600005149841309, + "tpp_threshold_100_unintended_diff_only": 0.017749980092048645, + "tpp_threshold_500_total_metric": 0.40375009179115295, + "tpp_threshold_500_intended_diff_only": 0.43400007486343384, + "tpp_threshold_500_unintended_diff_only": 0.030249983072280884 }, "3": { - "tpp_threshold_2_total_metric": 0.012499988079071045, - "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_2_unintended_diff_only": -0.0065000057220458984, - "tpp_threshold_5_total_metric": 0.015750005841255188, - "tpp_threshold_5_intended_diff_only": 0.013999998569488525, - "tpp_threshold_5_unintended_diff_only": -0.0017500072717666626, - "tpp_threshold_10_total_metric": 0.016500011086463928, - "tpp_threshold_10_intended_diff_only": 0.018000006675720215, - "tpp_threshold_10_unintended_diff_only": 0.0014999955892562866 + "tpp_threshold_2_total_metric": 0.003500029444694519, + "tpp_threshold_2_intended_diff_only": 0.003000020980834961, + "tpp_threshold_2_unintended_diff_only": -0.0005000084638595581, + "tpp_threshold_5_total_metric": 0.012999996542930603, + "tpp_threshold_5_intended_diff_only": 0.008000016212463379, + "tpp_threshold_5_unintended_diff_only": -0.004999980330467224, + "tpp_threshold_10_total_metric": 0.017499983310699463, + "tpp_threshold_10_intended_diff_only": 0.014999985694885254, + "tpp_threshold_10_unintended_diff_only": -0.002499997615814209, + "tpp_threshold_20_total_metric": 0.013499990105628967, + "tpp_threshold_20_intended_diff_only": 0.019999980926513672, + "tpp_threshold_20_unintended_diff_only": 0.006499990820884705, + "tpp_threshold_50_total_metric": 0.060499995946884155, + "tpp_threshold_50_intended_diff_only": 0.06199997663497925, + "tpp_threshold_50_unintended_diff_only": 0.0014999806880950928, + "tpp_threshold_100_total_metric": 0.13300003111362457, + "tpp_threshold_100_intended_diff_only": 0.14100003242492676, + "tpp_threshold_100_unintended_diff_only": 0.008000001311302185, + "tpp_threshold_500_total_metric": 0.36525003612041473, + "tpp_threshold_500_intended_diff_only": 0.39100003242492676, + "tpp_threshold_500_unintended_diff_only": 0.025749996304512024 }, "5": { - "tpp_threshold_2_total_metric": -0.007499992847442627, - "tpp_threshold_2_intended_diff_only": 0.0, - "tpp_threshold_2_unintended_diff_only": 0.007499992847442627, - "tpp_threshold_5_total_metric": -1.4901161193847656e-08, - "tpp_threshold_5_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_5_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_10_total_metric": 0.000999942421913147, - "tpp_threshold_10_intended_diff_only": 0.003999948501586914, - "tpp_threshold_10_unintended_diff_only": 0.003000006079673767 + "tpp_threshold_2_total_metric": 0.002249971032142639, + "tpp_threshold_2_intended_diff_only": 0.001999974250793457, + "tpp_threshold_2_unintended_diff_only": -0.00024999678134918213, + "tpp_threshold_5_total_metric": 0.00824996829032898, + "tpp_threshold_5_intended_diff_only": 0.006999969482421875, + "tpp_threshold_5_unintended_diff_only": -0.0012499988079071045, + "tpp_threshold_10_total_metric": -0.0007499903440475464, + "tpp_threshold_10_intended_diff_only": 0.004999995231628418, + "tpp_threshold_10_unintended_diff_only": 0.005749985575675964, + "tpp_threshold_20_total_metric": 0.012499988079071045, + "tpp_threshold_20_intended_diff_only": 0.013999998569488525, + "tpp_threshold_20_unintended_diff_only": 0.0015000104904174805, + "tpp_threshold_50_total_metric": 0.056499943137168884, + "tpp_threshold_50_intended_diff_only": 0.0639999508857727, + "tpp_threshold_50_unintended_diff_only": 0.007500007748603821, + "tpp_threshold_100_total_metric": 0.1487499624490738, + "tpp_threshold_100_intended_diff_only": 0.16399997472763062, + "tpp_threshold_100_unintended_diff_only": 0.015250012278556824, + "tpp_threshold_500_total_metric": 0.36150000989437103, + "tpp_threshold_500_intended_diff_only": 0.3920000195503235, + "tpp_threshold_500_unintended_diff_only": 0.030500009655952454 }, "6": { - "tpp_threshold_2_total_metric": 0.036250025033950806, - "tpp_threshold_2_intended_diff_only": 0.03600001335144043, - "tpp_threshold_2_unintended_diff_only": -0.000250011682510376, - "tpp_threshold_5_total_metric": 0.05049997568130493, - "tpp_threshold_5_intended_diff_only": 0.05699998140335083, - "tpp_threshold_5_unintended_diff_only": 0.0065000057220458984, - "tpp_threshold_10_total_metric": 0.08199998736381531, - "tpp_threshold_10_intended_diff_only": 0.08399999141693115, - "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447 + "tpp_threshold_2_total_metric": 0.025749966502189636, + "tpp_threshold_2_intended_diff_only": 0.029999971389770508, + "tpp_threshold_2_unintended_diff_only": 0.004250004887580872, + "tpp_threshold_5_total_metric": 0.05474996566772461, + "tpp_threshold_5_intended_diff_only": 0.05399996042251587, + "tpp_threshold_5_unintended_diff_only": -0.0007500052452087402, + "tpp_threshold_10_total_metric": 0.08724996447563171, + "tpp_threshold_10_intended_diff_only": 0.09099996089935303, + "tpp_threshold_10_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_20_total_metric": 0.12574999034404755, + "tpp_threshold_20_intended_diff_only": 0.13599997758865356, + "tpp_threshold_20_unintended_diff_only": 0.010249987244606018, + "tpp_threshold_50_total_metric": 0.195250004529953, + "tpp_threshold_50_intended_diff_only": 0.203000009059906, + "tpp_threshold_50_unintended_diff_only": 0.007750004529953003, + "tpp_threshold_100_total_metric": 0.27950000762939453, + "tpp_threshold_100_intended_diff_only": 0.29100000858306885, + "tpp_threshold_100_unintended_diff_only": 0.011500000953674316, + "tpp_threshold_500_total_metric": 0.3407500237226486, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.03225000202655792 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-170898-tokens-700000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-170898-tokens-700000000/scr/saebench_scr_custom_sae_eval_results.json index 1d742a5065b1c2f9867814a422752f852b72b6d2..054b5107f855c91e36fb39ea97c591f086a909bf 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-170898-tokens-700000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-170898-tokens-700000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "72f7ea16-cc54-4416-97a9-3b7d73da261b", - "datetime_epoch_millis": 1776884203624, + "eval_id": "b8f25c94-279d-4df8-bf72-7932d42c1f31", + "datetime_epoch_millis": 1777316076815, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.2861429048424335, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.3503226092500692, "scr_metric_threshold_10": 0.2318408624917077, "scr_dir2_threshold_10": 0.22440638480937283, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3109722239202221, + "scr_metric_threshold_20": 0.22861909642889994, + "scr_dir2_threshold_20": 0.21582578096982383, + "scr_dir1_threshold_50": 0.2613805711154749, + "scr_metric_threshold_50": 0.3253183183403778, + "scr_dir2_threshold_50": 0.3187309585351685, + "scr_dir1_threshold_100": 0.01770920368435549, + "scr_metric_threshold_100": 0.32543424271638177, + "scr_dir2_threshold_100": 0.30850803853263165, + "scr_dir1_threshold_500": -0.7972597500589058, + "scr_metric_threshold_500": 0.17700663121698362, + "scr_dir2_threshold_500": 0.16320574345720348 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.5641028776049328, "scr_metric_threshold_10": 0.06217613139950766, "scr_dir2_threshold_10": 0.06217613139950766, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.5128207283533913, + "scr_metric_threshold_20": 0.11658034288417893, + "scr_dir2_threshold_20": 0.11658034288417893, + "scr_dir1_threshold_50": 0.5384614208981502, + "scr_metric_threshold_50": 0.2564765999290303, + "scr_dir2_threshold_50": 0.2564765999290303, + "scr_dir1_threshold_100": 0.5641028776049328, + "scr_metric_threshold_100": 0.2564765999290303, + "scr_dir2_threshold_100": 0.2564765999290303, + "scr_dir1_threshold_500": -2.3461531701643636, + "scr_metric_threshold_500": 0.010362714302611827, + "scr_dir2_threshold_500": 0.010362714302611827 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.5000002257751198, "scr_metric_threshold_10": 0.321100954221084, "scr_dir2_threshold_10": 0.321100954221084, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.5000002257751198, + "scr_metric_threshold_20": 0.03363925521540607, + "scr_dir2_threshold_20": 0.03363925521540607, + "scr_dir1_threshold_50": 0.2196972536262871, + "scr_metric_threshold_50": 0.3333333940923963, + "scr_dir2_threshold_50": 0.3333333940923963, + "scr_dir1_threshold_100": -0.12121188859533107, + "scr_metric_threshold_100": 0.5412845073503276, + "scr_dir2_threshold_100": 0.5412845073503276, + "scr_dir1_threshold_500": -1.0606057185225457, + "scr_metric_threshold_500": 0.06727832815362321, + "scr_dir2_threshold_500": 0.06727832815362321 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.4687504074535596, "scr_metric_threshold_10": 0.12311562116192198, "scr_dir2_threshold_10": 0.12311562116192198, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3437501746229541, + "scr_metric_threshold_20": 0.17085429167603294, + "scr_dir2_threshold_20": 0.17085429167603294, + "scr_dir1_threshold_50": 0.09374970896174313, + "scr_metric_threshold_50": 0.22361817210553975, + "scr_dir2_threshold_50": 0.22361817210553975, + "scr_dir1_threshold_100": -1.6874994179234863, + "scr_metric_threshold_100": 0.04020108028160345, + "scr_dir2_threshold_100": 0.04020108028160345, + "scr_dir1_threshold_500": -2.2812500582076516, + "scr_metric_threshold_500": -0.06783921065491268, + "scr_dir2_threshold_500": -0.06783921065491268 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": 0.24161044561475795, "scr_metric_threshold_10": 0.12738859911246028, "scr_dir2_threshold_10": 0.12738859911246028, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.1208054228229802, + "scr_metric_threshold_20": 0.035031964413370725, + "scr_dir2_threshold_20": 0.035031964413370725, + "scr_dir1_threshold_50": 0.16778508723680446, + "scr_metric_threshold_50": 0.18471345922188226, + "scr_dir2_threshold_50": 0.18471345922188226, + "scr_dir1_threshold_100": 0.10738237582531437, + "scr_metric_threshold_100": -0.03184710232218929, + "scr_dir2_threshold_100": -0.03184710232218929, + "scr_dir1_threshold_500": -1.7852352483493712, + "scr_metric_threshold_500": -0.5414013089483273, + "scr_dir2_threshold_500": -0.5414013089483273 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.08421069803000372, "scr_metric_threshold_10": 0.4603173852297345, "scr_dir2_threshold_10": 0.4603173852297345, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.09473687842942387, + "scr_metric_threshold_20": 0.5515872218065929, + "scr_dir2_threshold_20": 0.5515872218065929, + "scr_dir1_threshold_50": -0.17894726275076694, + "scr_metric_threshold_50": 0.6507936996006726, + "scr_dir2_threshold_50": 0.6507936996006726, + "scr_dir1_threshold_100": -0.17894726275076694, + "scr_metric_threshold_100": 0.6626983066369999, + "scr_dir2_threshold_100": 0.6626983066369999, + "scr_dir1_threshold_500": -0.17894726275076694, + "scr_metric_threshold_500": 0.742063536177531, + "scr_dir2_threshold_500": 0.742063536177531 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.4043716594522235, "scr_metric_threshold_10": 0.14503810715685644, "scr_dir2_threshold_10": 0.14503810715685644, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.4316940424658739, + "scr_metric_threshold_20": 0.17938926174649328, + "scr_dir2_threshold_20": 0.17938926174649328, + "scr_dir1_threshold_50": 0.4316940424658739, + "scr_metric_threshold_50": 0.2671755204201627, + "scr_dir2_threshold_50": 0.2671755204201627, + "scr_dir1_threshold_100": 0.5136611915068252, + "scr_metric_threshold_100": 0.3931296780826234, + "scr_dir2_threshold_100": 0.3931296780826234, + "scr_dir1_threshold_500": 0.622950723561427, + "scr_metric_threshold_500": 0.3206106245041952, + "scr_dir2_threshold_500": 0.3206106245041952 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.3488370996994263, "scr_metric_threshold_10": 0.4248926412815665, "scr_dir2_threshold_10": 0.4248926412815665, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.45581383614879756, + "scr_metric_threshold_20": 0.47210310529992777, + "scr_dir2_threshold_20": 0.47210310529992777, + "scr_dir1_threshold_50": 0.5441858866203395, + "scr_metric_threshold_50": 0.41201726948797046, + "scr_dir2_threshold_50": 0.41201726948797046, + "scr_dir1_threshold_100": 0.6232557327183983, + "scr_metric_threshold_100": 0.4206008506837011, + "scr_dir2_threshold_100": 0.4206008506837011, + "scr_dir1_threshold_500": 0.2511628448544011, + "scr_metric_threshold_500": 0.48497847709352376, + "scr_dir2_threshold_500": 0.48497847709352376 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.19069746037053023, "scr_metric_threshold_10": 0.19069746037053023, "scr_dir2_threshold_10": 0.1312216389118512, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.26976732838919676, + "scr_metric_threshold_20": 0.26976732838919676, + "scr_dir2_threshold_20": 0.16742080471658802, + "scr_dir1_threshold_50": 0.2744184318653678, + "scr_metric_threshold_50": 0.2744184318653678, + "scr_dir2_threshold_50": 0.22171955342369326, + "scr_dir1_threshold_100": 0.3209300210889576, + "scr_metric_threshold_100": 0.3209300210889576, + "scr_dir2_threshold_100": 0.18552038761895642, + "scr_dir1_threshold_500": 0.3999998891076241, + "scr_metric_threshold_500": 0.3999998891076241, + "scr_dir2_threshold_500": 0.2895927870293829 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-170898-tokens-700000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-170898-tokens-700000000/tpp/saebench_tpp_custom_sae_eval_results.json index 29ec2ca1e930480126fea057347e0bebca996475..3d3440e0fa312aa7eed2842f5c033e96b11bee27 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-170898-tokens-700000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-170898-tokens-700000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "a9f771d2-6cd5-486e-8f85-db7c9d52ffa2", - "datetime_epoch_millis": 1776884352521, + "eval_id": "4d27b65a-6125-410a-a362-4f2cb046f70a", + "datetime_epoch_millis": 1777316363997, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.023974995315074923, - "tpp_threshold_2_intended_diff_only": 0.02619999647140503, - "tpp_threshold_2_unintended_diff_only": 0.0022250011563301085, - "tpp_threshold_5_total_metric": 0.07752500176429748, - "tpp_threshold_5_intended_diff_only": 0.09830000400543212, - "tpp_threshold_5_unintended_diff_only": 0.02077500224113464, - "tpp_threshold_10_total_metric": 0.09580000191926956, - "tpp_threshold_10_intended_diff_only": 0.13700000643730165, - "tpp_threshold_10_unintended_diff_only": 0.041200004518032074, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.022100001573562622, + "tpp_threshold_2_intended_diff_only": 0.0252000093460083, + "tpp_threshold_2_unintended_diff_only": 0.0031000077724456787, + "tpp_threshold_5_total_metric": 0.07795000076293945, + "tpp_threshold_5_intended_diff_only": 0.0971000075340271, + "tpp_threshold_5_unintended_diff_only": 0.019150006771087646, + "tpp_threshold_10_total_metric": 0.09620000272989274, + "tpp_threshold_10_intended_diff_only": 0.1386000096797943, + "tpp_threshold_10_unintended_diff_only": 0.04240000694990158, + "tpp_threshold_20_total_metric": 0.13112501055002213, + "tpp_threshold_20_intended_diff_only": 0.1983000159263611, + "tpp_threshold_20_unintended_diff_only": 0.06717500537633896, + "tpp_threshold_50_total_metric": 0.1575250133872032, + "tpp_threshold_50_intended_diff_only": 0.25490002036094667, + "tpp_threshold_50_unintended_diff_only": 0.09737500697374343, + "tpp_threshold_100_total_metric": 0.1825750067830086, + "tpp_threshold_100_intended_diff_only": 0.31490001678466795, + "tpp_threshold_100_unintended_diff_only": 0.1323250100016594, + "tpp_threshold_500_total_metric": 0.24650002270936966, + "tpp_threshold_500_intended_diff_only": 0.43620003461837764, + "tpp_threshold_500_unintended_diff_only": 0.18970001190900804 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.037199997901916505, - "tpp_threshold_2_intended_diff_only": 0.04140000343322754, - "tpp_threshold_2_unintended_diff_only": 0.004200005531311035, - "tpp_threshold_5_total_metric": 0.13485001623630524, - "tpp_threshold_5_intended_diff_only": 0.17480002641677855, - "tpp_threshold_5_unintended_diff_only": 0.039950010180473325, - "tpp_threshold_10_total_metric": 0.1674000084400177, + "tpp_threshold_2_total_metric": 0.03554998934268951, + "tpp_threshold_2_intended_diff_only": 0.041200006008148195, + "tpp_threshold_2_unintended_diff_only": 0.005650016665458679, + "tpp_threshold_5_total_metric": 0.1366000145673752, + "tpp_threshold_5_intended_diff_only": 0.17500002384185792, + "tpp_threshold_5_unintended_diff_only": 0.03840000927448273, + "tpp_threshold_10_total_metric": 0.16530000269412995, "tpp_threshold_10_intended_diff_only": 0.24740002155303956, - "tpp_threshold_10_unintended_diff_only": 0.08000001311302185, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_10_unintended_diff_only": 0.0821000188589096, + "tpp_threshold_20_total_metric": 0.216100013256073, + "tpp_threshold_20_intended_diff_only": 0.3470000267028809, + "tpp_threshold_20_unintended_diff_only": 0.13090001344680785, + "tpp_threshold_50_total_metric": 0.21220001578330994, + "tpp_threshold_50_intended_diff_only": 0.40200003385543825, + "tpp_threshold_50_unintended_diff_only": 0.1898000180721283, + "tpp_threshold_100_total_metric": 0.1813500016927719, + "tpp_threshold_100_intended_diff_only": 0.4350000262260437, + "tpp_threshold_100_unintended_diff_only": 0.2536500245332718, + "tpp_threshold_500_total_metric": 0.11185001730918884, + "tpp_threshold_500_intended_diff_only": 0.4670000433921814, + "tpp_threshold_500_unintended_diff_only": 0.35515002608299256 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.010749992728233338, - "tpp_threshold_2_intended_diff_only": 0.01099998950958252, - "tpp_threshold_2_unintended_diff_only": 0.00024999678134918213, - "tpp_threshold_5_total_metric": 0.020199987292289733, - "tpp_threshold_5_intended_diff_only": 0.021799981594085693, - "tpp_threshold_5_unintended_diff_only": 0.0015999943017959595, - "tpp_threshold_10_total_metric": 0.024199995398521423, - "tpp_threshold_10_intended_diff_only": 0.02659999132156372, - "tpp_threshold_10_unintended_diff_only": 0.0023999959230422974, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.00865001380443573, + "tpp_threshold_2_intended_diff_only": 0.009200012683868409, + "tpp_threshold_2_unintended_diff_only": 0.0005499988794326783, + "tpp_threshold_5_total_metric": 0.019299986958503722, + "tpp_threshold_5_intended_diff_only": 0.01919999122619629, + "tpp_threshold_5_unintended_diff_only": -9.999573230743409e-05, + "tpp_threshold_10_total_metric": 0.027100002765655516, + "tpp_threshold_10_intended_diff_only": 0.029799997806549072, + "tpp_threshold_10_unintended_diff_only": 0.0026999950408935548, + "tpp_threshold_20_total_metric": 0.046150007843971254, + "tpp_threshold_20_intended_diff_only": 0.04960000514984131, + "tpp_threshold_20_unintended_diff_only": 0.003449997305870056, + "tpp_threshold_50_total_metric": 0.1028500109910965, + "tpp_threshold_50_intended_diff_only": 0.10780000686645508, + "tpp_threshold_50_unintended_diff_only": 0.004949995875358581, + "tpp_threshold_100_total_metric": 0.18380001187324524, + "tpp_threshold_100_intended_diff_only": 0.19480000734329223, + "tpp_threshold_100_unintended_diff_only": 0.010999995470046996, + "tpp_threshold_500_total_metric": 0.38115002810955045, + "tpp_threshold_500_intended_diff_only": 0.40540002584457396, + "tpp_threshold_500_unintended_diff_only": 0.0242499977350235 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.003250017762184143, "tpp_threshold_10_total_metric": 0.015749961137771606, "tpp_threshold_10_intended_diff_only": 0.018999993801116943, - "tpp_threshold_10_unintended_diff_only": 0.003250032663345337 + "tpp_threshold_10_unintended_diff_only": 0.003250032663345337, + "tpp_threshold_20_total_metric": 0.21974997222423553, + "tpp_threshold_20_intended_diff_only": 0.32999998331069946, + "tpp_threshold_20_unintended_diff_only": 0.11025001108646393, + "tpp_threshold_50_total_metric": 0.2395000010728836, + "tpp_threshold_50_intended_diff_only": 0.3920000195503235, + "tpp_threshold_50_unintended_diff_only": 0.15250001847743988, + "tpp_threshold_100_total_metric": 0.2202499508857727, + "tpp_threshold_100_intended_diff_only": 0.43699997663497925, + "tpp_threshold_100_unintended_diff_only": 0.21675002574920654, + "tpp_threshold_500_total_metric": 0.1302500069141388, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.3257500231266022 }, "1": { - "tpp_threshold_2_total_metric": 0.05525003373622894, - "tpp_threshold_2_intended_diff_only": 0.06700003147125244, - "tpp_threshold_2_unintended_diff_only": 0.011749997735023499, - "tpp_threshold_5_total_metric": 0.17875003814697266, + "tpp_threshold_2_total_metric": 0.057999953627586365, + "tpp_threshold_2_intended_diff_only": 0.06599998474121094, + "tpp_threshold_2_unintended_diff_only": 0.008000031113624573, + "tpp_threshold_5_total_metric": 0.18100003898143768, "tpp_threshold_5_intended_diff_only": 0.2420000433921814, - "tpp_threshold_5_unintended_diff_only": 0.06325000524520874, - "tpp_threshold_10_total_metric": 0.19749999046325684, + "tpp_threshold_5_unintended_diff_only": 0.06100000441074371, + "tpp_threshold_10_total_metric": 0.19049999117851257, "tpp_threshold_10_intended_diff_only": 0.3100000023841858, - "tpp_threshold_10_unintended_diff_only": 0.11250001192092896 + "tpp_threshold_10_unintended_diff_only": 0.11950001120567322, + "tpp_threshold_20_total_metric": 0.21650002896785736, + "tpp_threshold_20_intended_diff_only": 0.34400004148483276, + "tpp_threshold_20_unintended_diff_only": 0.1275000125169754, + "tpp_threshold_50_total_metric": 0.2004999965429306, + "tpp_threshold_50_intended_diff_only": 0.37700003385543823, + "tpp_threshold_50_unintended_diff_only": 0.17650003731250763, + "tpp_threshold_100_total_metric": 0.16725000739097595, + "tpp_threshold_100_intended_diff_only": 0.4150000214576721, + "tpp_threshold_100_unintended_diff_only": 0.24775001406669617, + "tpp_threshold_500_total_metric": 0.09400002658367157, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.36500002443790436 }, "2": { - "tpp_threshold_2_total_metric": 0.10600000619888306, - "tpp_threshold_2_intended_diff_only": 0.11500000953674316, - "tpp_threshold_2_unintended_diff_only": 0.009000003337860107, - "tpp_threshold_5_total_metric": 0.2092500627040863, - "tpp_threshold_5_intended_diff_only": 0.2730000615119934, - "tpp_threshold_5_unintended_diff_only": 0.0637499988079071, - "tpp_threshold_10_total_metric": 0.17775003612041473, - "tpp_threshold_10_intended_diff_only": 0.3250000476837158, - "tpp_threshold_10_unintended_diff_only": 0.1472500115633011 + "tpp_threshold_2_total_metric": 0.09425003826618195, + "tpp_threshold_2_intended_diff_only": 0.11200004816055298, + "tpp_threshold_2_unintended_diff_only": 0.017750009894371033, + "tpp_threshold_5_total_metric": 0.2120000123977661, + "tpp_threshold_5_intended_diff_only": 0.2720000147819519, + "tpp_threshold_5_unintended_diff_only": 0.06000000238418579, + "tpp_threshold_10_total_metric": 0.17725001275539398, + "tpp_threshold_10_intended_diff_only": 0.32600003480911255, + "tpp_threshold_10_unintended_diff_only": 0.14875002205371857, + "tpp_threshold_20_total_metric": 0.138000026345253, + "tpp_threshold_20_intended_diff_only": 0.359000027179718, + "tpp_threshold_20_unintended_diff_only": 0.22100000083446503, + "tpp_threshold_50_total_metric": 0.11350005865097046, + "tpp_threshold_50_intended_diff_only": 0.37900006771087646, + "tpp_threshold_50_unintended_diff_only": 0.265500009059906, + "tpp_threshold_100_total_metric": 0.08675001561641693, + "tpp_threshold_100_intended_diff_only": 0.409000039100647, + "tpp_threshold_100_unintended_diff_only": 0.32225002348423004, + "tpp_threshold_500_total_metric": 0.054749995470047, + "tpp_threshold_500_intended_diff_only": 0.4520000219345093, + "tpp_threshold_500_unintended_diff_only": 0.3972500264644623 }, "6": { - "tpp_threshold_2_total_metric": 0.010750025510787964, + "tpp_threshold_2_total_metric": 0.008249998092651367, "tpp_threshold_2_intended_diff_only": 0.008000016212463379, - "tpp_threshold_2_unintended_diff_only": -0.002750009298324585, - "tpp_threshold_5_total_metric": 0.24574998021125793, - "tpp_threshold_5_intended_diff_only": 0.3050000071525574, - "tpp_threshold_5_unintended_diff_only": 0.05925002694129944, - "tpp_threshold_10_total_metric": 0.27025002241134644, + "tpp_threshold_2_unintended_diff_only": -0.0002499818801879883, + "tpp_threshold_5_total_metric": 0.2525000423192978, + "tpp_threshold_5_intended_diff_only": 0.3060000538825989, + "tpp_threshold_5_unintended_diff_only": 0.053500011563301086, + "tpp_threshold_10_total_metric": 0.2680000215768814, "tpp_threshold_10_intended_diff_only": 0.3450000286102295, - "tpp_threshold_10_unintended_diff_only": 0.07475000619888306 + "tpp_threshold_10_unintended_diff_only": 0.07700000703334808, + "tpp_threshold_20_total_metric": 0.28675003349781036, + "tpp_threshold_20_intended_diff_only": 0.41200006008148193, + "tpp_threshold_20_unintended_diff_only": 0.12525002658367157, + "tpp_threshold_50_total_metric": 0.2657500058412552, + "tpp_threshold_50_intended_diff_only": 0.47600001096725464, + "tpp_threshold_50_unintended_diff_only": 0.21025000512599945, + "tpp_threshold_100_total_metric": 0.2095000147819519, + "tpp_threshold_100_intended_diff_only": 0.4830000400543213, + "tpp_threshold_100_unintended_diff_only": 0.2735000252723694, + "tpp_threshold_500_total_metric": 0.14275000989437103, + "tpp_threshold_500_intended_diff_only": 0.4920000433921814, + "tpp_threshold_500_unintended_diff_only": 0.34925003349781036 }, "9": { - "tpp_threshold_2_total_metric": 0.0037499815225601196, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983, - "tpp_threshold_5_total_metric": 0.0207500159740448, - "tpp_threshold_5_intended_diff_only": 0.03100001811981201, - "tpp_threshold_5_unintended_diff_only": 0.010250002145767212, - "tpp_threshold_10_total_metric": 0.1757500320672989, - "tpp_threshold_10_intended_diff_only": 0.2380000352859497, - "tpp_threshold_10_unintended_diff_only": 0.06225000321865082 + "tpp_threshold_2_total_metric": 0.0070000141859054565, + "tpp_threshold_2_intended_diff_only": 0.008000016212463379, + "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, + "tpp_threshold_5_total_metric": 0.01774999499320984, + "tpp_threshold_5_intended_diff_only": 0.03200000524520874, + "tpp_threshold_5_unintended_diff_only": 0.014250010251998901, + "tpp_threshold_10_total_metric": 0.17500002682209015, + "tpp_threshold_10_intended_diff_only": 0.23700004816055298, + "tpp_threshold_10_unintended_diff_only": 0.06200002133846283, + "tpp_threshold_20_total_metric": 0.21950000524520874, + "tpp_threshold_20_intended_diff_only": 0.2900000214576721, + "tpp_threshold_20_unintended_diff_only": 0.07050001621246338, + "tpp_threshold_50_total_metric": 0.24175001680850983, + "tpp_threshold_50_intended_diff_only": 0.38600003719329834, + "tpp_threshold_50_unintended_diff_only": 0.1442500203847885, + "tpp_threshold_100_total_metric": 0.22300001978874207, + "tpp_threshold_100_intended_diff_only": 0.4310000538825989, + "tpp_threshold_100_unintended_diff_only": 0.2080000340938568, + "tpp_threshold_500_total_metric": 0.13750004768371582, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.3385000228881836 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.012249961495399475, - "tpp_threshold_2_intended_diff_only": 0.006999969482421875, - "tpp_threshold_2_unintended_diff_only": -0.0052499920129776, - "tpp_threshold_5_total_metric": 0.013999983668327332, - "tpp_threshold_5_intended_diff_only": 0.010999977588653564, - "tpp_threshold_5_unintended_diff_only": -0.003000006079673767, - "tpp_threshold_10_total_metric": 0.003250032663345337, - "tpp_threshold_10_intended_diff_only": 0.008000016212463379, - "tpp_threshold_10_unintended_diff_only": 0.004749983549118042 + "tpp_threshold_2_total_metric": 0.005250021815299988, + "tpp_threshold_2_intended_diff_only": 0.008000016212463379, + "tpp_threshold_2_unintended_diff_only": 0.002749994397163391, + "tpp_threshold_5_total_metric": 0.0065000057220458984, + "tpp_threshold_5_intended_diff_only": 0.008000016212463379, + "tpp_threshold_5_unintended_diff_only": 0.0015000104904174805, + "tpp_threshold_10_total_metric": 0.004749968647956848, + "tpp_threshold_10_intended_diff_only": 0.006999969482421875, + "tpp_threshold_10_unintended_diff_only": 0.002250000834465027, + "tpp_threshold_20_total_metric": 0.0169999897480011, + "tpp_threshold_20_intended_diff_only": 0.018999993801116943, + "tpp_threshold_20_unintended_diff_only": 0.0020000040531158447, + "tpp_threshold_50_total_metric": 0.058500006794929504, + "tpp_threshold_50_intended_diff_only": 0.0559999942779541, + "tpp_threshold_50_unintended_diff_only": -0.002500012516975403, + "tpp_threshold_100_total_metric": 0.1300000250339508, + "tpp_threshold_100_intended_diff_only": 0.13700002431869507, + "tpp_threshold_100_unintended_diff_only": 0.006999999284744263, + "tpp_threshold_500_total_metric": 0.4007500261068344, + "tpp_threshold_500_intended_diff_only": 0.4150000214576721, + "tpp_threshold_500_unintended_diff_only": 0.014249995350837708 }, "2": { - "tpp_threshold_2_total_metric": 0.003250017762184143, - "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": 0.005749985575675964, - "tpp_threshold_5_total_metric": 0.021250024437904358, - "tpp_threshold_5_intended_diff_only": 0.02799999713897705, - "tpp_threshold_5_unintended_diff_only": 0.006749972701072693, - "tpp_threshold_10_total_metric": 0.03425002098083496, - "tpp_threshold_10_intended_diff_only": 0.03600001335144043, - "tpp_threshold_10_unintended_diff_only": 0.0017499923706054688 + "tpp_threshold_2_total_metric": 0.012000054121017456, + "tpp_threshold_2_intended_diff_only": 0.010000050067901611, + "tpp_threshold_2_unintended_diff_only": -0.0020000040531158447, + "tpp_threshold_5_total_metric": 0.017000064253807068, + "tpp_threshold_5_intended_diff_only": 0.024000048637390137, + "tpp_threshold_5_unintended_diff_only": 0.006999984383583069, + "tpp_threshold_10_total_metric": 0.034000054001808167, + "tpp_threshold_10_intended_diff_only": 0.03900003433227539, + "tpp_threshold_10_unintended_diff_only": 0.004999980330467224, + "tpp_threshold_20_total_metric": 0.07200005650520325, + "tpp_threshold_20_intended_diff_only": 0.07500004768371582, + "tpp_threshold_20_unintended_diff_only": 0.0029999911785125732, + "tpp_threshold_50_total_metric": 0.1272500604391098, + "tpp_threshold_50_intended_diff_only": 0.1390000581741333, + "tpp_threshold_50_unintended_diff_only": 0.011749997735023499, + "tpp_threshold_100_total_metric": 0.20150001347064972, + "tpp_threshold_100_intended_diff_only": 0.2160000205039978, + "tpp_threshold_100_unintended_diff_only": 0.014500007033348083, + "tpp_threshold_500_total_metric": 0.4147500544786453, + "tpp_threshold_500_intended_diff_only": 0.43300002813339233, + "tpp_threshold_500_unintended_diff_only": 0.01824997365474701 }, "3": { - "tpp_threshold_2_total_metric": 0.013750031590461731, - "tpp_threshold_2_intended_diff_only": 0.00700002908706665, - "tpp_threshold_2_unintended_diff_only": -0.006750002503395081, - "tpp_threshold_5_total_metric": 0.01825001835823059, - "tpp_threshold_5_intended_diff_only": 0.017000019550323486, - "tpp_threshold_5_unintended_diff_only": -0.0012499988079071045, - "tpp_threshold_10_total_metric": 0.018749982118606567, - "tpp_threshold_10_intended_diff_only": 0.019999980926513672, - "tpp_threshold_10_unintended_diff_only": 0.0012499988079071045 + "tpp_threshold_2_total_metric": 0.0045000165700912476, + "tpp_threshold_2_intended_diff_only": 0.003000020980834961, + "tpp_threshold_2_unintended_diff_only": -0.0014999955892562866, + "tpp_threshold_5_total_metric": 0.014499977231025696, + "tpp_threshold_5_intended_diff_only": 0.009999990463256836, + "tpp_threshold_5_unintended_diff_only": -0.00449998676776886, + "tpp_threshold_10_total_metric": 0.019250020384788513, + "tpp_threshold_10_intended_diff_only": 0.017000019550323486, + "tpp_threshold_10_unintended_diff_only": -0.002250000834465027, + "tpp_threshold_20_total_metric": 0.00875002145767212, + "tpp_threshold_20_intended_diff_only": 0.012000024318695068, + "tpp_threshold_20_unintended_diff_only": 0.0032500028610229492, + "tpp_threshold_50_total_metric": 0.05500000715255737, + "tpp_threshold_50_intended_diff_only": 0.0559999942779541, + "tpp_threshold_50_unintended_diff_only": 0.0009999871253967285, + "tpp_threshold_100_total_metric": 0.14850005507469177, + "tpp_threshold_100_intended_diff_only": 0.15900003910064697, + "tpp_threshold_100_unintended_diff_only": 0.0104999840259552, + "tpp_threshold_500_total_metric": 0.36250004172325134, + "tpp_threshold_500_intended_diff_only": 0.39000004529953003, + "tpp_threshold_500_unintended_diff_only": 0.027500003576278687 }, "5": { - "tpp_threshold_2_total_metric": -0.006750002503395081, - "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 0.007749989628791809, - "tpp_threshold_5_total_metric": 0.0032499581575393677, - "tpp_threshold_5_intended_diff_only": 0.003999948501586914, - "tpp_threshold_5_unintended_diff_only": 0.0007499903440475464, - "tpp_threshold_10_total_metric": 0.005999937653541565, - "tpp_threshold_10_intended_diff_only": 0.008999943733215332, - "tpp_threshold_10_unintended_diff_only": 0.003000006079673767 + "tpp_threshold_2_total_metric": 0.001999974250793457, + "tpp_threshold_2_intended_diff_only": 0.001999974250793457, + "tpp_threshold_2_unintended_diff_only": 0.0, + "tpp_threshold_5_total_metric": 0.011749938130378723, + "tpp_threshold_5_intended_diff_only": 0.008999943733215332, + "tpp_threshold_5_unintended_diff_only": -0.002749994397163391, + "tpp_threshold_10_total_metric": 0.005499988794326782, + "tpp_threshold_10_intended_diff_only": 0.010999977588653564, + "tpp_threshold_10_unintended_diff_only": 0.005499988794326782, + "tpp_threshold_20_total_metric": 0.025499969720840454, + "tpp_threshold_20_intended_diff_only": 0.02599996328353882, + "tpp_threshold_20_unintended_diff_only": 0.0004999935626983643, + "tpp_threshold_50_total_metric": 0.07799996435642242, + "tpp_threshold_50_intended_diff_only": 0.08599996566772461, + "tpp_threshold_50_unintended_diff_only": 0.008000001311302185, + "tpp_threshold_100_total_metric": 0.16124998033046722, + "tpp_threshold_100_intended_diff_only": 0.17299997806549072, + "tpp_threshold_100_unintended_diff_only": 0.011749997735023499, + "tpp_threshold_500_total_metric": 0.38324999809265137, + "tpp_threshold_500_intended_diff_only": 0.41600000858306885, + "tpp_threshold_500_unintended_diff_only": 0.03275001049041748 }, "6": { - "tpp_threshold_2_total_metric": 0.03124995529651642, - "tpp_threshold_2_intended_diff_only": 0.030999958515167236, - "tpp_threshold_2_unintended_diff_only": -0.00024999678134918213, - "tpp_threshold_5_total_metric": 0.04424995183944702, - "tpp_threshold_5_intended_diff_only": 0.04899996519088745, - "tpp_threshold_5_unintended_diff_only": 0.00475001335144043, - "tpp_threshold_10_total_metric": 0.058750003576278687, - "tpp_threshold_10_intended_diff_only": 0.06000000238418579, - "tpp_threshold_10_unintended_diff_only": 0.0012499988079071045 + "tpp_threshold_2_total_metric": 0.0195000022649765, + "tpp_threshold_2_intended_diff_only": 0.023000001907348633, + "tpp_threshold_2_unintended_diff_only": 0.0034999996423721313, + "tpp_threshold_5_total_metric": 0.04674994945526123, + "tpp_threshold_5_intended_diff_only": 0.04499995708465576, + "tpp_threshold_5_unintended_diff_only": -0.0017499923706054688, + "tpp_threshold_10_total_metric": 0.07199998199939728, + "tpp_threshold_10_intended_diff_only": 0.07499998807907104, + "tpp_threshold_10_unintended_diff_only": 0.003000006079673767, + "tpp_threshold_20_total_metric": 0.10750000178813934, + "tpp_threshold_20_intended_diff_only": 0.11599999666213989, + "tpp_threshold_20_unintended_diff_only": 0.00849999487400055, + "tpp_threshold_50_total_metric": 0.19550001621246338, + "tpp_threshold_50_intended_diff_only": 0.20200002193450928, + "tpp_threshold_50_unintended_diff_only": 0.0065000057220458984, + "tpp_threshold_100_total_metric": 0.2777499854564667, + "tpp_threshold_100_intended_diff_only": 0.2889999747276306, + "tpp_threshold_100_unintended_diff_only": 0.01124998927116394, + "tpp_threshold_500_total_metric": 0.34450002014636993, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.02850000560283661 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-183105-tokens-750000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-183105-tokens-750000000/scr/saebench_scr_custom_sae_eval_results.json index 18257ecaed933373ed8c6e5d73c02bddd08beb2a..267da6ab475302bbc69429aece5689cc5701ea5d 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-183105-tokens-750000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-183105-tokens-750000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "1b2531f8-5985-405b-b248-aadde8f689d2", - "datetime_epoch_millis": 1776884434552, + "eval_id": "dea99386-a55b-43a6-b020-692d06d95f2b", + "datetime_epoch_millis": 1777316345064, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.29073645881291527, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.24672639960801526, "scr_metric_threshold_10": 0.21722385233565517, "scr_dir2_threshold_10": 0.20410166840481794, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.12416420833329343, + "scr_metric_threshold_20": 0.2561844355577999, + "scr_dir2_threshold_20": 0.23883465463813552, + "scr_dir1_threshold_50": 0.10869257614240875, + "scr_metric_threshold_50": 0.30557380609423856, + "scr_dir2_threshold_50": 0.29326717124636625, + "scr_dir1_threshold_100": 0.0319769995665256, + "scr_metric_threshold_100": 0.3304550201245743, + "scr_dir2_threshold_100": 0.3197189525437431, + "scr_dir1_threshold_500": -0.8822934937256783, + "scr_metric_threshold_500": 0.1986650188748059, + "scr_dir2_threshold_500": 0.17055020921041797 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.25641074625770754, "scr_metric_threshold_10": 0.05440405706850797, "scr_dir2_threshold_10": 0.05440405706850797, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.16666679402700396, + "scr_metric_threshold_20": 0.1062176285815671, + "scr_dir2_threshold_20": 0.1062176285815671, + "scr_dir1_threshold_50": 0.5897435701496917, + "scr_metric_threshold_50": 0.22797925140897027, + "scr_dir2_threshold_50": 0.22797925140897027, + "scr_dir1_threshold_100": 0.5000003820810118, + "scr_metric_threshold_100": 0.33937815993376164, + "scr_dir2_threshold_100": 0.33937815993376164, + "scr_dir1_threshold_500": -2.217948179116522, + "scr_metric_threshold_500": 0.020725428605223653, + "scr_dir2_threshold_500": 0.020725428605223653 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.26515165540575625, "scr_metric_threshold_10": 0.2966360744784594, "scr_dir2_threshold_10": 0.2966360744784594, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.31060605718522544, + "scr_metric_threshold_20": 0.22018355312924365, + "scr_dir2_threshold_20": 0.22018355312924365, + "scr_dir1_threshold_50": 0.46969714073872715, + "scr_metric_threshold_50": 0.38532103569898746, + "scr_dir2_threshold_50": 0.38532103569898746, + "scr_dir1_threshold_100": 0.37878788562954907, + "scr_metric_threshold_100": 0.6391436617664482, + "scr_dir2_threshold_100": 0.6391436617664482, + "scr_dir1_threshold_500": -1.446969262523633, + "scr_metric_threshold_500": 0.11620790536168352, + "scr_dir2_threshold_500": 0.11620790536168352 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.4843752037267798, "scr_metric_threshold_10": 0.09547734102822189, "scr_dir2_threshold_10": 0.09547734102822189, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.1093754365573853, + "scr_metric_threshold_20": 0.1783920316689313, + "scr_dir2_threshold_20": 0.1783920316689313, + "scr_dir1_threshold_50": -0.921875087311477, + "scr_metric_threshold_50": 0.20854269211974302, + "scr_dir2_threshold_50": 0.20854269211974302, + "scr_dir1_threshold_100": -1.1250002328306055, + "scr_metric_threshold_100": 0.017588010063299225, + "scr_dir2_threshold_100": 0.017588010063299225, + "scr_dir1_threshold_500": -2.9218741559890553, + "scr_metric_threshold_500": 0.0, + "scr_dir2_threshold_500": 0.0 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": -0.1409397933038777, "scr_metric_threshold_10": 0.08917196243161127, "scr_dir2_threshold_10": 0.08917196243161127, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.9060406711723514, + "scr_metric_threshold_20": 0.08917196243161127, + "scr_dir2_threshold_20": 0.08917196243161127, + "scr_dir1_threshold_50": -0.40939633291396726, + "scr_metric_threshold_50": 0.01910841325227608, + "scr_dir2_threshold_50": 0.01910841325227608, + "scr_dir1_threshold_100": -0.6510067785287252, + "scr_metric_threshold_100": -0.09872616905774931, + "scr_dir2_threshold_100": -0.09872616905774931, + "scr_dir1_threshold_500": -1.8389266362776298, + "scr_metric_threshold_500": -0.5350317745896676, + "scr_dir2_threshold_500": -0.5350317745896676 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.08421069803000372, "scr_metric_threshold_10": 0.4722222287923982, "scr_dir2_threshold_10": 0.4722222287923982, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.09473687842942387, + "scr_metric_threshold_20": 0.515872927644938, + "scr_dir2_threshold_20": 0.515872927644938, + "scr_dir1_threshold_50": -0.11578955293692479, + "scr_metric_threshold_50": 0.6428570583834513, + "scr_dir2_threshold_50": 0.6428570583834513, + "scr_dir1_threshold_100": -0.18947375685884774, + "scr_metric_threshold_100": 0.654761901946115, + "scr_dir2_threshold_100": 0.654761901946115, + "scr_dir1_threshold_500": -0.12105264313663487, + "scr_metric_threshold_500": 0.7301586926148672, + "scr_dir2_threshold_500": 0.7301586926148672 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.4153004823742847, "scr_metric_threshold_10": 0.15648856785294307, "scr_dir2_threshold_10": 0.15648856785294307, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.4644808369405549, + "scr_metric_threshold_20": 0.20992367193697561, + "scr_dir2_threshold_20": 0.20992367193697561, + "scr_dir1_threshold_50": 0.4153004823742847, + "scr_metric_threshold_50": 0.29007644181233594, + "scr_dir2_threshold_50": 0.29007644181233594, + "scr_dir1_threshold_100": 0.4262296310048433, + "scr_metric_threshold_100": 0.36641223978991866, + "scr_dir2_threshold_100": 0.36641223978991866, + "scr_dir1_threshold_500": 0.5628415460730956, + "scr_metric_threshold_500": 0.38549618928431434, + "scr_dir2_threshold_500": 0.38549618928431434 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.4093022598196558, "scr_metric_threshold_10": 0.3733906424792874, "scr_dir2_threshold_10": 0.3733906424792874, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.4744185221266269, + "scr_metric_threshold_20": 0.4506438964966536, + "scr_dir2_threshold_20": 0.4506438964966536, + "scr_dir1_threshold_50": 0.548837266037944, + "scr_metric_threshold_50": 0.3776824330771527, + "scr_dir2_threshold_50": 0.3776824330771527, + "scr_dir1_threshold_100": 0.5860463607627397, + "scr_metric_threshold_100": 0.3948498512825616, + "scr_dir2_threshold_100": 0.3948498512825616, + "scr_dir1_threshold_500": 0.4744185221266269, + "scr_metric_threshold_500": 0.4206008506837011, + "scr_dir2_threshold_500": 0.4206008506837011 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.19999994455381204, "scr_metric_threshold_10": 0.19999994455381204, "scr_dir2_threshold_10": 0.0950224731071144, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.27906981257247854, + "scr_metric_threshold_20": 0.27906981257247854, + "scr_dir2_threshold_20": 0.14027156521516332, + "scr_dir1_threshold_50": 0.29302312300099165, + "scr_metric_threshold_50": 0.29302312300099165, + "scr_dir2_threshold_50": 0.19457004421801274, + "scr_dir1_threshold_100": 0.3302325052722394, + "scr_metric_threshold_100": 0.3302325052722394, + "scr_dir2_threshold_100": 0.2443439646255898, + "scr_dir1_threshold_500": 0.4511628590383247, + "scr_metric_threshold_500": 0.4511628590383247, + "scr_dir2_threshold_500": 0.2262443817232214 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-183105-tokens-750000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-183105-tokens-750000000/tpp/saebench_tpp_custom_sae_eval_results.json index de54f4ba64a4f2c44cce7f3d4f73c9fffebf925f..188ebe5cb3f26c23c95b05372b0da037a02aff2a 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-183105-tokens-750000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-183105-tokens-750000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "1a07b655-3fb7-411e-9cba-1797a4142eee", - "datetime_epoch_millis": 1776884595301, + "eval_id": "df5a6daf-328f-4f93-b99e-dae43f128b94", + "datetime_epoch_millis": 1777316627848, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.035474997758865354, - "tpp_threshold_2_intended_diff_only": 0.038699996471405027, - "tpp_threshold_2_unintended_diff_only": 0.003224998712539673, - "tpp_threshold_5_total_metric": 0.08667500615119933, - "tpp_threshold_5_intended_diff_only": 0.10940001010894776, - "tpp_threshold_5_unintended_diff_only": 0.022725003957748412, - "tpp_threshold_10_total_metric": 0.10779999345541001, - "tpp_threshold_10_intended_diff_only": 0.15280000567436217, - "tpp_threshold_10_unintended_diff_only": 0.04500001221895218, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.034175002574920656, + "tpp_threshold_2_intended_diff_only": 0.038300007581710815, + "tpp_threshold_2_unintended_diff_only": 0.0041250050067901615, + "tpp_threshold_5_total_metric": 0.08677499294281006, + "tpp_threshold_5_intended_diff_only": 0.1081000030040741, + "tpp_threshold_5_unintended_diff_only": 0.02132501006126404, + "tpp_threshold_10_total_metric": 0.10925000607967378, + "tpp_threshold_10_intended_diff_only": 0.15460001230239867, + "tpp_threshold_10_unintended_diff_only": 0.04535000622272491, + "tpp_threshold_20_total_metric": 0.13745000660419465, + "tpp_threshold_20_intended_diff_only": 0.2049000084400177, + "tpp_threshold_20_unintended_diff_only": 0.06745000183582306, + "tpp_threshold_50_total_metric": 0.16090001314878463, + "tpp_threshold_50_intended_diff_only": 0.2610000193119049, + "tpp_threshold_50_unintended_diff_only": 0.10010000616312027, + "tpp_threshold_100_total_metric": 0.18385001718997956, + "tpp_threshold_100_intended_diff_only": 0.32000002861022947, + "tpp_threshold_100_unintended_diff_only": 0.13615001142024993, + "tpp_threshold_500_total_metric": 0.2407000184059143, + "tpp_threshold_500_intended_diff_only": 0.4353000283241272, + "tpp_threshold_500_unintended_diff_only": 0.19460000991821289 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.06124999523162842, - "tpp_threshold_2_intended_diff_only": 0.06740000247955322, - "tpp_threshold_2_unintended_diff_only": 0.006150007247924805, - "tpp_threshold_5_total_metric": 0.1564000129699707, - "tpp_threshold_5_intended_diff_only": 0.19980002641677858, - "tpp_threshold_5_unintended_diff_only": 0.04340001344680786, - "tpp_threshold_10_total_metric": 0.19074999988079072, - "tpp_threshold_10_intended_diff_only": 0.2782000184059143, - "tpp_threshold_10_unintended_diff_only": 0.0874500185251236, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.05995000600814819, + "tpp_threshold_2_intended_diff_only": 0.06720001697540283, + "tpp_threshold_2_unintended_diff_only": 0.007250010967254639, + "tpp_threshold_5_total_metric": 0.15739999413490297, + "tpp_threshold_5_intended_diff_only": 0.1998000144958496, + "tpp_threshold_5_unintended_diff_only": 0.04240002036094666, + "tpp_threshold_10_total_metric": 0.1895000159740448, + "tpp_threshold_10_intended_diff_only": 0.27820003032684326, + "tpp_threshold_10_unintended_diff_only": 0.08870001435279846, + "tpp_threshold_20_total_metric": 0.2262500047683716, + "tpp_threshold_20_intended_diff_only": 0.35780001878738404, + "tpp_threshold_20_unintended_diff_only": 0.13155001401901245, + "tpp_threshold_50_total_metric": 0.21665000915527344, + "tpp_threshold_50_intended_diff_only": 0.41160002946853635, + "tpp_threshold_50_unintended_diff_only": 0.19495002031326295, + "tpp_threshold_100_total_metric": 0.18435002267360687, + "tpp_threshold_100_intended_diff_only": 0.44460004568099976, + "tpp_threshold_100_unintended_diff_only": 0.26025002300739286, + "tpp_threshold_500_total_metric": 0.10345001816749573, + "tpp_threshold_500_intended_diff_only": 0.4670000433921814, + "tpp_threshold_500_unintended_diff_only": 0.36355002522468566 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.009700000286102295, - "tpp_threshold_2_intended_diff_only": 0.009999990463256836, - "tpp_threshold_2_unintended_diff_only": 0.000299990177154541, - "tpp_threshold_5_total_metric": 0.016949999332427978, - "tpp_threshold_5_intended_diff_only": 0.018999993801116943, - "tpp_threshold_5_unintended_diff_only": 0.0020499944686889647, - "tpp_threshold_10_total_metric": 0.024849987030029295, - "tpp_threshold_10_intended_diff_only": 0.027399992942810057, - "tpp_threshold_10_unintended_diff_only": 0.0025500059127807617, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.008399999141693116, + "tpp_threshold_2_intended_diff_only": 0.009399998188018798, + "tpp_threshold_2_unintended_diff_only": 0.0009999990463256836, + "tpp_threshold_5_total_metric": 0.016149991750717164, + "tpp_threshold_5_intended_diff_only": 0.016399991512298585, + "tpp_threshold_5_unintended_diff_only": 0.0002499997615814209, + "tpp_threshold_10_total_metric": 0.028999996185302735, + "tpp_threshold_10_intended_diff_only": 0.0309999942779541, + "tpp_threshold_10_unintended_diff_only": 0.0019999980926513673, + "tpp_threshold_20_total_metric": 0.0486500084400177, + "tpp_threshold_20_intended_diff_only": 0.051999998092651364, + "tpp_threshold_20_unintended_diff_only": 0.003349989652633667, + "tpp_threshold_50_total_metric": 0.10515001714229584, + "tpp_threshold_50_intended_diff_only": 0.11040000915527344, + "tpp_threshold_50_unintended_diff_only": 0.0052499920129776, + "tpp_threshold_100_total_metric": 0.18335001170635223, + "tpp_threshold_100_intended_diff_only": 0.19540001153945924, + "tpp_threshold_100_unintended_diff_only": 0.012049999833106995, + "tpp_threshold_500_total_metric": 0.37795001864433286, + "tpp_threshold_500_intended_diff_only": 0.403600013256073, + "tpp_threshold_500_unintended_diff_only": 0.025649994611740112 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.01150001585483551, "tpp_threshold_10_total_metric": 0.11299999058246613, "tpp_threshold_10_intended_diff_only": 0.125, - "tpp_threshold_10_unintended_diff_only": 0.012000009417533875 + "tpp_threshold_10_unintended_diff_only": 0.012000009417533875, + "tpp_threshold_20_total_metric": 0.23149995505809784, + "tpp_threshold_20_intended_diff_only": 0.3399999737739563, + "tpp_threshold_20_unintended_diff_only": 0.10850001871585846, + "tpp_threshold_50_total_metric": 0.22649997472763062, + "tpp_threshold_50_intended_diff_only": 0.40299999713897705, + "tpp_threshold_50_unintended_diff_only": 0.17650002241134644, + "tpp_threshold_100_total_metric": 0.20749999582767487, + "tpp_threshold_100_intended_diff_only": 0.43800002336502075, + "tpp_threshold_100_unintended_diff_only": 0.2305000275373459, + "tpp_threshold_500_total_metric": 0.11124999821186066, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.3447500318288803 }, "1": { - "tpp_threshold_2_total_metric": 0.08724997937679291, - "tpp_threshold_2_intended_diff_only": 0.10199999809265137, - "tpp_threshold_2_unintended_diff_only": 0.01475001871585846, - "tpp_threshold_5_total_metric": 0.19999998807907104, + "tpp_threshold_2_total_metric": 0.09000000357627869, + "tpp_threshold_2_intended_diff_only": 0.10000002384185791, + "tpp_threshold_2_unintended_diff_only": 0.010000020265579224, + "tpp_threshold_5_total_metric": 0.20249998569488525, "tpp_threshold_5_intended_diff_only": 0.2680000066757202, - "tpp_threshold_5_unintended_diff_only": 0.06800001859664917, - "tpp_threshold_10_total_metric": 0.22450001537799835, + "tpp_threshold_5_unintended_diff_only": 0.06550002098083496, + "tpp_threshold_10_total_metric": 0.2185000330209732, "tpp_threshold_10_intended_diff_only": 0.33000004291534424, - "tpp_threshold_10_unintended_diff_only": 0.10550002753734589 + "tpp_threshold_10_unintended_diff_only": 0.11150000989437103, + "tpp_threshold_20_total_metric": 0.236999973654747, + "tpp_threshold_20_intended_diff_only": 0.3569999933242798, + "tpp_threshold_20_unintended_diff_only": 0.12000001966953278, + "tpp_threshold_50_total_metric": 0.23200002312660217, + "tpp_threshold_50_intended_diff_only": 0.3940000534057617, + "tpp_threshold_50_unintended_diff_only": 0.16200003027915955, + "tpp_threshold_100_total_metric": 0.19675002992153168, + "tpp_threshold_100_intended_diff_only": 0.437000036239624, + "tpp_threshold_100_unintended_diff_only": 0.24025000631809235, + "tpp_threshold_500_total_metric": 0.09050002694129944, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.3685000240802765 }, "2": { - "tpp_threshold_2_total_metric": 0.12700000405311584, - "tpp_threshold_2_intended_diff_only": 0.1380000114440918, - "tpp_threshold_2_unintended_diff_only": 0.011000007390975952, - "tpp_threshold_5_total_metric": 0.2275000363588333, - "tpp_threshold_5_intended_diff_only": 0.29200005531311035, - "tpp_threshold_5_unintended_diff_only": 0.06450001895427704, - "tpp_threshold_10_total_metric": 0.17399999499320984, - "tpp_threshold_10_intended_diff_only": 0.3370000123977661, - "tpp_threshold_10_unintended_diff_only": 0.16300001740455627 + "tpp_threshold_2_total_metric": 0.11675003170967102, + "tpp_threshold_2_intended_diff_only": 0.13600003719329834, + "tpp_threshold_2_unintended_diff_only": 0.01925000548362732, + "tpp_threshold_5_total_metric": 0.2292499840259552, + "tpp_threshold_5_intended_diff_only": 0.29100000858306885, + "tpp_threshold_5_unintended_diff_only": 0.06175002455711365, + "tpp_threshold_10_total_metric": 0.1717500388622284, + "tpp_threshold_10_intended_diff_only": 0.3380000591278076, + "tpp_threshold_10_unintended_diff_only": 0.16625002026557922, + "tpp_threshold_20_total_metric": 0.1365000307559967, + "tpp_threshold_20_intended_diff_only": 0.36600005626678467, + "tpp_threshold_20_unintended_diff_only": 0.22950002551078796, + "tpp_threshold_50_total_metric": 0.12025004625320435, + "tpp_threshold_50_intended_diff_only": 0.3840000629425049, + "tpp_threshold_50_unintended_diff_only": 0.26375001668930054, + "tpp_threshold_100_total_metric": 0.08700002729892731, + "tpp_threshold_100_intended_diff_only": 0.41700005531311035, + "tpp_threshold_100_unintended_diff_only": 0.33000002801418304, + "tpp_threshold_500_total_metric": 0.05349999666213989, + "tpp_threshold_500_intended_diff_only": 0.4520000219345093, + "tpp_threshold_500_unintended_diff_only": 0.3985000252723694 }, "6": { - "tpp_threshold_2_total_metric": 0.012000009417533875, + "tpp_threshold_2_total_metric": 0.009749993681907654, "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": -0.003000006079673767, - "tpp_threshold_5_total_metric": 0.23975002765655518, - "tpp_threshold_5_intended_diff_only": 0.3030000329017639, - "tpp_threshold_5_unintended_diff_only": 0.06325000524520874, - "tpp_threshold_10_total_metric": 0.2722500115633011, + "tpp_threshold_2_unintended_diff_only": -0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.24324999749660492, + "tpp_threshold_5_intended_diff_only": 0.30400002002716064, + "tpp_threshold_5_unintended_diff_only": 0.060750022530555725, + "tpp_threshold_10_total_metric": 0.27525001764297485, "tpp_threshold_10_intended_diff_only": 0.3680000305175781, - "tpp_threshold_10_unintended_diff_only": 0.09575001895427704 + "tpp_threshold_10_unintended_diff_only": 0.09275001287460327, + "tpp_threshold_20_total_metric": 0.30025002360343933, + "tpp_threshold_20_intended_diff_only": 0.4150000214576721, + "tpp_threshold_20_unintended_diff_only": 0.11474999785423279, + "tpp_threshold_50_total_metric": 0.26474998891353607, + "tpp_threshold_50_intended_diff_only": 0.47600001096725464, + "tpp_threshold_50_unintended_diff_only": 0.21125002205371857, + "tpp_threshold_100_total_metric": 0.21375001966953278, + "tpp_threshold_100_intended_diff_only": 0.48200005292892456, + "tpp_threshold_100_unintended_diff_only": 0.2682500332593918, + "tpp_threshold_500_total_metric": 0.1302500218153, + "tpp_threshold_500_intended_diff_only": 0.4920000433921814, + "tpp_threshold_500_unintended_diff_only": 0.3617500215768814 }, "9": { - "tpp_threshold_2_total_metric": 0.003000006079673767, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_5_total_metric": 0.019250035285949707, + "tpp_threshold_2_total_metric": 0.00625002384185791, + "tpp_threshold_2_intended_diff_only": 0.00700002908706665, + "tpp_threshold_2_unintended_diff_only": 0.0007500052452087402, + "tpp_threshold_5_total_metric": 0.016500025987625122, "tpp_threshold_5_intended_diff_only": 0.029000043869018555, - "tpp_threshold_5_unintended_diff_only": 0.009750008583068848, - "tpp_threshold_10_total_metric": 0.16999998688697815, - "tpp_threshold_10_intended_diff_only": 0.23100000619888306, - "tpp_threshold_10_unintended_diff_only": 0.06100001931190491 + "tpp_threshold_5_unintended_diff_only": 0.012500017881393433, + "tpp_threshold_10_total_metric": 0.16899999976158142, + "tpp_threshold_10_intended_diff_only": 0.23000001907348633, + "tpp_threshold_10_unintended_diff_only": 0.06100001931190491, + "tpp_threshold_20_total_metric": 0.22600004076957703, + "tpp_threshold_20_intended_diff_only": 0.3110000491142273, + "tpp_threshold_20_unintended_diff_only": 0.08500000834465027, + "tpp_threshold_50_total_metric": 0.23975001275539398, + "tpp_threshold_50_intended_diff_only": 0.4010000228881836, + "tpp_threshold_50_unintended_diff_only": 0.1612500101327896, + "tpp_threshold_100_total_metric": 0.21675004065036774, + "tpp_threshold_100_intended_diff_only": 0.4490000605583191, + "tpp_threshold_100_unintended_diff_only": 0.23225001990795135, + "tpp_threshold_500_total_metric": 0.13175004720687866, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.34425002336502075 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.008750006556510925, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": -0.004749998450279236, - "tpp_threshold_5_total_metric": 0.01199999451637268, - "tpp_threshold_5_intended_diff_only": 0.009999990463256836, - "tpp_threshold_5_unintended_diff_only": -0.0020000040531158447, - "tpp_threshold_10_total_metric": -0.0014999955892562866, - "tpp_threshold_10_intended_diff_only": 0.003000020980834961, - "tpp_threshold_10_unintended_diff_only": 0.0045000165700912476 + "tpp_threshold_2_total_metric": 0.0014999955892562866, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 0.0034999996423721313, + "tpp_threshold_5_total_metric": 0.0037499964237213135, + "tpp_threshold_5_intended_diff_only": 0.004999995231628418, + "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_10_total_metric": 0.0032500028610229492, + "tpp_threshold_10_intended_diff_only": 0.004999995231628418, + "tpp_threshold_10_unintended_diff_only": 0.0017499923706054688, + "tpp_threshold_20_total_metric": 0.007500007748603821, + "tpp_threshold_20_intended_diff_only": 0.009999990463256836, + "tpp_threshold_20_unintended_diff_only": 0.002499982714653015, + "tpp_threshold_50_total_metric": 0.046750009059906006, + "tpp_threshold_50_intended_diff_only": 0.046999990940093994, + "tpp_threshold_50_unintended_diff_only": 0.0002499818801879883, + "tpp_threshold_100_total_metric": 0.10199998319149017, + "tpp_threshold_100_intended_diff_only": 0.10799998044967651, + "tpp_threshold_100_unintended_diff_only": 0.00599999725818634, + "tpp_threshold_500_total_metric": 0.38975000381469727, + "tpp_threshold_500_intended_diff_only": 0.40799999237060547, + "tpp_threshold_500_unintended_diff_only": 0.018249988555908203 }, "2": { - "tpp_threshold_2_total_metric": 0.003000020980834961, - "tpp_threshold_2_intended_diff_only": 0.008000016212463379, - "tpp_threshold_2_unintended_diff_only": 0.004999995231628418, - "tpp_threshold_5_total_metric": 0.01325005292892456, - "tpp_threshold_5_intended_diff_only": 0.020000040531158447, - "tpp_threshold_5_unintended_diff_only": 0.006749987602233887, - "tpp_threshold_10_total_metric": 0.04275001585483551, - "tpp_threshold_10_intended_diff_only": 0.046000003814697266, - "tpp_threshold_10_unintended_diff_only": 0.0032499879598617554 + "tpp_threshold_2_total_metric": 0.013250038027763367, + "tpp_threshold_2_intended_diff_only": 0.01100003719329834, + "tpp_threshold_2_unintended_diff_only": -0.002250000834465027, + "tpp_threshold_5_total_metric": 0.009750038385391235, + "tpp_threshold_5_intended_diff_only": 0.016000032424926758, + "tpp_threshold_5_unintended_diff_only": 0.0062499940395355225, + "tpp_threshold_10_total_metric": 0.04600003361701965, + "tpp_threshold_10_intended_diff_only": 0.04900002479553223, + "tpp_threshold_10_unintended_diff_only": 0.0029999911785125732, + "tpp_threshold_20_total_metric": 0.07525007426738739, + "tpp_threshold_20_intended_diff_only": 0.07900005578994751, + "tpp_threshold_20_unintended_diff_only": 0.0037499815225601196, + "tpp_threshold_50_total_metric": 0.13650007545948029, + "tpp_threshold_50_intended_diff_only": 0.1480000615119934, + "tpp_threshold_50_unintended_diff_only": 0.011499986052513123, + "tpp_threshold_100_total_metric": 0.22400006651878357, + "tpp_threshold_100_intended_diff_only": 0.24100005626678467, + "tpp_threshold_100_unintended_diff_only": 0.0169999897480011, + "tpp_threshold_500_total_metric": 0.4162500500679016, + "tpp_threshold_500_intended_diff_only": 0.4360000491142273, + "tpp_threshold_500_unintended_diff_only": 0.019749999046325684 }, "3": { - "tpp_threshold_2_total_metric": 0.012250006198883057, - "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_2_unintended_diff_only": -0.00625002384185791, - "tpp_threshold_5_total_metric": 0.017749980092048645, - "tpp_threshold_5_intended_diff_only": 0.015999972820281982, - "tpp_threshold_5_unintended_diff_only": -0.0017500072717666626, - "tpp_threshold_10_total_metric": 0.01724998652935028, - "tpp_threshold_10_intended_diff_only": 0.018999993801116943, - "tpp_threshold_10_unintended_diff_only": 0.0017500072717666626 + "tpp_threshold_2_total_metric": 0.003750026226043701, + "tpp_threshold_2_intended_diff_only": 0.003000020980834961, + "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402, + "tpp_threshold_5_total_metric": 0.012750014662742615, + "tpp_threshold_5_intended_diff_only": 0.008000016212463379, + "tpp_threshold_5_unintended_diff_only": -0.004749998450279236, + "tpp_threshold_10_total_metric": 0.019250020384788513, + "tpp_threshold_10_intended_diff_only": 0.017000019550323486, + "tpp_threshold_10_unintended_diff_only": -0.002250000834465027, + "tpp_threshold_20_total_metric": 0.012499988079071045, + "tpp_threshold_20_intended_diff_only": 0.014999985694885254, + "tpp_threshold_20_unintended_diff_only": 0.002499997615814209, + "tpp_threshold_50_total_metric": 0.05125001072883606, + "tpp_threshold_50_intended_diff_only": 0.050000011920928955, + "tpp_threshold_50_unintended_diff_only": -0.0012499988079071045, + "tpp_threshold_100_total_metric": 0.10474999248981476, + "tpp_threshold_100_intended_diff_only": 0.11599999666213989, + "tpp_threshold_100_unintended_diff_only": 0.011250004172325134, + "tpp_threshold_500_total_metric": 0.35875003039836884, + "tpp_threshold_500_intended_diff_only": 0.38700002431869507, + "tpp_threshold_500_unintended_diff_only": 0.028249993920326233 }, "5": { - "tpp_threshold_2_total_metric": -0.007249996066093445, - "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 0.008249983191490173, - "tpp_threshold_5_total_metric": 0.0037499964237213135, - "tpp_threshold_5_intended_diff_only": 0.004999995231628418, - "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045, - "tpp_threshold_10_total_metric": 0.006499946117401123, + "tpp_threshold_2_total_metric": 0.0014999806880950928, + "tpp_threshold_2_intended_diff_only": 0.001999974250793457, + "tpp_threshold_2_unintended_diff_only": 0.0004999935626983643, + "tpp_threshold_5_total_metric": 0.013499945402145386, + "tpp_threshold_5_intended_diff_only": 0.012999951839447021, + "tpp_threshold_5_unintended_diff_only": -0.0004999935626983643, + "tpp_threshold_10_total_metric": 0.003999948501586914, "tpp_threshold_10_intended_diff_only": 0.007999956607818604, - "tpp_threshold_10_unintended_diff_only": 0.0015000104904174805 + "tpp_threshold_10_unintended_diff_only": 0.0040000081062316895, + "tpp_threshold_20_total_metric": 0.04649999737739563, + "tpp_threshold_20_intended_diff_only": 0.046999990940093994, + "tpp_threshold_20_unintended_diff_only": 0.0004999935626983643, + "tpp_threshold_50_total_metric": 0.0949999988079071, + "tpp_threshold_50_intended_diff_only": 0.10199999809265137, + "tpp_threshold_50_unintended_diff_only": 0.006999999284744263, + "tpp_threshold_100_total_metric": 0.2212499976158142, + "tpp_threshold_100_intended_diff_only": 0.23600000143051147, + "tpp_threshold_100_unintended_diff_only": 0.014750003814697266, + "tpp_threshold_500_total_metric": 0.3804999738931656, + "tpp_threshold_500_intended_diff_only": 0.4139999747276306, + "tpp_threshold_500_unintended_diff_only": 0.03350000083446503 }, "6": { - "tpp_threshold_2_total_metric": 0.03174996376037598, - "tpp_threshold_2_intended_diff_only": 0.030999958515167236, - "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402, - "tpp_threshold_5_total_metric": 0.03799997270107269, - "tpp_threshold_5_intended_diff_only": 0.04399996995925903, - "tpp_threshold_5_unintended_diff_only": 0.00599999725818634, - "tpp_threshold_10_total_metric": 0.05924998223781586, - "tpp_threshold_10_intended_diff_only": 0.06099998950958252, - "tpp_threshold_10_unintended_diff_only": 0.0017500072717666626 + "tpp_threshold_2_total_metric": 0.02199995517730713, + "tpp_threshold_2_intended_diff_only": 0.02599996328353882, + "tpp_threshold_2_unintended_diff_only": 0.0040000081062316895, + "tpp_threshold_5_total_metric": 0.040999963879585266, + "tpp_threshold_5_intended_diff_only": 0.039999961853027344, + "tpp_threshold_5_unintended_diff_only": -0.0010000020265579224, + "tpp_threshold_10_total_metric": 0.07249997556209564, + "tpp_threshold_10_intended_diff_only": 0.07599997520446777, + "tpp_threshold_10_unintended_diff_only": 0.0034999996423721313, + "tpp_threshold_20_total_metric": 0.10149997472763062, + "tpp_threshold_20_intended_diff_only": 0.10899996757507324, + "tpp_threshold_20_unintended_diff_only": 0.007499992847442627, + "tpp_threshold_50_total_metric": 0.19624999165534973, + "tpp_threshold_50_intended_diff_only": 0.20499998331069946, + "tpp_threshold_50_unintended_diff_only": 0.008749991655349731, + "tpp_threshold_100_total_metric": 0.26475001871585846, + "tpp_threshold_100_intended_diff_only": 0.2760000228881836, + "tpp_threshold_100_unintended_diff_only": 0.011250004172325134, + "tpp_threshold_500_total_metric": 0.34450003504753113, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.028499990701675415 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-195312-tokens-800000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-195312-tokens-800000000/scr/saebench_scr_custom_sae_eval_results.json index 0f099a714f8c031aec9870b91b53d3b7fe8418fc..bc0d8343d151a6e4fa8ccd7ed28db048fda66378 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-195312-tokens-800000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-195312-tokens-800000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "c9b01a50-123e-4537-87b5-41a091d4564a", - "datetime_epoch_millis": 1776884615448, + "eval_id": "6c5700ba-083a-4d96-a7b9-a05419f3d808", + "datetime_epoch_millis": 1777316489620, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.2947372238881211, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.35524561939979316, "scr_metric_threshold_10": 0.21637876785214583, "scr_dir2_threshold_10": 0.2019990997881127, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.31020072750215694, + "scr_metric_threshold_20": 0.23214022507635007, + "scr_dir2_threshold_20": 0.21760271115984237, + "scr_dir1_threshold_50": 0.32320885874957717, + "scr_metric_threshold_50": 0.30508886941015756, + "scr_dir2_threshold_50": 0.28692093177725986, + "scr_dir1_threshold_100": -0.12096203532666253, + "scr_metric_threshold_100": 0.30685690316563835, + "scr_dir2_threshold_100": 0.28652122499326227, + "scr_dir1_threshold_500": -0.8702652497939531, + "scr_metric_threshold_500": 0.1961407921858782, + "scr_dir2_threshold_500": 0.17429507670451402 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.5769232238773122, "scr_metric_threshold_10": 0.036269422851059746, "scr_dir2_threshold_10": 0.036269422851059746, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.5128207283533913, + "scr_metric_threshold_20": 0.10362698860995496, + "scr_dir2_threshold_20": 0.10362698860995496, + "scr_dir1_threshold_50": 0.5256410746257707, + "scr_metric_threshold_50": 0.15803104567846293, + "scr_dir2_threshold_50": 0.15803104567846293, + "scr_dir1_threshold_100": 0.30769289550924905, + "scr_metric_threshold_100": 0.2927460227800901, + "scr_dir2_threshold_100": 0.2927460227800901, + "scr_dir1_threshold_500": -2.8717942447901343, + "scr_metric_threshold_500": 0.05958549142789553, + "scr_dir2_threshold_500": 0.05958549142789553 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.4469697140738727, "scr_metric_threshold_10": 0.1743120403197142, "scr_dir2_threshold_10": 0.1743120403197142, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.43181839733079624, + "scr_metric_threshold_20": 0.11009177656462182, + "scr_dir2_threshold_20": 0.11009177656462182, + "scr_dir1_threshold_50": 0.4090909706659418, + "scr_metric_threshold_50": 0.32721708301814567, + "scr_dir2_threshold_50": 0.32721708301814567, + "scr_dir1_threshold_100": 0.1742424002965782, + "scr_metric_threshold_100": 0.4678898681224538, + "scr_dir2_threshold_100": 0.4678898681224538, + "scr_dir1_threshold_500": -0.6818178328929966, + "scr_metric_threshold_500": 0.14984716057708958, + "scr_dir2_threshold_500": 0.14984716057708958 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.4062502910382569, "scr_metric_threshold_10": 0.13316589123232284, "scr_dir2_threshold_10": 0.13316589123232284, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.06250011641530274, + "scr_metric_threshold_20": 0.1507537515352312, + "scr_dir2_threshold_20": 0.1507537515352312, + "scr_dir1_threshold_50": 0.31249965075409175, + "scr_metric_threshold_50": 0.19346736189433716, + "scr_dir2_threshold_50": 0.19346736189433716, + "scr_dir1_threshold_100": -1.9374998835846973, + "scr_metric_threshold_100": 0.03266334028870509, + "scr_dir2_threshold_100": 0.03266334028870509, + "scr_dir1_threshold_500": -3.1249993015081836, + "scr_metric_threshold_500": -0.05527641050700931, + "scr_dir2_threshold_500": -0.05527641050700931 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": 0.30201315702624804, "scr_metric_threshold_10": 0.124203926844982, "scr_dir2_threshold_10": 0.124203926844982, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.43624162684689405, + "scr_metric_threshold_20": 0.006369534358659739, + "scr_dir2_threshold_20": 0.006369534358659739, + "scr_dir1_threshold_50": 0.25503349261242375, + "scr_metric_threshold_50": 0.11464972021884395, + "scr_dir2_threshold_50": 0.11464972021884395, + "scr_dir1_threshold_100": -0.5436244027034108, + "scr_metric_threshold_100": -0.07006373900303832, + "scr_dir2_threshold_100": -0.07006373900303832, + "scr_dir1_threshold_500": -1.583893143665206, + "scr_metric_threshold_500": -0.5254777577872327, + "scr_dir2_threshold_500": -0.5254777577872327 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.09473687842942387, "scr_metric_threshold_10": 0.5198413665167169, "scr_dir2_threshold_10": 0.5198413665167169, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.13684191373576507, + "scr_metric_threshold_20": 0.5873015159682478, + "scr_dir2_threshold_20": 0.5873015159682478, + "scr_dir1_threshold_50": -0.22631570196547887, + "scr_metric_threshold_50": 0.6904761961077699, + "scr_dir2_threshold_50": 0.6904761961077699, + "scr_dir1_threshold_100": -0.1947368470585578, + "scr_metric_threshold_100": 0.742063536177531, + "scr_dir2_threshold_100": 0.742063536177531, + "scr_dir1_threshold_500": -0.16315799215163673, + "scr_metric_threshold_500": 0.801587280938177, + "scr_dir2_threshold_500": 0.801587280938177 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.3825136878996036, "scr_metric_threshold_10": 0.14122136275770192, "scr_dir2_threshold_10": 0.14122136275770192, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.4262296310048433, + "scr_metric_threshold_20": 0.19465646684173443, + "scr_dir2_threshold_20": 0.19465646684173443, + "scr_dir1_threshold_50": 0.3934425108216648, + "scr_metric_threshold_50": 0.29007644181233594, + "scr_dir2_threshold_50": 0.29007644181233594, + "scr_dir1_threshold_100": 0.4262296310048433, + "scr_metric_threshold_100": 0.35496177909383203, + "scr_dir2_threshold_100": 0.35496177909383203, + "scr_dir1_threshold_500": 0.5519127231510342, + "scr_metric_threshold_500": 0.36259549539076413, + "scr_dir2_threshold_500": 0.36259549539076413 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.3953486760285681, "scr_metric_threshold_10": 0.3648068054696092, "scr_dir2_threshold_10": 0.3648068054696092, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.46511631775314366, + "scr_metric_threshold_20": 0.4206008506837011, + "scr_dir2_threshold_20": 0.4206008506837011, + "scr_dir1_threshold_50": 0.5627905725981688, + "scr_metric_threshold_50": 0.31330480666733007, + "scr_dir2_threshold_50": 0.31330480666733007, + "scr_dir1_threshold_100": 0.4744185221266269, + "scr_metric_threshold_100": 0.30901301606946474, + "scr_dir2_threshold_100": 0.30901301606946474, + "scr_dir1_threshold_500": 0.4744185221266269, + "scr_metric_threshold_500": 0.33905580606846963, + "scr_dir2_threshold_500": 0.33905580606846963 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.2372093268250598, "scr_metric_threshold_10": 0.2372093268250598, "scr_dir2_threshold_10": 0.1221719823127949, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2837209160486496, + "scr_metric_threshold_20": 0.2837209160486496, + "scr_dir2_threshold_20": 0.16742080471658802, + "scr_dir1_threshold_50": 0.3534882998840343, + "scr_metric_threshold_50": 0.3534882998840343, + "scr_dir2_threshold_50": 0.208144798820853, + "scr_dir1_threshold_100": 0.32558140179606837, + "scr_metric_threshold_100": 0.32558140179606837, + "scr_dir2_threshold_100": 0.16289597641705988, + "scr_dir1_threshold_500": 0.43720927137887183, + "scr_metric_threshold_500": 0.43720927137887183, + "scr_dir2_threshold_500": 0.26244354752795823 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-195312-tokens-800000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-195312-tokens-800000000/tpp/saebench_tpp_custom_sae_eval_results.json index 0e85d4df41d670a94279ac3fc153b5c534a5b879..ef9070636c133d68c45978dc7339a49f7a904117 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-195312-tokens-800000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-195312-tokens-800000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "9c47242c-3ddc-42b2-994c-e801ae56458b", - "datetime_epoch_millis": 1776884779578, + "eval_id": "05b75ee8-70ef-4f88-803f-f67922902635", + "datetime_epoch_millis": 1777316762706, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.033774998784065244, - "tpp_threshold_2_intended_diff_only": 0.03669999837875366, - "tpp_threshold_2_unintended_diff_only": 0.002924999594688416, - "tpp_threshold_5_total_metric": 0.09659999907016754, - "tpp_threshold_5_intended_diff_only": 0.12450000643730164, - "tpp_threshold_5_unintended_diff_only": 0.027900007367134095, - "tpp_threshold_10_total_metric": 0.1063500002026558, - "tpp_threshold_10_intended_diff_only": 0.15390000939369203, - "tpp_threshold_10_unintended_diff_only": 0.047550009191036226, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.03249999582767486, + "tpp_threshold_2_intended_diff_only": 0.03650000095367432, + "tpp_threshold_2_unintended_diff_only": 0.0040000051259994505, + "tpp_threshold_5_total_metric": 0.09695000648498535, + "tpp_threshold_5_intended_diff_only": 0.12340001463890075, + "tpp_threshold_5_unintended_diff_only": 0.026450008153915405, + "tpp_threshold_10_total_metric": 0.10630000531673432, + "tpp_threshold_10_intended_diff_only": 0.1552000105381012, + "tpp_threshold_10_unintended_diff_only": 0.04890000522136688, + "tpp_threshold_20_total_metric": 0.1335500106215477, + "tpp_threshold_20_intended_diff_only": 0.20650001764297488, + "tpp_threshold_20_unintended_diff_only": 0.07295000702142715, + "tpp_threshold_50_total_metric": 0.15557500422000886, + "tpp_threshold_50_intended_diff_only": 0.2589000165462494, + "tpp_threshold_50_unintended_diff_only": 0.10332501232624054, + "tpp_threshold_100_total_metric": 0.17610000520944596, + "tpp_threshold_100_intended_diff_only": 0.3164000153541565, + "tpp_threshold_100_unintended_diff_only": 0.14030001014471055, + "tpp_threshold_500_total_metric": 0.23960001170635223, + "tpp_threshold_500_intended_diff_only": 0.43610002398490905, + "tpp_threshold_500_unintended_diff_only": 0.19650001227855682 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.05894999802112579, - "tpp_threshold_2_intended_diff_only": 0.06460000276565551, - "tpp_threshold_2_unintended_diff_only": 0.0056500047445297245, - "tpp_threshold_5_total_metric": 0.17910000085830688, - "tpp_threshold_5_intended_diff_only": 0.23280001878738404, - "tpp_threshold_5_unintended_diff_only": 0.05370001792907715, - "tpp_threshold_10_total_metric": 0.18945000767707826, + "tpp_threshold_2_total_metric": 0.05729999244213104, + "tpp_threshold_2_intended_diff_only": 0.06440000534057617, + "tpp_threshold_2_unintended_diff_only": 0.007100012898445129, + "tpp_threshold_5_total_metric": 0.18060002028942107, + "tpp_threshold_5_intended_diff_only": 0.2330000400543213, + "tpp_threshold_5_unintended_diff_only": 0.052400019764900205, + "tpp_threshold_10_total_metric": 0.18715001344680787, "tpp_threshold_10_intended_diff_only": 0.28240002393722535, - "tpp_threshold_10_unintended_diff_only": 0.0929500162601471, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_10_unintended_diff_only": 0.09525001049041748, + "tpp_threshold_20_total_metric": 0.21870001554489135, + "tpp_threshold_20_intended_diff_only": 0.36140003204345705, + "tpp_threshold_20_unintended_diff_only": 0.14270001649856567, + "tpp_threshold_50_total_metric": 0.20645001232624055, + "tpp_threshold_50_intended_diff_only": 0.40660003423690794, + "tpp_threshold_50_unintended_diff_only": 0.20015002191066741, + "tpp_threshold_100_total_metric": 0.16890000998973848, + "tpp_threshold_100_intended_diff_only": 0.4372000336647034, + "tpp_threshold_100_unintended_diff_only": 0.2683000236749649, + "tpp_threshold_500_total_metric": 0.10315001010894775, + "tpp_threshold_500_intended_diff_only": 0.46720004081726074, + "tpp_threshold_500_unintended_diff_only": 0.364050030708313 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.008599999547004699, - "tpp_threshold_2_intended_diff_only": 0.008799993991851806, - "tpp_threshold_2_unintended_diff_only": 0.00019999444484710693, - "tpp_threshold_5_total_metric": 0.014099997282028199, - "tpp_threshold_5_intended_diff_only": 0.016199994087219238, - "tpp_threshold_5_unintended_diff_only": 0.00209999680519104, - "tpp_threshold_10_total_metric": 0.02324999272823334, - "tpp_threshold_10_intended_diff_only": 0.025399994850158692, - "tpp_threshold_10_unintended_diff_only": 0.002150002121925354, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.007699999213218689, + "tpp_threshold_2_intended_diff_only": 0.008599996566772461, + "tpp_threshold_2_unintended_diff_only": 0.000899997353553772, + "tpp_threshold_5_total_metric": 0.013299992680549622, + "tpp_threshold_5_intended_diff_only": 0.013799989223480224, + "tpp_threshold_5_unintended_diff_only": 0.000499996542930603, + "tpp_threshold_10_total_metric": 0.025449997186660765, + "tpp_threshold_10_intended_diff_only": 0.02799999713897705, + "tpp_threshold_10_unintended_diff_only": 0.0025499999523162843, + "tpp_threshold_20_total_metric": 0.04840000569820404, + "tpp_threshold_20_intended_diff_only": 0.05160000324249268, + "tpp_threshold_20_unintended_diff_only": 0.0031999975442886354, + "tpp_threshold_50_total_metric": 0.10469999611377716, + "tpp_threshold_50_intended_diff_only": 0.11119999885559081, + "tpp_threshold_50_unintended_diff_only": 0.0065000027418136595, + "tpp_threshold_100_total_metric": 0.18330000042915345, + "tpp_threshold_100_intended_diff_only": 0.1955999970436096, + "tpp_threshold_100_unintended_diff_only": 0.012299996614456177, + "tpp_threshold_500_total_metric": 0.3760500133037567, + "tpp_threshold_500_intended_diff_only": 0.40500000715255735, + "tpp_threshold_500_unintended_diff_only": 0.02894999384880066 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.013250008225440979, "tpp_threshold_10_total_metric": 0.11374999582767487, "tpp_threshold_10_intended_diff_only": 0.1290000081062317, - "tpp_threshold_10_unintended_diff_only": 0.015250012278556824 + "tpp_threshold_10_unintended_diff_only": 0.015250012278556824, + "tpp_threshold_20_total_metric": 0.21850000321865082, + "tpp_threshold_20_intended_diff_only": 0.3500000238418579, + "tpp_threshold_20_unintended_diff_only": 0.1315000206232071, + "tpp_threshold_50_total_metric": 0.23600000143051147, + "tpp_threshold_50_intended_diff_only": 0.4020000100135803, + "tpp_threshold_50_unintended_diff_only": 0.16600000858306885, + "tpp_threshold_100_total_metric": 0.1992499977350235, + "tpp_threshold_100_intended_diff_only": 0.4390000104904175, + "tpp_threshold_100_unintended_diff_only": 0.23975001275539398, + "tpp_threshold_500_total_metric": 0.10874998569488525, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.3472500443458557 }, "1": { - "tpp_threshold_2_total_metric": 0.04924999177455902, - "tpp_threshold_2_intended_diff_only": 0.06000000238418579, - "tpp_threshold_2_unintended_diff_only": 0.01075001060962677, - "tpp_threshold_5_total_metric": 0.18025003373622894, + "tpp_threshold_2_total_metric": 0.05149999260902405, + "tpp_threshold_2_intended_diff_only": 0.05900001525878906, + "tpp_threshold_2_unintended_diff_only": 0.007500022649765015, + "tpp_threshold_5_total_metric": 0.1810000091791153, "tpp_threshold_5_intended_diff_only": 0.2420000433921814, - "tpp_threshold_5_unintended_diff_only": 0.061750009655952454, - "tpp_threshold_10_total_metric": 0.19750000536441803, + "tpp_threshold_5_unintended_diff_only": 0.0610000342130661, + "tpp_threshold_10_total_metric": 0.19075001776218414, "tpp_threshold_10_intended_diff_only": 0.2940000295639038, - "tpp_threshold_10_unintended_diff_only": 0.09650002419948578 + "tpp_threshold_10_unintended_diff_only": 0.10325001180171967, + "tpp_threshold_20_total_metric": 0.21824999153614044, + "tpp_threshold_20_intended_diff_only": 0.3370000123977661, + "tpp_threshold_20_unintended_diff_only": 0.11875002086162567, + "tpp_threshold_50_total_metric": 0.1939999908208847, + "tpp_threshold_50_intended_diff_only": 0.36900001764297485, + "tpp_threshold_50_unintended_diff_only": 0.17500002682209015, + "tpp_threshold_100_total_metric": 0.17275001108646393, + "tpp_threshold_100_intended_diff_only": 0.41300004720687866, + "tpp_threshold_100_unintended_diff_only": 0.24025003612041473, + "tpp_threshold_500_total_metric": 0.09325002133846283, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.3657500296831131 }, "2": { - "tpp_threshold_2_total_metric": 0.14550001919269562, - "tpp_threshold_2_intended_diff_only": 0.15500003099441528, - "tpp_threshold_2_unintended_diff_only": 0.009500011801719666, - "tpp_threshold_5_total_metric": 0.2265000343322754, - "tpp_threshold_5_intended_diff_only": 0.29700005054473877, - "tpp_threshold_5_unintended_diff_only": 0.07050001621246338, - "tpp_threshold_10_total_metric": 0.179500013589859, - "tpp_threshold_10_intended_diff_only": 0.3540000319480896, - "tpp_threshold_10_unintended_diff_only": 0.1745000183582306 + "tpp_threshold_2_total_metric": 0.13374999165534973, + "tpp_threshold_2_intended_diff_only": 0.15200001001358032, + "tpp_threshold_2_unintended_diff_only": 0.01825001835823059, + "tpp_threshold_5_total_metric": 0.23100005090236664, + "tpp_threshold_5_intended_diff_only": 0.29600006341934204, + "tpp_threshold_5_unintended_diff_only": 0.0650000125169754, + "tpp_threshold_10_total_metric": 0.17475000023841858, + "tpp_threshold_10_intended_diff_only": 0.35500001907348633, + "tpp_threshold_10_unintended_diff_only": 0.18025001883506775, + "tpp_threshold_20_total_metric": 0.14325003325939178, + "tpp_threshold_20_intended_diff_only": 0.3720000386238098, + "tpp_threshold_20_unintended_diff_only": 0.22875000536441803, + "tpp_threshold_50_total_metric": 0.10474999248981476, + "tpp_threshold_50_intended_diff_only": 0.3920000195503235, + "tpp_threshold_50_unintended_diff_only": 0.28725002706050873, + "tpp_threshold_100_total_metric": 0.09550002217292786, + "tpp_threshold_100_intended_diff_only": 0.42200005054473877, + "tpp_threshold_100_unintended_diff_only": 0.3265000283718109, + "tpp_threshold_500_total_metric": 0.05274999141693115, + "tpp_threshold_500_intended_diff_only": 0.4520000219345093, + "tpp_threshold_500_unintended_diff_only": 0.3992500305175781 }, "6": { - "tpp_threshold_2_total_metric": 0.010750025510787964, + "tpp_threshold_2_total_metric": 0.00849999487400055, "tpp_threshold_2_intended_diff_only": 0.008000016212463379, - "tpp_threshold_2_unintended_diff_only": -0.002750009298324585, - "tpp_threshold_5_total_metric": 0.24124997854232788, - "tpp_threshold_5_intended_diff_only": 0.3050000071525574, - "tpp_threshold_5_unintended_diff_only": 0.06375002861022949, - "tpp_threshold_10_total_metric": 0.2695000171661377, + "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704, + "tpp_threshold_5_total_metric": 0.2460000365972519, + "tpp_threshold_5_intended_diff_only": 0.3060000538825989, + "tpp_threshold_5_unintended_diff_only": 0.060000017285346985, + "tpp_threshold_10_total_metric": 0.2737500071525574, "tpp_threshold_10_intended_diff_only": 0.37800002098083496, - "tpp_threshold_10_unintended_diff_only": 0.10850000381469727 + "tpp_threshold_10_unintended_diff_only": 0.10425001382827759, + "tpp_threshold_20_total_metric": 0.2937500327825546, + "tpp_threshold_20_intended_diff_only": 0.4360000491142273, + "tpp_threshold_20_unintended_diff_only": 0.14225001633167267, + "tpp_threshold_50_total_metric": 0.26350003480911255, + "tpp_threshold_50_intended_diff_only": 0.47700005769729614, + "tpp_threshold_50_unintended_diff_only": 0.2135000228881836, + "tpp_threshold_100_total_metric": 0.19550001621246338, + "tpp_threshold_100_intended_diff_only": 0.4830000400543213, + "tpp_threshold_100_unintended_diff_only": 0.2875000238418579, + "tpp_threshold_500_total_metric": 0.13250000774860382, + "tpp_threshold_500_intended_diff_only": 0.4930000305175781, + "tpp_threshold_500_unintended_diff_only": 0.3605000227689743 }, "9": { - "tpp_threshold_2_total_metric": 0.0037499964237213135, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": 0.0012499988079071045, - "tpp_threshold_5_total_metric": 0.14374998211860657, - "tpp_threshold_5_intended_diff_only": 0.203000009059906, - "tpp_threshold_5_unintended_diff_only": 0.05925002694129944, - "tpp_threshold_10_total_metric": 0.18700000643730164, - "tpp_threshold_10_intended_diff_only": 0.25700002908706665, - "tpp_threshold_10_unintended_diff_only": 0.07000002264976501 + "tpp_threshold_2_total_metric": 0.0072500258684158325, + "tpp_threshold_2_intended_diff_only": 0.008000016212463379, + "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.14125002920627594, + "tpp_threshold_5_intended_diff_only": 0.2040000557899475, + "tpp_threshold_5_unintended_diff_only": 0.06275002658367157, + "tpp_threshold_10_total_metric": 0.18275004625320435, + "tpp_threshold_10_intended_diff_only": 0.2560000419616699, + "tpp_threshold_10_unintended_diff_only": 0.07324999570846558, + "tpp_threshold_20_total_metric": 0.21975001692771912, + "tpp_threshold_20_intended_diff_only": 0.312000036239624, + "tpp_threshold_20_unintended_diff_only": 0.09225001931190491, + "tpp_threshold_50_total_metric": 0.2340000420808792, + "tpp_threshold_50_intended_diff_only": 0.393000066280365, + "tpp_threshold_50_unintended_diff_only": 0.15900002419948578, + "tpp_threshold_100_total_metric": 0.18150000274181366, + "tpp_threshold_100_intended_diff_only": 0.42900002002716064, + "tpp_threshold_100_unintended_diff_only": 0.24750001728534698, + "tpp_threshold_500_total_metric": 0.1285000443458557, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.3475000262260437 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.009000003337860107, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": -0.004999995231628418, - "tpp_threshold_5_total_metric": 0.013249963521957397, - "tpp_threshold_5_intended_diff_only": 0.010999977588653564, - "tpp_threshold_5_unintended_diff_only": -0.002249985933303833, - "tpp_threshold_10_total_metric": 0.0007499903440475464, + "tpp_threshold_2_total_metric": 0.002749994397163391, + "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_2_unintended_diff_only": 0.0032499879598617554, + "tpp_threshold_5_total_metric": 0.007500007748603821, + "tpp_threshold_5_intended_diff_only": 0.009000003337860107, + "tpp_threshold_5_unintended_diff_only": 0.0014999955892562866, + "tpp_threshold_10_total_metric": 0.0032500028610229492, "tpp_threshold_10_intended_diff_only": 0.004999995231628418, - "tpp_threshold_10_unintended_diff_only": 0.004250004887580872 + "tpp_threshold_10_unintended_diff_only": 0.0017499923706054688, + "tpp_threshold_20_total_metric": 0.018499985337257385, + "tpp_threshold_20_intended_diff_only": 0.019999980926513672, + "tpp_threshold_20_unintended_diff_only": 0.0014999955892562866, + "tpp_threshold_50_total_metric": 0.05600002408027649, + "tpp_threshold_50_intended_diff_only": 0.058000028133392334, + "tpp_threshold_50_unintended_diff_only": 0.0020000040531158447, + "tpp_threshold_100_total_metric": 0.1354999840259552, + "tpp_threshold_100_intended_diff_only": 0.1499999761581421, + "tpp_threshold_100_unintended_diff_only": 0.01449999213218689, + "tpp_threshold_500_total_metric": 0.4037500023841858, + "tpp_threshold_500_intended_diff_only": 0.4259999990463257, + "tpp_threshold_500_unintended_diff_only": 0.022249996662139893 }, "2": { - "tpp_threshold_2_total_metric": 0.0037500113248825073, - "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": 0.0052499920129776, - "tpp_threshold_5_total_metric": 0.007750064134597778, - "tpp_threshold_5_intended_diff_only": 0.01500004529953003, - "tpp_threshold_5_unintended_diff_only": 0.007249981164932251, - "tpp_threshold_10_total_metric": 0.0310000479221344, - "tpp_threshold_10_intended_diff_only": 0.033000051975250244, - "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447 + "tpp_threshold_2_total_metric": 0.013250038027763367, + "tpp_threshold_2_intended_diff_only": 0.01100003719329834, + "tpp_threshold_2_unintended_diff_only": -0.002250000834465027, + "tpp_threshold_5_total_metric": 0.004000052809715271, + "tpp_threshold_5_intended_diff_only": 0.01100003719329834, + "tpp_threshold_5_unintended_diff_only": 0.006999984383583069, + "tpp_threshold_10_total_metric": 0.03100001811981201, + "tpp_threshold_10_intended_diff_only": 0.03600001335144043, + "tpp_threshold_10_unintended_diff_only": 0.004999995231628418, + "tpp_threshold_20_total_metric": 0.0792500227689743, + "tpp_threshold_20_intended_diff_only": 0.0820000171661377, + "tpp_threshold_20_unintended_diff_only": 0.002749994397163391, + "tpp_threshold_50_total_metric": 0.15000002086162567, + "tpp_threshold_50_intended_diff_only": 0.16100001335144043, + "tpp_threshold_50_unintended_diff_only": 0.010999992489814758, + "tpp_threshold_100_total_metric": 0.24075005948543549, + "tpp_threshold_100_intended_diff_only": 0.2600000500679016, + "tpp_threshold_100_unintended_diff_only": 0.019249990582466125, + "tpp_threshold_500_total_metric": 0.40950004756450653, + "tpp_threshold_500_intended_diff_only": 0.437000036239624, + "tpp_threshold_500_unintended_diff_only": 0.027499988675117493 }, "3": { - "tpp_threshold_2_total_metric": 0.012000009417533875, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": -0.0070000141859054565, - "tpp_threshold_5_total_metric": 0.01724998652935028, - "tpp_threshold_5_intended_diff_only": 0.014999985694885254, - "tpp_threshold_5_unintended_diff_only": -0.002250000834465027, - "tpp_threshold_10_total_metric": 0.01799999177455902, - "tpp_threshold_10_intended_diff_only": 0.018999993801116943, - "tpp_threshold_10_unintended_diff_only": 0.0010000020265579224 + "tpp_threshold_2_total_metric": 0.0032499730587005615, + "tpp_threshold_2_intended_diff_only": 0.001999974250793457, + "tpp_threshold_2_unintended_diff_only": -0.0012499988079071045, + "tpp_threshold_5_total_metric": 0.011499986052513123, + "tpp_threshold_5_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_5_unintended_diff_only": -0.005500003695487976, + "tpp_threshold_10_total_metric": 0.016749978065490723, + "tpp_threshold_10_intended_diff_only": 0.014999985694885254, + "tpp_threshold_10_unintended_diff_only": -0.0017499923706054688, + "tpp_threshold_20_total_metric": 0.00950002670288086, + "tpp_threshold_20_intended_diff_only": 0.012000024318695068, + "tpp_threshold_20_unintended_diff_only": 0.002499997615814209, + "tpp_threshold_50_total_metric": 0.059749990701675415, + "tpp_threshold_50_intended_diff_only": 0.06499999761581421, + "tpp_threshold_50_unintended_diff_only": 0.005250006914138794, + "tpp_threshold_100_total_metric": 0.10475000739097595, + "tpp_threshold_100_intended_diff_only": 0.11500000953674316, + "tpp_threshold_100_unintended_diff_only": 0.010250002145767212, + "tpp_threshold_500_total_metric": 0.3474999964237213, + "tpp_threshold_500_intended_diff_only": 0.3799999952316284, + "tpp_threshold_500_unintended_diff_only": 0.032499998807907104 }, "5": { - "tpp_threshold_2_total_metric": -0.008250042796134949, - "tpp_threshold_2_intended_diff_only": -0.001000046730041504, - "tpp_threshold_2_unintended_diff_only": 0.007249996066093445, - "tpp_threshold_5_total_metric": 0.002249971032142639, - "tpp_threshold_5_intended_diff_only": 0.0029999613761901855, - "tpp_threshold_5_unintended_diff_only": 0.0007499903440475464, - "tpp_threshold_10_total_metric": 0.007499948143959045, - "tpp_threshold_10_intended_diff_only": 0.008999943733215332, - "tpp_threshold_10_unintended_diff_only": 0.0014999955892562866 + "tpp_threshold_2_total_metric": 0.0012499839067459106, + "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, + "tpp_threshold_2_unintended_diff_only": -0.00024999678134918213, + "tpp_threshold_5_total_metric": 0.010499924421310425, + "tpp_threshold_5_intended_diff_only": 0.008999943733215332, + "tpp_threshold_5_unintended_diff_only": -0.0014999806880950928, + "tpp_threshold_10_total_metric": 0.005499988794326782, + "tpp_threshold_10_intended_diff_only": 0.009999990463256836, + "tpp_threshold_10_unintended_diff_only": 0.004500001668930054, + "tpp_threshold_20_total_metric": 0.027749985456466675, + "tpp_threshold_20_intended_diff_only": 0.02799999713897705, + "tpp_threshold_20_unintended_diff_only": 0.000250011682510376, + "tpp_threshold_50_total_metric": 0.07424995303153992, + "tpp_threshold_50_intended_diff_only": 0.08199995756149292, + "tpp_threshold_50_unintended_diff_only": 0.007750004529953003, + "tpp_threshold_100_total_metric": 0.17574995756149292, + "tpp_threshold_100_intended_diff_only": 0.1839999556541443, + "tpp_threshold_100_unintended_diff_only": 0.008249998092651367, + "tpp_threshold_500_total_metric": 0.3725000023841858, + "tpp_threshold_500_intended_diff_only": 0.4089999794960022, + "tpp_threshold_500_unintended_diff_only": 0.036499977111816406 }, "6": { - "tpp_threshold_2_total_metric": 0.026500016450881958, - "tpp_threshold_2_intended_diff_only": 0.027000010013580322, - "tpp_threshold_2_unintended_diff_only": 0.0004999935626983643, - "tpp_threshold_5_total_metric": 0.030000001192092896, - "tpp_threshold_5_intended_diff_only": 0.03700000047683716, - "tpp_threshold_5_unintended_diff_only": 0.006999999284744263, - "tpp_threshold_10_total_metric": 0.058999985456466675, - "tpp_threshold_10_intended_diff_only": 0.06099998950958252, - "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447 + "tpp_threshold_2_total_metric": 0.018000006675720215, + "tpp_threshold_2_intended_diff_only": 0.023000001907348633, + "tpp_threshold_2_unintended_diff_only": 0.004999995231628418, + "tpp_threshold_5_total_metric": 0.03299999237060547, + "tpp_threshold_5_intended_diff_only": 0.0339999794960022, + "tpp_threshold_5_unintended_diff_only": 0.0009999871253967285, + "tpp_threshold_10_total_metric": 0.07074999809265137, + "tpp_threshold_10_intended_diff_only": 0.07400000095367432, + "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492, + "tpp_threshold_20_total_metric": 0.10700000822544098, + "tpp_threshold_20_intended_diff_only": 0.11599999666213989, + "tpp_threshold_20_unintended_diff_only": 0.008999988436698914, + "tpp_threshold_50_total_metric": 0.1834999918937683, + "tpp_threshold_50_intended_diff_only": 0.1899999976158142, + "tpp_threshold_50_unintended_diff_only": 0.0065000057220458984, + "tpp_threshold_100_total_metric": 0.25974999368190765, + "tpp_threshold_100_intended_diff_only": 0.26899999380111694, + "tpp_threshold_100_unintended_diff_only": 0.00925000011920929, + "tpp_threshold_500_total_metric": 0.34700001776218414, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.0260000079870224 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-207519-tokens-850000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-207519-tokens-850000000/scr/saebench_scr_custom_sae_eval_results.json index 8057fd2bd5cd69617470ef9d914e325d23bf6699..54d70b62889296a07418d0d5efdc2032a642275f 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-207519-tokens-850000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-207519-tokens-850000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "ffe01158-7a7e-449c-945a-589f0b31c74c", - "datetime_epoch_millis": 1776884870701, + "eval_id": "779ccae0-952e-42cb-a4f6-3510240b489d", + "datetime_epoch_millis": 1777316491341, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.27512449920371496, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.22405199724731734, "scr_metric_threshold_10": 0.20891156459134436, "scr_dir2_threshold_10": 0.20149287130608984, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3165134937976086, + "scr_metric_threshold_20": 0.2213786835087885, + "scr_dir2_threshold_20": 0.20979549805083175, + "scr_dir1_threshold_50": 0.36370478567064457, + "scr_metric_threshold_50": 0.3134490732198989, + "scr_dir2_threshold_50": 0.3034364550289833, + "scr_dir1_threshold_100": 0.1412199235950044, + "scr_metric_threshold_100": 0.345270308848593, + "scr_dir2_threshold_100": 0.3432393798501045, + "scr_dir1_threshold_500": -1.0077250248002592, + "scr_metric_threshold_500": 0.19186126401474754, + "scr_dir2_threshold_500": 0.17056536767374397 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.5256410746257707, "scr_metric_threshold_10": 0.05958549142789553, "scr_dir2_threshold_10": 0.05958549142789553, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.47435892537422925, + "scr_metric_threshold_20": 0.1062176285815671, + "scr_dir2_threshold_20": 0.1062176285815671, + "scr_dir1_threshold_50": 0.5000003820810118, + "scr_metric_threshold_50": 0.19170982855791055, + "scr_dir2_threshold_50": 0.19170982855791055, + "scr_dir1_threshold_100": 0.29487178507484585, + "scr_metric_threshold_100": 0.32383416568792556, + "scr_dir2_threshold_100": 0.32383416568792556, + "scr_dir1_threshold_500": -3.217948179116522, + "scr_metric_threshold_500": 0.044041497182059436, + "scr_dir2_threshold_500": 0.044041497182059436 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.3560604589646946, "scr_metric_threshold_10": 0.1804281691167759, "scr_dir2_threshold_10": 0.1804281691167759, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.27272731377729453, + "scr_metric_threshold_20": 0.08256883242346637, + "scr_dir2_threshold_20": 0.08256883242346637, + "scr_dir1_threshold_50": 0.5075758841466581, + "scr_metric_threshold_50": 0.46483180372392296, + "scr_dir2_threshold_50": 0.46483180372392296, + "scr_dir1_threshold_100": 0.12121234014557077, + "scr_metric_threshold_100": 0.6177370286995434, + "scr_dir2_threshold_100": 0.6177370286995434, + "scr_dir1_threshold_500": -0.5303028592612729, + "scr_metric_threshold_500": -0.1467889139013698, + "scr_dir2_threshold_500": -0.1467889139013698 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": -0.04687532014208255, "scr_metric_threshold_10": 0.16331655168313458, "scr_dir2_threshold_10": 0.16331655168313458, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.28125005820765137, + "scr_metric_threshold_20": 0.20351763196473802, + "scr_dir2_threshold_20": 0.20351763196473802, + "scr_dir1_threshold_50": 0.4062502910382569, + "scr_metric_threshold_50": 0.21859296219014387, + "scr_dir2_threshold_50": 0.21859296219014387, + "scr_dir1_threshold_100": -0.6093745052349633, + "scr_metric_threshold_100": 0.128140681316927, + "scr_dir2_threshold_100": 0.128140681316927, + "scr_dir1_threshold_500": -4.187499417923486, + "scr_metric_threshold_500": -0.005025060155005, + "scr_dir2_threshold_500": -0.005025060155005 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": 0.013422646966463385, "scr_metric_threshold_10": 0.08917196243161127, "scr_dir2_threshold_10": 0.08917196243161127, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.39597288588509905, + "scr_metric_threshold_20": -0.08917196243161127, + "scr_dir2_threshold_20": -0.08917196243161127, + "scr_dir1_threshold_50": 0.4630873208110233, + "scr_metric_threshold_50": -0.08917196243161127, + "scr_dir2_threshold_50": -0.08917196243161127, + "scr_dir1_threshold_100": 0.3422818979880431, + "scr_metric_threshold_100": -0.15605083934346814, + "scr_dir2_threshold_100": -0.15605083934346814, + "scr_dir1_threshold_500": -1.3557049449857088, + "scr_metric_threshold_500": -0.5350317745896676, + "scr_dir2_threshold_500": -0.5350317745896676 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.08421069803000372, "scr_metric_threshold_10": 0.45238098053884956, "scr_dir2_threshold_10": 0.45238098053884956, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.08947378822971379, + "scr_metric_threshold_20": 0.543650817115708, + "scr_dir2_threshold_20": 0.543650817115708, + "scr_dir1_threshold_50": -0.16315799215163673, + "scr_metric_threshold_50": 0.654761901946115, + "scr_dir2_threshold_50": 0.654761901946115, + "scr_dir1_threshold_100": -0.17894726275076694, + "scr_metric_threshold_100": 0.6666667455087788, + "scr_dir2_threshold_100": 0.6666667455087788, + "scr_dir1_threshold_500": -0.14736840784384586, + "scr_metric_threshold_500": 0.7857142350300708, + "scr_dir2_threshold_500": 0.7857142350300708 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.371584539269045, "scr_metric_threshold_10": 0.14122136275770192, "scr_dir2_threshold_10": 0.14122136275770192, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.4043716594522235, + "scr_metric_threshold_20": 0.20992367193697561, + "scr_dir2_threshold_20": 0.20992367193697561, + "scr_dir1_threshold_50": 0.4098360709132541, + "scr_metric_threshold_50": 0.25954203162185363, + "scr_dir2_threshold_50": 0.25954203162185363, + "scr_dir1_threshold_100": 0.4480872768489657, + "scr_metric_threshold_100": 0.35877852349298656, + "scr_dir2_threshold_100": 0.35877852349298656, + "scr_dir1_threshold_500": 0.5956283405477766, + "scr_metric_threshold_500": 0.4007633943795555, + "scr_dir2_threshold_500": 0.4007633943795555 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.3023255233702845, "scr_metric_threshold_10": 0.39914164188042695, "scr_dir2_threshold_10": 0.39914164188042695, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3674417856772556, + "scr_metric_threshold_20": 0.46781131470206244, + "scr_dir2_threshold_20": 0.46781131470206244, + "scr_dir1_threshold_50": 0.5023254124779394, + "scr_metric_threshold_50": 0.5236051041022068, + "scr_dir2_threshold_50": 0.5236051041022068, + "scr_dir1_threshold_100": 0.4465116317753144, + "scr_metric_threshold_100": 0.5579399405130246, + "scr_dir2_threshold_100": 0.5579399405130246, + "scr_dir1_threshold_500": 0.3395348953259431, + "scr_metric_threshold_500": 0.549356359317294, + "scr_dir2_threshold_500": 0.549356359317294 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.18604635689435922, "scr_metric_threshold_10": 0.18604635689435922, "scr_dir2_threshold_10": 0.12669681061232305, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.24651153377740184, + "scr_metric_threshold_20": 0.24651153377740184, + "scr_dir2_threshold_20": 0.15384605011374777, + "scr_dir1_threshold_50": 0.2837209160486496, + "scr_metric_threshold_50": 0.2837209160486496, + "scr_dir2_threshold_50": 0.20361997052132483, + "scr_dir1_threshold_100": 0.2651162249130257, + "scr_metric_threshold_100": 0.2651162249130257, + "scr_dir2_threshold_100": 0.24886879292511796, + "scr_dir1_threshold_500": 0.44186037485504287, + "scr_metric_threshold_500": 0.44186037485504287, + "scr_dir2_threshold_500": 0.2714932041270145 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-207519-tokens-850000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-207519-tokens-850000000/tpp/saebench_tpp_custom_sae_eval_results.json index af0efb6badc534b0b3bbb7f5c18fae2390966ccb..248dfd6af42b0f2f3bed093aeacade18bf8a68c6 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-207519-tokens-850000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-207519-tokens-850000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "6235b8d8-7ed2-4ec6-9cfc-9e062b53e4f6", - "datetime_epoch_millis": 1776885041130, + "eval_id": "8db2fbcf-b982-4d0b-99f8-66791dffce4d", + "datetime_epoch_millis": 1777316765632, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.035725003480911253, - "tpp_threshold_2_intended_diff_only": 0.03910000920295715, - "tpp_threshold_2_unintended_diff_only": 0.0033750057220458983, - "tpp_threshold_5_total_metric": 0.08787501454353332, - "tpp_threshold_5_intended_diff_only": 0.11270002126693726, - "tpp_threshold_5_unintended_diff_only": 0.02482500672340393, - "tpp_threshold_10_total_metric": 0.11080000847578049, - "tpp_threshold_10_intended_diff_only": 0.1576000154018402, - "tpp_threshold_10_unintended_diff_only": 0.04680000692605972, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.033775000274181365, + "tpp_threshold_2_intended_diff_only": 0.03810000419616699, + "tpp_threshold_2_unintended_diff_only": 0.004325003921985626, + "tpp_threshold_5_total_metric": 0.0899750053882599, + "tpp_threshold_5_intended_diff_only": 0.11310001015663147, + "tpp_threshold_5_unintended_diff_only": 0.023125004768371583, + "tpp_threshold_10_total_metric": 0.11182500272989272, + "tpp_threshold_10_intended_diff_only": 0.15940001010894775, + "tpp_threshold_10_unintended_diff_only": 0.047575007379055026, + "tpp_threshold_20_total_metric": 0.1332250028848648, + "tpp_threshold_20_intended_diff_only": 0.20190001130104065, + "tpp_threshold_20_unintended_diff_only": 0.06867500841617584, + "tpp_threshold_50_total_metric": 0.16047500669956208, + "tpp_threshold_50_intended_diff_only": 0.2641000211238861, + "tpp_threshold_50_unintended_diff_only": 0.10362501442432404, + "tpp_threshold_100_total_metric": 0.18515002131462097, + "tpp_threshold_100_intended_diff_only": 0.32330002784729006, + "tpp_threshold_100_unintended_diff_only": 0.13815000653266907, + "tpp_threshold_500_total_metric": 0.2392500162124634, + "tpp_threshold_500_intended_diff_only": 0.4368000328540802, + "tpp_threshold_500_unintended_diff_only": 0.1975500166416168 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.06110000014305115, - "tpp_threshold_2_intended_diff_only": 0.06780000925064086, - "tpp_threshold_2_unintended_diff_only": 0.006700009107589722, - "tpp_threshold_5_total_metric": 0.16075001955032348, - "tpp_threshold_5_intended_diff_only": 0.20820003747940063, - "tpp_threshold_5_unintended_diff_only": 0.047450017929077146, - "tpp_threshold_10_total_metric": 0.19885001182556153, + "tpp_threshold_2_total_metric": 0.0595999926328659, + "tpp_threshold_2_intended_diff_only": 0.06740000247955322, + "tpp_threshold_2_unintended_diff_only": 0.007800009846687317, + "tpp_threshold_5_total_metric": 0.16245001256465913, + "tpp_threshold_5_intended_diff_only": 0.20840002298355104, + "tpp_threshold_5_unintended_diff_only": 0.04595001041889191, + "tpp_threshold_10_total_metric": 0.19715000987052916, "tpp_threshold_10_intended_diff_only": 0.28940002918243407, - "tpp_threshold_10_unintended_diff_only": 0.09055001735687256, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_10_unintended_diff_only": 0.09225001931190491, + "tpp_threshold_20_total_metric": 0.21865000426769257, + "tpp_threshold_20_intended_diff_only": 0.3510000228881836, + "tpp_threshold_20_unintended_diff_only": 0.13235001862049103, + "tpp_threshold_50_total_metric": 0.20735000669956208, + "tpp_threshold_50_intended_diff_only": 0.409600031375885, + "tpp_threshold_50_unintended_diff_only": 0.20225002467632294, + "tpp_threshold_100_total_metric": 0.1747500091791153, + "tpp_threshold_100_intended_diff_only": 0.4380000352859497, + "tpp_threshold_100_unintended_diff_only": 0.2632500261068344, + "tpp_threshold_500_total_metric": 0.09995001256465912, + "tpp_threshold_500_intended_diff_only": 0.46720004081726074, + "tpp_threshold_500_unintended_diff_only": 0.3672500282526016 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.010350006818771362, - "tpp_threshold_2_intended_diff_only": 0.010400009155273438, - "tpp_threshold_2_unintended_diff_only": 5.0002336502075194e-05, - "tpp_threshold_5_total_metric": 0.015000009536743164, - "tpp_threshold_5_intended_diff_only": 0.017200005054473878, - "tpp_threshold_5_unintended_diff_only": 0.002199995517730713, - "tpp_threshold_10_total_metric": 0.02275000512599945, - "tpp_threshold_10_intended_diff_only": 0.02580000162124634, - "tpp_threshold_10_unintended_diff_only": 0.003049996495246887, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.007950007915496826, + "tpp_threshold_2_intended_diff_only": 0.008800005912780762, + "tpp_threshold_2_unintended_diff_only": 0.0008499979972839356, + "tpp_threshold_5_total_metric": 0.017499998211860657, + "tpp_threshold_5_intended_diff_only": 0.017799997329711915, + "tpp_threshold_5_unintended_diff_only": 0.0002999991178512573, + "tpp_threshold_10_total_metric": 0.026499995589256288, + "tpp_threshold_10_intended_diff_only": 0.029399991035461426, + "tpp_threshold_10_unintended_diff_only": 0.002899995446205139, + "tpp_threshold_20_total_metric": 0.047800001502037046, + "tpp_threshold_20_intended_diff_only": 0.052799999713897705, + "tpp_threshold_20_unintended_diff_only": 0.004999998211860657, + "tpp_threshold_50_total_metric": 0.11360000669956208, + "tpp_threshold_50_intended_diff_only": 0.11860001087188721, + "tpp_threshold_50_unintended_diff_only": 0.005000004172325134, + "tpp_threshold_100_total_metric": 0.19555003345012664, + "tpp_threshold_100_intended_diff_only": 0.20860002040863038, + "tpp_threshold_100_unintended_diff_only": 0.013049986958503724, + "tpp_threshold_500_total_metric": 0.37855001986026765, + "tpp_threshold_500_intended_diff_only": 0.40640002489089966, + "tpp_threshold_500_unintended_diff_only": 0.02785000503063202 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.015750020742416382, "tpp_threshold_10_total_metric": 0.1552499681711197, "tpp_threshold_10_intended_diff_only": 0.17100000381469727, - "tpp_threshold_10_unintended_diff_only": 0.015750035643577576 + "tpp_threshold_10_unintended_diff_only": 0.015750035643577576, + "tpp_threshold_20_total_metric": 0.2319999784231186, + "tpp_threshold_20_intended_diff_only": 0.35600000619888306, + "tpp_threshold_20_unintended_diff_only": 0.12400002777576447, + "tpp_threshold_50_total_metric": 0.2149999737739563, + "tpp_threshold_50_intended_diff_only": 0.4169999957084656, + "tpp_threshold_50_unintended_diff_only": 0.20200002193450928, + "tpp_threshold_100_total_metric": 0.19324997067451477, + "tpp_threshold_100_intended_diff_only": 0.44099998474121094, + "tpp_threshold_100_unintended_diff_only": 0.24775001406669617, + "tpp_threshold_500_total_metric": 0.10175000131130219, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.3542500287294388 }, "1": { - "tpp_threshold_2_total_metric": 0.07550002634525299, - "tpp_threshold_2_intended_diff_only": 0.09000003337860107, - "tpp_threshold_2_unintended_diff_only": 0.014500007033348083, - "tpp_threshold_5_total_metric": 0.1940000206232071, + "tpp_threshold_2_total_metric": 0.07924997806549072, + "tpp_threshold_2_intended_diff_only": 0.08899998664855957, + "tpp_threshold_2_unintended_diff_only": 0.009750008583068848, + "tpp_threshold_5_total_metric": 0.19575004279613495, "tpp_threshold_5_intended_diff_only": 0.26500004529953003, - "tpp_threshold_5_unintended_diff_only": 0.07100002467632294, - "tpp_threshold_10_total_metric": 0.21450001001358032, + "tpp_threshold_5_unintended_diff_only": 0.06925000250339508, + "tpp_threshold_10_total_metric": 0.20799998939037323, "tpp_threshold_10_intended_diff_only": 0.3270000219345093, - "tpp_threshold_10_unintended_diff_only": 0.11250001192092896 + "tpp_threshold_10_unintended_diff_only": 0.11900003254413605, + "tpp_threshold_20_total_metric": 0.21424999833106995, + "tpp_threshold_20_intended_diff_only": 0.3410000205039978, + "tpp_threshold_20_unintended_diff_only": 0.12675002217292786, + "tpp_threshold_50_total_metric": 0.21249999105930328, + "tpp_threshold_50_intended_diff_only": 0.38700002431869507, + "tpp_threshold_50_unintended_diff_only": 0.17450003325939178, + "tpp_threshold_100_total_metric": 0.20249998569488525, + "tpp_threshold_100_intended_diff_only": 0.4280000329017639, + "tpp_threshold_100_unintended_diff_only": 0.22550004720687866, + "tpp_threshold_500_total_metric": 0.08500000834465027, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.37400004267692566 }, "2": { - "tpp_threshold_2_total_metric": 0.11775001883506775, - "tpp_threshold_2_intended_diff_only": 0.12700003385543823, - "tpp_threshold_2_unintended_diff_only": 0.009250015020370483, - "tpp_threshold_5_total_metric": 0.21900004148483276, - "tpp_threshold_5_intended_diff_only": 0.29600006341934204, - "tpp_threshold_5_unintended_diff_only": 0.07700002193450928, - "tpp_threshold_10_total_metric": 0.1625000387430191, - "tpp_threshold_10_intended_diff_only": 0.34300005435943604, - "tpp_threshold_10_unintended_diff_only": 0.18050001561641693 + "tpp_threshold_2_total_metric": 0.10625000298023224, + "tpp_threshold_2_intended_diff_only": 0.12400001287460327, + "tpp_threshold_2_unintended_diff_only": 0.017750009894371033, + "tpp_threshold_5_total_metric": 0.22300000488758087, + "tpp_threshold_5_intended_diff_only": 0.29500001668930054, + "tpp_threshold_5_unintended_diff_only": 0.07200001180171967, + "tpp_threshold_10_total_metric": 0.16225002706050873, + "tpp_threshold_10_intended_diff_only": 0.34400004148483276, + "tpp_threshold_10_unintended_diff_only": 0.18175001442432404, + "tpp_threshold_20_total_metric": 0.12725003063678741, + "tpp_threshold_20_intended_diff_only": 0.3670000433921814, + "tpp_threshold_20_unintended_diff_only": 0.23975001275539398, + "tpp_threshold_50_total_metric": 0.10850000381469727, + "tpp_threshold_50_intended_diff_only": 0.38200002908706665, + "tpp_threshold_50_unintended_diff_only": 0.2735000252723694, + "tpp_threshold_100_total_metric": 0.08225004374980927, + "tpp_threshold_100_intended_diff_only": 0.41200006008148193, + "tpp_threshold_100_unintended_diff_only": 0.32975001633167267, + "tpp_threshold_500_total_metric": 0.051750004291534424, + "tpp_threshold_500_intended_diff_only": 0.4520000219345093, + "tpp_threshold_500_unintended_diff_only": 0.40025001764297485 }, "6": { - "tpp_threshold_2_total_metric": 0.012000009417533875, + "tpp_threshold_2_total_metric": 0.009749993681907654, "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": -0.003000006079673767, - "tpp_threshold_5_total_metric": 0.11325003206729889, - "tpp_threshold_5_intended_diff_only": 0.12700003385543823, - "tpp_threshold_5_unintended_diff_only": 0.013750001788139343, - "tpp_threshold_10_total_metric": 0.2862500101327896, + "tpp_threshold_2_unintended_diff_only": -0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.11925001442432404, + "tpp_threshold_5_intended_diff_only": 0.12800002098083496, + "tpp_threshold_5_unintended_diff_only": 0.008750006556510925, + "tpp_threshold_10_total_metric": 0.2880000174045563, "tpp_threshold_10_intended_diff_only": 0.36000001430511475, - "tpp_threshold_10_unintended_diff_only": 0.07375000417232513 + "tpp_threshold_10_unintended_diff_only": 0.07199999690055847, + "tpp_threshold_20_total_metric": 0.31299999356269836, + "tpp_threshold_20_intended_diff_only": 0.4150000214576721, + "tpp_threshold_20_unintended_diff_only": 0.10200002789497375, + "tpp_threshold_50_total_metric": 0.26225003600120544, + "tpp_threshold_50_intended_diff_only": 0.47700005769729614, + "tpp_threshold_50_unintended_diff_only": 0.2147500216960907, + "tpp_threshold_100_total_metric": 0.2082500159740448, + "tpp_threshold_100_intended_diff_only": 0.4830000400543213, + "tpp_threshold_100_unintended_diff_only": 0.2747500240802765, + "tpp_threshold_500_total_metric": 0.13300000131130219, + "tpp_threshold_500_intended_diff_only": 0.4930000305175781, + "tpp_threshold_500_unintended_diff_only": 0.36000002920627594 }, "9": { - "tpp_threshold_2_total_metric": 0.0037499964237213135, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": 0.0012499988079071045, - "tpp_threshold_5_total_metric": 0.14525002241134644, - "tpp_threshold_5_intended_diff_only": 0.20500004291534424, - "tpp_threshold_5_unintended_diff_only": 0.0597500205039978, - "tpp_threshold_10_total_metric": 0.1757500320672989, - "tpp_threshold_10_intended_diff_only": 0.24600005149841309, - "tpp_threshold_10_unintended_diff_only": 0.0702500194311142 + "tpp_threshold_2_total_metric": 0.006250038743019104, + "tpp_threshold_2_intended_diff_only": 0.00700002908706665, + "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.1420000195503235, + "tpp_threshold_5_intended_diff_only": 0.20600003004074097, + "tpp_threshold_5_unintended_diff_only": 0.06400001049041748, + "tpp_threshold_10_total_metric": 0.17225004732608795, + "tpp_threshold_10_intended_diff_only": 0.24500006437301636, + "tpp_threshold_10_unintended_diff_only": 0.0727500170469284, + "tpp_threshold_20_total_metric": 0.2067500203847885, + "tpp_threshold_20_intended_diff_only": 0.2760000228881836, + "tpp_threshold_20_unintended_diff_only": 0.06925000250339508, + "tpp_threshold_50_total_metric": 0.23850002884864807, + "tpp_threshold_50_intended_diff_only": 0.3850000500679016, + "tpp_threshold_50_unintended_diff_only": 0.14650002121925354, + "tpp_threshold_100_total_metric": 0.1875000298023224, + "tpp_threshold_100_intended_diff_only": 0.42600005865097046, + "tpp_threshold_100_unintended_diff_only": 0.23850002884864807, + "tpp_threshold_500_total_metric": 0.12825004756450653, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.3477500230073929 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.007500022649765015, - "tpp_threshold_2_intended_diff_only": 0.003000020980834961, - "tpp_threshold_2_unintended_diff_only": -0.004500001668930054, - "tpp_threshold_5_total_metric": 0.011250004172325134, - "tpp_threshold_5_intended_diff_only": 0.009000003337860107, - "tpp_threshold_5_unintended_diff_only": -0.002250000834465027, - "tpp_threshold_10_total_metric": -0.0007499903440475464, + "tpp_threshold_2_total_metric": 0.0012499988079071045, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_5_total_metric": 0.004249989986419678, + "tpp_threshold_5_intended_diff_only": 0.004999995231628418, + "tpp_threshold_5_unintended_diff_only": 0.0007500052452087402, + "tpp_threshold_10_total_metric": 0.0015000104904174805, "tpp_threshold_10_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_10_unintended_diff_only": 0.004749998450279236 + "tpp_threshold_10_unintended_diff_only": 0.002499997615814209, + "tpp_threshold_20_total_metric": 0.011500000953674316, + "tpp_threshold_20_intended_diff_only": 0.013999998569488525, + "tpp_threshold_20_unintended_diff_only": 0.002499997615814209, + "tpp_threshold_50_total_metric": 0.0442499965429306, + "tpp_threshold_50_intended_diff_only": 0.041999995708465576, + "tpp_threshold_50_unintended_diff_only": -0.002250000834465027, + "tpp_threshold_100_total_metric": 0.10675004124641418, + "tpp_threshold_100_intended_diff_only": 0.11800003051757812, + "tpp_threshold_100_unintended_diff_only": 0.01124998927116394, + "tpp_threshold_500_total_metric": 0.3985000252723694, + "tpp_threshold_500_intended_diff_only": 0.4140000343322754, + "tpp_threshold_500_unintended_diff_only": 0.015500009059906006 }, "2": { - "tpp_threshold_2_total_metric": 0.0020000338554382324, - "tpp_threshold_2_intended_diff_only": 0.00700002908706665, - "tpp_threshold_2_unintended_diff_only": 0.004999995231628418, - "tpp_threshold_5_total_metric": 0.0032500624656677246, - "tpp_threshold_5_intended_diff_only": 0.010000050067901611, - "tpp_threshold_5_unintended_diff_only": 0.006749987602233887, - "tpp_threshold_10_total_metric": 0.02000001072883606, - "tpp_threshold_10_intended_diff_only": 0.023000001907348633, - "tpp_threshold_10_unintended_diff_only": 0.0029999911785125732 + "tpp_threshold_2_total_metric": 0.012000054121017456, + "tpp_threshold_2_intended_diff_only": 0.010000050067901611, + "tpp_threshold_2_unintended_diff_only": -0.0020000040531158447, + "tpp_threshold_5_total_metric": 0.007500007748603821, + "tpp_threshold_5_intended_diff_only": 0.013999998569488525, + "tpp_threshold_5_unintended_diff_only": 0.006499990820884705, + "tpp_threshold_10_total_metric": 0.019500046968460083, + "tpp_threshold_10_intended_diff_only": 0.025000035762786865, + "tpp_threshold_10_unintended_diff_only": 0.005499988794326782, + "tpp_threshold_20_total_metric": 0.07250003516674042, + "tpp_threshold_20_intended_diff_only": 0.08100003004074097, + "tpp_threshold_20_unintended_diff_only": 0.00849999487400055, + "tpp_threshold_50_total_metric": 0.1405000537633896, + "tpp_threshold_50_intended_diff_only": 0.15300005674362183, + "tpp_threshold_50_unintended_diff_only": 0.012500002980232239, + "tpp_threshold_100_total_metric": 0.25400005280971527, + "tpp_threshold_100_intended_diff_only": 0.2710000276565552, + "tpp_threshold_100_unintended_diff_only": 0.016999974846839905, + "tpp_threshold_500_total_metric": 0.4137500524520874, + "tpp_threshold_500_intended_diff_only": 0.437000036239624, + "tpp_threshold_500_unintended_diff_only": 0.02324998378753662 }, "3": { - "tpp_threshold_2_total_metric": 0.014250010251998901, - "tpp_threshold_2_intended_diff_only": 0.008000016212463379, - "tpp_threshold_2_unintended_diff_only": -0.0062499940395355225, - "tpp_threshold_5_total_metric": 0.017750024795532227, - "tpp_threshold_5_intended_diff_only": 0.017000019550323486, - "tpp_threshold_5_unintended_diff_only": -0.0007500052452087402, - "tpp_threshold_10_total_metric": 0.015000030398368835, - "tpp_threshold_10_intended_diff_only": 0.017000019550323486, - "tpp_threshold_10_unintended_diff_only": 0.001999989151954651 + "tpp_threshold_2_total_metric": 0.005000010132789612, + "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_2_unintended_diff_only": -0.0010000020265579224, + "tpp_threshold_5_total_metric": 0.011750012636184692, + "tpp_threshold_5_intended_diff_only": 0.008000016212463379, + "tpp_threshold_5_unintended_diff_only": -0.0037499964237213135, + "tpp_threshold_10_total_metric": 0.01824997365474701, + "tpp_threshold_10_intended_diff_only": 0.015999972820281982, + "tpp_threshold_10_unintended_diff_only": -0.002250000834465027, + "tpp_threshold_20_total_metric": 0.008499979972839355, + "tpp_threshold_20_intended_diff_only": 0.010999977588653564, + "tpp_threshold_20_unintended_diff_only": 0.002499997615814209, + "tpp_threshold_50_total_metric": 0.051249995827674866, + "tpp_threshold_50_intended_diff_only": 0.050999999046325684, + "tpp_threshold_50_unintended_diff_only": -0.00024999678134918213, + "tpp_threshold_100_total_metric": 0.11775004863739014, + "tpp_threshold_100_intended_diff_only": 0.12700003385543823, + "tpp_threshold_100_unintended_diff_only": 0.009249985218048096, + "tpp_threshold_500_total_metric": 0.359000027179718, + "tpp_threshold_500_intended_diff_only": 0.39100003242492676, + "tpp_threshold_500_unintended_diff_only": 0.03200000524520874 }, "5": { - "tpp_threshold_2_total_metric": -0.006750017404556274, - "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 0.007750004529953003, - "tpp_threshold_5_total_metric": 0.001999959349632263, - "tpp_threshold_5_intended_diff_only": 0.0029999613761901855, - "tpp_threshold_5_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_10_total_metric": 0.007499977946281433, - "tpp_threshold_10_intended_diff_only": 0.010999977588653564, - "tpp_threshold_10_unintended_diff_only": 0.0034999996423721313 + "tpp_threshold_2_total_metric": 0.001499965786933899, + "tpp_threshold_2_intended_diff_only": 0.001999974250793457, + "tpp_threshold_2_unintended_diff_only": 0.0005000084638595581, + "tpp_threshold_5_total_metric": 0.019999995827674866, + "tpp_threshold_5_intended_diff_only": 0.018999993801116943, + "tpp_threshold_5_unintended_diff_only": -0.0010000020265579224, + "tpp_threshold_10_total_metric": 0.006749972701072693, + "tpp_threshold_10_intended_diff_only": 0.011999964714050293, + "tpp_threshold_10_unintended_diff_only": 0.0052499920129776, + "tpp_threshold_20_total_metric": 0.034999996423721313, + "tpp_threshold_20_intended_diff_only": 0.03700000047683716, + "tpp_threshold_20_unintended_diff_only": 0.0020000040531158447, + "tpp_threshold_50_total_metric": 0.12074998021125793, + "tpp_threshold_50_intended_diff_only": 0.12599998712539673, + "tpp_threshold_50_unintended_diff_only": 0.005250006914138794, + "tpp_threshold_100_total_metric": 0.21724998950958252, + "tpp_threshold_100_intended_diff_only": 0.23199999332427979, + "tpp_threshold_100_unintended_diff_only": 0.014750003814697266, + "tpp_threshold_500_total_metric": 0.37574999034404755, + "tpp_threshold_500_intended_diff_only": 0.4169999957084656, + "tpp_threshold_500_unintended_diff_only": 0.04125000536441803 }, "6": { - "tpp_threshold_2_total_metric": 0.03474998474121094, - "tpp_threshold_2_intended_diff_only": 0.03299999237060547, - "tpp_threshold_2_unintended_diff_only": -0.0017499923706054688, - "tpp_threshold_5_total_metric": 0.04074999690055847, - "tpp_threshold_5_intended_diff_only": 0.046999990940093994, - "tpp_threshold_5_unintended_diff_only": 0.0062499940395355225, - "tpp_threshold_10_total_metric": 0.07199999690055847, - "tpp_threshold_10_intended_diff_only": 0.07400000095367432, - "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447 + "tpp_threshold_2_total_metric": 0.02000001072883606, + "tpp_threshold_2_intended_diff_only": 0.023000001907348633, + "tpp_threshold_2_unintended_diff_only": 0.0029999911785125732, + "tpp_threshold_5_total_metric": 0.04399998486042023, + "tpp_threshold_5_intended_diff_only": 0.042999982833862305, + "tpp_threshold_5_unintended_diff_only": -0.0010000020265579224, + "tpp_threshold_10_total_metric": 0.08649997413158417, + "tpp_threshold_10_intended_diff_only": 0.0899999737739563, + "tpp_threshold_10_unintended_diff_only": 0.0034999996423721313, + "tpp_threshold_20_total_metric": 0.11149999499320984, + "tpp_threshold_20_intended_diff_only": 0.12099999189376831, + "tpp_threshold_20_unintended_diff_only": 0.009499996900558472, + "tpp_threshold_50_total_metric": 0.21125000715255737, + "tpp_threshold_50_intended_diff_only": 0.22100001573562622, + "tpp_threshold_50_unintended_diff_only": 0.009750008583068848, + "tpp_threshold_100_total_metric": 0.28200003504753113, + "tpp_threshold_100_intended_diff_only": 0.29500001668930054, + "tpp_threshold_100_unintended_diff_only": 0.01299998164176941, + "tpp_threshold_500_total_metric": 0.34575000405311584, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.027250021696090698 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-219726-tokens-900000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-219726-tokens-900000000/scr/saebench_scr_custom_sae_eval_results.json index 6d5a55b4850d6926849f02599c7fdccd83dd1257..db38c904eef595ef84f4f58c70dc82d4e30931b5 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-219726-tokens-900000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-219726-tokens-900000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "da2a6887-913b-4c10-8426-ea8bfa58f10f", - "datetime_epoch_millis": 1776884712767, + "eval_id": "d6582b6e-d498-4999-9ec3-a0af5792c811", + "datetime_epoch_millis": 1777316550644, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.2896188146090331, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.2889266157448796, "scr_metric_threshold_10": 0.23494818923208868, "scr_dir2_threshold_10": 0.22345971109744955, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.21992587333994348, + "scr_metric_threshold_20": 0.22353422720014693, + "scr_dir2_threshold_20": 0.21245350675047425, + "scr_dir1_threshold_50": 0.08432192518155021, + "scr_metric_threshold_50": 0.33824373291787563, + "scr_dir2_threshold_50": 0.32354837314889334, + "scr_dir1_threshold_100": -0.2103083440146591, + "scr_metric_threshold_100": 0.33500288328703, + "scr_dir2_threshold_100": 0.31697700710952653, + "scr_dir1_threshold_500": -0.9815309813458944, + "scr_metric_threshold_500": 0.1855971065432701, + "scr_dir2_threshold_500": 0.17933854684292247 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.5256410746257707, "scr_metric_threshold_10": 0.05181341709689583, "scr_dir2_threshold_10": 0.05181341709689583, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.37179539103317, + "scr_metric_threshold_20": 0.10880826855317922, + "scr_dir2_threshold_20": 0.10880826855317922, + "scr_dir1_threshold_50": 0.5384614208981502, + "scr_metric_threshold_50": 0.26683931423164214, + "scr_dir2_threshold_50": 0.26683931423164214, + "scr_dir1_threshold_100": 0.3333335880540079, + "scr_metric_threshold_100": 0.30569953147047735, + "scr_dir2_threshold_100": 0.30569953147047735, + "scr_dir1_threshold_500": -2.7435892537422926, + "scr_metric_threshold_500": 0.07772012564534375, + "scr_dir2_threshold_500": 0.07772012564534375 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.3560604589646946, "scr_metric_threshold_10": 0.3455658339637086, "scr_dir2_threshold_10": 0.3455658339637086, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.48484845748180366, + "scr_metric_threshold_20": 0.08256883242346637, + "scr_dir2_threshold_20": 0.08256883242346637, + "scr_dir1_threshold_50": 0.5681820542194435, + "scr_metric_threshold_50": 0.45871567492686127, + "scr_dir2_threshold_50": 0.45871567492686127, + "scr_dir1_threshold_100": -0.5075754325964184, + "scr_metric_threshold_100": 0.608562653226762, + "scr_dir2_threshold_100": 0.608562653226762, + "scr_dir1_threshold_500": -2.113635778673553, + "scr_metric_threshold_500": 0.04281344841099861, + "scr_dir2_threshold_500": 0.04281344841099861 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.45312467985791743, "scr_metric_threshold_10": 0.13819095138732784, "scr_dir2_threshold_10": 0.13819095138732784, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3437501746229541, + "scr_metric_threshold_20": 0.1809045617464338, + "scr_dir2_threshold_20": 0.1809045617464338, + "scr_dir1_threshold_50": -0.8437501746229541, + "scr_metric_threshold_50": 0.24623124232384397, + "scr_dir2_threshold_50": 0.24623124232384397, + "scr_dir1_threshold_100": -1.1874994179234863, + "scr_metric_threshold_100": 0.13567842130982535, + "scr_dir2_threshold_100": 0.13567842130982535, + "scr_dir1_threshold_500": -2.6406250291038256, + "scr_metric_threshold_500": -0.017587860302908362, + "scr_dir2_threshold_500": -0.017587860302908362 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": -0.134228469820646, "scr_metric_threshold_10": 0.09872616905774931, "scr_dir2_threshold_10": 0.09872616905774931, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.6711411490096227, + "scr_metric_threshold_20": -0.025477757787232685, + "scr_dir2_threshold_20": -0.025477757787232685, + "scr_dir1_threshold_50": -0.7181212134546494, + "scr_metric_threshold_50": 0.13694280573859832, + "scr_dir2_threshold_50": 0.13694280573859832, + "scr_dir1_threshold_100": -1.4630877208422257, + "scr_metric_threshold_100": -0.11464972021884395, + "scr_dir2_threshold_100": -0.11464972021884395, + "scr_dir1_threshold_500": -1.5906044671484376, + "scr_metric_threshold_500": -0.5509553257507622, + "scr_dir2_threshold_500": -0.5509553257507622 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.06842111372221286, "scr_metric_threshold_10": 0.515872927644938, "scr_dir2_threshold_10": 0.515872927644938, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.09473687842942387, + "scr_metric_threshold_20": 0.5753969089319204, + "scr_dir2_threshold_20": 0.5753969089319204, + "scr_dir1_threshold_50": -0.12631573333634494, + "scr_metric_threshold_50": 0.6865079937623274, + "scr_dir2_threshold_50": 0.6865079937623274, + "scr_dir1_threshold_100": -0.184210352950477, + "scr_metric_threshold_100": 0.714285646706761, + "scr_dir2_threshold_100": 0.714285646706761, + "scr_dir1_threshold_500": -0.16315799215163673, + "scr_metric_threshold_500": 0.801587280938177, + "scr_dir2_threshold_500": 0.801587280938177 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.4098360709132541, "scr_metric_threshold_10": 0.17175577294818423, "scr_dir2_threshold_10": 0.17175577294818423, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.4098360709132541, + "scr_metric_threshold_20": 0.2061069275378211, + "scr_dir2_threshold_20": 0.2061069275378211, + "scr_dir1_threshold_50": 0.377049276438573, + "scr_metric_threshold_50": 0.2671755204201627, + "scr_dir2_threshold_50": 0.2671755204201627, + "scr_dir1_threshold_100": 0.4754099855711136, + "scr_metric_threshold_100": 0.36259549539076413, + "scr_dir2_threshold_100": 0.36259549539076413, + "scr_dir1_threshold_500": 0.4644808369405549, + "scr_metric_threshold_500": 0.36641223978991866, + "scr_dir2_threshold_500": 0.36641223978991866 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.4139533620063974, "scr_metric_threshold_10": 0.33905580606846963, "scr_dir2_threshold_10": 0.33905580606846963, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.46046493833553914, + "scr_metric_threshold_20": 0.3948498512825616, + "scr_dir2_threshold_20": 0.3948498512825616, + "scr_dir1_threshold_50": 0.548837266037944, + "scr_metric_threshold_50": 0.31330480666733007, + "scr_dir2_threshold_50": 0.31330480666733007, + "scr_dir1_threshold_100": 0.5395347844335979, + "scr_metric_threshold_100": 0.3562232242738785, + "scr_dir2_threshold_100": 0.3562232242738785, + "scr_dir1_threshold_500": 0.5906977401803443, + "scr_metric_threshold_500": 0.4206008506837011, + "scr_dir2_threshold_500": 0.4206008506837011 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.21860463568943592, "scr_metric_threshold_10": 0.21860463568943592, "scr_dir2_threshold_10": 0.12669681061232305, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2651162249130257, + "scr_metric_threshold_20": 0.2651162249130257, + "scr_dir2_threshold_20": 0.17647046131564434, + "scr_dir1_threshold_50": 0.3302325052722394, + "scr_metric_threshold_50": 0.3302325052722394, + "scr_dir2_threshold_50": 0.21266962712038115, + "scr_dir1_threshold_100": 0.3116278141366155, + "scr_metric_threshold_100": 0.3116278141366155, + "scr_dir2_threshold_100": 0.16742080471658802, + "scr_dir1_threshold_500": 0.3441860929316922, + "scr_metric_threshold_500": 0.3441860929316922, + "scr_dir2_threshold_500": 0.29411761532891106 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-219726-tokens-900000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-219726-tokens-900000000/tpp/saebench_tpp_custom_sae_eval_results.json index 797caef4a58774f7ccfbecb8510b76f41f5665d8..4357dd734517c8d18be9fc6512f45b5f6970f023 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-219726-tokens-900000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-219726-tokens-900000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "892cbbec-a6ec-499c-b18e-a7866ca2fb4d", - "datetime_epoch_millis": 1776884868502, + "eval_id": "e239af40-39b3-4458-84cb-08e5f69c02af", + "datetime_epoch_millis": 1777316823043, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.037274999916553496, - "tpp_threshold_2_intended_diff_only": 0.04110000133514404, - "tpp_threshold_2_unintended_diff_only": 0.0038250014185905457, - "tpp_threshold_5_total_metric": 0.09112499803304672, - "tpp_threshold_5_intended_diff_only": 0.11650000214576721, - "tpp_threshold_5_unintended_diff_only": 0.02537500411272049, - "tpp_threshold_10_total_metric": 0.11410000622272491, - "tpp_threshold_10_intended_diff_only": 0.16100001335144043, - "tpp_threshold_10_unintended_diff_only": 0.04690000712871552, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.035974989831447604, + "tpp_threshold_2_intended_diff_only": 0.04059999585151672, + "tpp_threshold_2_unintended_diff_only": 0.004625006020069122, + "tpp_threshold_5_total_metric": 0.09110001027584076, + "tpp_threshold_5_intended_diff_only": 0.11510001420974732, + "tpp_threshold_5_unintended_diff_only": 0.024000003933906555, + "tpp_threshold_10_total_metric": 0.11435000449419021, + "tpp_threshold_10_intended_diff_only": 0.16220000982284544, + "tpp_threshold_10_unintended_diff_only": 0.04785000532865524, + "tpp_threshold_20_total_metric": 0.13832499384880065, + "tpp_threshold_20_intended_diff_only": 0.20879999995231627, + "tpp_threshold_20_unintended_diff_only": 0.07047500610351562, + "tpp_threshold_50_total_metric": 0.16092500686645508, + "tpp_threshold_50_intended_diff_only": 0.2679000198841095, + "tpp_threshold_50_unintended_diff_only": 0.10697501301765441, + "tpp_threshold_100_total_metric": 0.19520001709461213, + "tpp_threshold_100_intended_diff_only": 0.331900030374527, + "tpp_threshold_100_unintended_diff_only": 0.13670001327991485, + "tpp_threshold_500_total_metric": 0.2363000199198723, + "tpp_threshold_500_intended_diff_only": 0.4355000376701355, + "tpp_threshold_500_unintended_diff_only": 0.19920001775026322 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.06464999914169312, - "tpp_threshold_2_intended_diff_only": 0.0718000054359436, - "tpp_threshold_2_unintended_diff_only": 0.007150006294250488, - "tpp_threshold_5_total_metric": 0.1643500119447708, + "tpp_threshold_2_total_metric": 0.0632999986410141, + "tpp_threshold_2_intended_diff_only": 0.0714000105857849, + "tpp_threshold_2_unintended_diff_only": 0.008100011944770813, + "tpp_threshold_5_total_metric": 0.16535001397132873, "tpp_threshold_5_intended_diff_only": 0.2128000259399414, - "tpp_threshold_5_unintended_diff_only": 0.04845001399517059, - "tpp_threshold_10_total_metric": 0.2018500030040741, + "tpp_threshold_5_unintended_diff_only": 0.04745001196861267, + "tpp_threshold_10_total_metric": 0.1999499976634979, "tpp_threshold_10_intended_diff_only": 0.29300001859664915, - "tpp_threshold_10_unintended_diff_only": 0.09115001559257507, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_10_unintended_diff_only": 0.09305002093315125, + "tpp_threshold_20_total_metric": 0.223799991607666, + "tpp_threshold_20_intended_diff_only": 0.35940001010894773, + "tpp_threshold_20_unintended_diff_only": 0.13560001850128173, + "tpp_threshold_50_total_metric": 0.2084500163793564, + "tpp_threshold_50_intended_diff_only": 0.41620004177093506, + "tpp_threshold_50_unintended_diff_only": 0.20775002539157866, + "tpp_threshold_100_total_metric": 0.18575001358985901, + "tpp_threshold_100_intended_diff_only": 0.44400004148483274, + "tpp_threshold_100_unintended_diff_only": 0.25825002789497375, + "tpp_threshold_500_total_metric": 0.10000000596046447, + "tpp_threshold_500_intended_diff_only": 0.46720004081726074, + "tpp_threshold_500_unintended_diff_only": 0.36720003485679625 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.00990000069141388, - "tpp_threshold_2_intended_diff_only": 0.010399997234344482, - "tpp_threshold_2_unintended_diff_only": 0.000499996542930603, - "tpp_threshold_5_total_metric": 0.017899984121322633, - "tpp_threshold_5_intended_diff_only": 0.02019997835159302, - "tpp_threshold_5_unintended_diff_only": 0.0022999942302703857, - "tpp_threshold_10_total_metric": 0.02635000944137573, - "tpp_threshold_10_intended_diff_only": 0.02900000810623169, - "tpp_threshold_10_unintended_diff_only": 0.002649998664855957, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.008649981021881104, + "tpp_threshold_2_intended_diff_only": 0.009799981117248535, + "tpp_threshold_2_unintended_diff_only": 0.0011500000953674316, + "tpp_threshold_5_total_metric": 0.016850006580352784, + "tpp_threshold_5_intended_diff_only": 0.01740000247955322, + "tpp_threshold_5_unintended_diff_only": 0.0005499958992004394, + "tpp_threshold_10_total_metric": 0.02875001132488251, + "tpp_threshold_10_intended_diff_only": 0.03140000104904175, + "tpp_threshold_10_unintended_diff_only": 0.0026499897241592405, + "tpp_threshold_20_total_metric": 0.052849996089935306, + "tpp_threshold_20_intended_diff_only": 0.05819998979568482, + "tpp_threshold_20_unintended_diff_only": 0.005349993705749512, + "tpp_threshold_50_total_metric": 0.11339999735355377, + "tpp_threshold_50_intended_diff_only": 0.11959999799728394, + "tpp_threshold_50_unintended_diff_only": 0.006200000643730164, + "tpp_threshold_100_total_metric": 0.20465002059936524, + "tpp_threshold_100_intended_diff_only": 0.21980001926422119, + "tpp_threshold_100_unintended_diff_only": 0.015149998664855956, + "tpp_threshold_500_total_metric": 0.3726000338792801, + "tpp_threshold_500_intended_diff_only": 0.4038000345230103, + "tpp_threshold_500_unintended_diff_only": 0.031200000643730165 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.0155000239610672, "tpp_threshold_10_total_metric": 0.1409999579191208, "tpp_threshold_10_intended_diff_only": 0.15799999237060547, - "tpp_threshold_10_unintended_diff_only": 0.01700003445148468 + "tpp_threshold_10_unintended_diff_only": 0.01700003445148468, + "tpp_threshold_20_total_metric": 0.23149995505809784, + "tpp_threshold_20_intended_diff_only": 0.343999981880188, + "tpp_threshold_20_unintended_diff_only": 0.11250002682209015, + "tpp_threshold_50_total_metric": 0.2254999577999115, + "tpp_threshold_50_intended_diff_only": 0.4179999828338623, + "tpp_threshold_50_unintended_diff_only": 0.1925000250339508, + "tpp_threshold_100_total_metric": 0.21774999797344208, + "tpp_threshold_100_intended_diff_only": 0.44200003147125244, + "tpp_threshold_100_unintended_diff_only": 0.22425003349781036, + "tpp_threshold_500_total_metric": 0.10149998962879181, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.35450004041194916 }, "1": { - "tpp_threshold_2_total_metric": 0.08574998378753662, - "tpp_threshold_2_intended_diff_only": 0.10199999809265137, - "tpp_threshold_2_unintended_diff_only": 0.016250014305114746, - "tpp_threshold_5_total_metric": 0.19975002110004425, + "tpp_threshold_2_total_metric": 0.08900000154972076, + "tpp_threshold_2_intended_diff_only": 0.10000002384185791, + "tpp_threshold_2_unintended_diff_only": 0.011000022292137146, + "tpp_threshold_5_total_metric": 0.20075003802776337, "tpp_threshold_5_intended_diff_only": 0.27500003576278687, - "tpp_threshold_5_unintended_diff_only": 0.07525001466274261, - "tpp_threshold_10_total_metric": 0.22324998676776886, + "tpp_threshold_5_unintended_diff_only": 0.0742499977350235, + "tpp_threshold_10_total_metric": 0.21774998307228088, "tpp_threshold_10_intended_diff_only": 0.3330000042915344, - "tpp_threshold_10_unintended_diff_only": 0.10975001752376556 + "tpp_threshold_10_unintended_diff_only": 0.11525002121925354, + "tpp_threshold_20_total_metric": 0.23224997520446777, + "tpp_threshold_20_intended_diff_only": 0.3569999933242798, + "tpp_threshold_20_unintended_diff_only": 0.12475001811981201, + "tpp_threshold_50_total_metric": 0.21050000190734863, + "tpp_threshold_50_intended_diff_only": 0.39500004053115845, + "tpp_threshold_50_unintended_diff_only": 0.18450003862380981, + "tpp_threshold_100_total_metric": 0.1807500123977661, + "tpp_threshold_100_intended_diff_only": 0.4360000491142273, + "tpp_threshold_100_unintended_diff_only": 0.2552500367164612, + "tpp_threshold_500_total_metric": 0.0885000079870224, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.37050004303455353 }, "2": { - "tpp_threshold_2_total_metric": 0.11775003373622894, - "tpp_threshold_2_intended_diff_only": 0.12700003385543823, - "tpp_threshold_2_unintended_diff_only": 0.00925000011920929, - "tpp_threshold_5_total_metric": 0.22825002670288086, - "tpp_threshold_5_intended_diff_only": 0.2980000376701355, - "tpp_threshold_5_unintended_diff_only": 0.06975001096725464, - "tpp_threshold_10_total_metric": 0.18250004947185516, - "tpp_threshold_10_intended_diff_only": 0.34800004959106445, - "tpp_threshold_10_unintended_diff_only": 0.1655000001192093 + "tpp_threshold_2_total_metric": 0.1067500114440918, + "tpp_threshold_2_intended_diff_only": 0.12400001287460327, + "tpp_threshold_2_unintended_diff_only": 0.017250001430511475, + "tpp_threshold_5_total_metric": 0.23225004971027374, + "tpp_threshold_5_intended_diff_only": 0.29700005054473877, + "tpp_threshold_5_unintended_diff_only": 0.06475000083446503, + "tpp_threshold_10_total_metric": 0.18050001561641693, + "tpp_threshold_10_intended_diff_only": 0.3490000367164612, + "tpp_threshold_10_unintended_diff_only": 0.16850002110004425, + "tpp_threshold_20_total_metric": 0.1367499977350235, + "tpp_threshold_20_intended_diff_only": 0.36900001764297485, + "tpp_threshold_20_unintended_diff_only": 0.23225001990795135, + "tpp_threshold_50_total_metric": 0.11875003576278687, + "tpp_threshold_50_intended_diff_only": 0.393000066280365, + "tpp_threshold_50_unintended_diff_only": 0.2742500305175781, + "tpp_threshold_100_total_metric": 0.08375002443790436, + "tpp_threshold_100_intended_diff_only": 0.41700005531311035, + "tpp_threshold_100_unintended_diff_only": 0.333250030875206, + "tpp_threshold_500_total_metric": 0.05399997532367706, + "tpp_threshold_500_intended_diff_only": 0.4520000219345093, + "tpp_threshold_500_unintended_diff_only": 0.3980000466108322 }, "6": { - "tpp_threshold_2_total_metric": 0.011750012636184692, + "tpp_threshold_2_total_metric": 0.009499981999397278, "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": -0.002750009298324585, - "tpp_threshold_5_total_metric": 0.24400001764297485, - "tpp_threshold_5_intended_diff_only": 0.31300002336502075, - "tpp_threshold_5_unintended_diff_only": 0.0690000057220459, - "tpp_threshold_10_total_metric": 0.27275000512599945, + "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704, + "tpp_threshold_5_total_metric": 0.24699999392032623, + "tpp_threshold_5_intended_diff_only": 0.3140000104904175, + "tpp_threshold_5_unintended_diff_only": 0.06700001657009125, + "tpp_threshold_10_total_metric": 0.2749999910593033, "tpp_threshold_10_intended_diff_only": 0.36500000953674316, - "tpp_threshold_10_unintended_diff_only": 0.09225000441074371 + "tpp_threshold_10_unintended_diff_only": 0.09000001847743988, + "tpp_threshold_20_total_metric": 0.3005000352859497, + "tpp_threshold_20_intended_diff_only": 0.4270000457763672, + "tpp_threshold_20_unintended_diff_only": 0.12650001049041748, + "tpp_threshold_50_total_metric": 0.2460000365972519, + "tpp_threshold_50_intended_diff_only": 0.47700005769729614, + "tpp_threshold_50_unintended_diff_only": 0.23100002110004425, + "tpp_threshold_100_total_metric": 0.22150002419948578, + "tpp_threshold_100_intended_diff_only": 0.4830000400543213, + "tpp_threshold_100_unintended_diff_only": 0.2615000158548355, + "tpp_threshold_500_total_metric": 0.13225001096725464, + "tpp_threshold_500_intended_diff_only": 0.4930000305175781, + "tpp_threshold_500_unintended_diff_only": 0.3607500195503235 }, "9": { - "tpp_threshold_2_total_metric": 0.003999993205070496, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_5_total_metric": 0.025250032544136047, + "tpp_threshold_2_total_metric": 0.0072500258684158325, + "tpp_threshold_2_intended_diff_only": 0.008000016212463379, + "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.02225002646446228, "tpp_threshold_5_intended_diff_only": 0.03800004720687866, - "tpp_threshold_5_unintended_diff_only": 0.012750014662742615, - "tpp_threshold_10_total_metric": 0.18975001573562622, - "tpp_threshold_10_intended_diff_only": 0.26100003719329834, - "tpp_threshold_10_unintended_diff_only": 0.07125002145767212 + "tpp_threshold_5_unintended_diff_only": 0.015750020742416382, + "tpp_threshold_10_total_metric": 0.18550004065036774, + "tpp_threshold_10_intended_diff_only": 0.2600000500679016, + "tpp_threshold_10_unintended_diff_only": 0.07450000941753387, + "tpp_threshold_20_total_metric": 0.21799999475479126, + "tpp_threshold_20_intended_diff_only": 0.30000001192092896, + "tpp_threshold_20_unintended_diff_only": 0.0820000171661377, + "tpp_threshold_50_total_metric": 0.24150004982948303, + "tpp_threshold_50_intended_diff_only": 0.3980000615119934, + "tpp_threshold_50_unintended_diff_only": 0.15650001168251038, + "tpp_threshold_100_total_metric": 0.22500000894069672, + "tpp_threshold_100_intended_diff_only": 0.44200003147125244, + "tpp_threshold_100_unintended_diff_only": 0.21700002253055573, + "tpp_threshold_500_total_metric": 0.12375004589557648, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.35225002467632294 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.008750006556510925, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": -0.004749998450279236, - "tpp_threshold_5_total_metric": 0.009499981999397278, - "tpp_threshold_5_intended_diff_only": 0.006999969482421875, - "tpp_threshold_5_unintended_diff_only": -0.002500012516975403, - "tpp_threshold_10_total_metric": -0.0007499903440475464, - "tpp_threshold_10_intended_diff_only": 0.003000020980834961, - "tpp_threshold_10_unintended_diff_only": 0.0037500113248825073 + "tpp_threshold_2_total_metric": 0.0012499988079071045, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_5_total_metric": 0.0015000253915786743, + "tpp_threshold_5_intended_diff_only": 0.003000020980834961, + "tpp_threshold_5_unintended_diff_only": 0.0014999955892562866, + "tpp_threshold_10_total_metric": 0.0007499903440475464, + "tpp_threshold_10_intended_diff_only": 0.001999974250793457, + "tpp_threshold_10_unintended_diff_only": 0.0012499839067459106, + "tpp_threshold_20_total_metric": 0.01000000536441803, + "tpp_threshold_20_intended_diff_only": 0.013999998569488525, + "tpp_threshold_20_unintended_diff_only": 0.003999993205070496, + "tpp_threshold_50_total_metric": 0.046249985694885254, + "tpp_threshold_50_intended_diff_only": 0.046999990940093994, + "tpp_threshold_50_unintended_diff_only": 0.0007500052452087402, + "tpp_threshold_100_total_metric": 0.09825000166893005, + "tpp_threshold_100_intended_diff_only": 0.10600000619888306, + "tpp_threshold_100_unintended_diff_only": 0.007750004529953003, + "tpp_threshold_500_total_metric": 0.3865000456571579, + "tpp_threshold_500_intended_diff_only": 0.40000003576278687, + "tpp_threshold_500_unintended_diff_only": 0.013499990105628967 }, "2": { - "tpp_threshold_2_total_metric": 0.0052500516176223755, - "tpp_threshold_2_intended_diff_only": 0.010000050067901611, - "tpp_threshold_2_unintended_diff_only": 0.004749998450279236, - "tpp_threshold_5_total_metric": 0.011250019073486328, - "tpp_threshold_5_intended_diff_only": 0.018000006675720215, - "tpp_threshold_5_unintended_diff_only": 0.006749987602233887, - "tpp_threshold_10_total_metric": 0.027250051498413086, - "tpp_threshold_10_intended_diff_only": 0.029000043869018555, - "tpp_threshold_10_unintended_diff_only": 0.0017499923706054688 + "tpp_threshold_2_total_metric": 0.015000015497207642, + "tpp_threshold_2_intended_diff_only": 0.013000011444091797, + "tpp_threshold_2_unintended_diff_only": -0.0020000040531158447, + "tpp_threshold_5_total_metric": 0.007750004529953003, + "tpp_threshold_5_intended_diff_only": 0.013999998569488525, + "tpp_threshold_5_unintended_diff_only": 0.0062499940395355225, + "tpp_threshold_10_total_metric": 0.02800007164478302, + "tpp_threshold_10_intended_diff_only": 0.033000051975250244, + "tpp_threshold_10_unintended_diff_only": 0.004999980330467224, + "tpp_threshold_20_total_metric": 0.0975000411272049, + "tpp_threshold_20_intended_diff_only": 0.1040000319480896, + "tpp_threshold_20_unintended_diff_only": 0.006499990820884705, + "tpp_threshold_50_total_metric": 0.166500061750412, + "tpp_threshold_50_intended_diff_only": 0.18100005388259888, + "tpp_threshold_50_unintended_diff_only": 0.01449999213218689, + "tpp_threshold_100_total_metric": 0.28525008261203766, + "tpp_threshold_100_intended_diff_only": 0.30500006675720215, + "tpp_threshold_100_unintended_diff_only": 0.01974998414516449, + "tpp_threshold_500_total_metric": 0.40500007569789886, + "tpp_threshold_500_intended_diff_only": 0.43900007009506226, + "tpp_threshold_500_unintended_diff_only": 0.03399999439716339 }, "3": { - "tpp_threshold_2_total_metric": 0.011999979615211487, - "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_2_unintended_diff_only": -0.00599999725818634, - "tpp_threshold_5_total_metric": 0.016749992966651917, - "tpp_threshold_5_intended_diff_only": 0.014999985694885254, - "tpp_threshold_5_unintended_diff_only": -0.0017500072717666626, - "tpp_threshold_10_total_metric": 0.019250035285949707, - "tpp_threshold_10_intended_diff_only": 0.021000027656555176, - "tpp_threshold_10_unintended_diff_only": 0.0017499923706054688 + "tpp_threshold_2_total_metric": 0.0029999762773513794, + "tpp_threshold_2_intended_diff_only": 0.001999974250793457, + "tpp_threshold_2_unintended_diff_only": -0.0010000020265579224, + "tpp_threshold_5_total_metric": 0.013250008225440979, + "tpp_threshold_5_intended_diff_only": 0.008000016212463379, + "tpp_threshold_5_unintended_diff_only": -0.0052499920129776, + "tpp_threshold_10_total_metric": 0.020500004291534424, + "tpp_threshold_10_intended_diff_only": 0.018000006675720215, + "tpp_threshold_10_unintended_diff_only": -0.002499997615814209, + "tpp_threshold_20_total_metric": 0.012249991297721863, + "tpp_threshold_20_intended_diff_only": 0.014999985694885254, + "tpp_threshold_20_unintended_diff_only": 0.002749994397163391, + "tpp_threshold_50_total_metric": 0.05524998903274536, + "tpp_threshold_50_intended_diff_only": 0.05699998140335083, + "tpp_threshold_50_unintended_diff_only": 0.0017499923706054688, + "tpp_threshold_100_total_metric": 0.1325000375509262, + "tpp_threshold_100_intended_diff_only": 0.15000003576278687, + "tpp_threshold_100_unintended_diff_only": 0.017499998211860657, + "tpp_threshold_500_total_metric": 0.3602500259876251, + "tpp_threshold_500_intended_diff_only": 0.3920000195503235, + "tpp_threshold_500_unintended_diff_only": 0.031749993562698364 }, "5": { - "tpp_threshold_2_total_metric": -0.007249996066093445, - "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 0.008249983191490173, - "tpp_threshold_5_total_metric": 0.007249951362609863, - "tpp_threshold_5_intended_diff_only": 0.008999943733215332, - "tpp_threshold_5_unintended_diff_only": 0.0017499923706054688, - "tpp_threshold_10_total_metric": 0.014499947428703308, - "tpp_threshold_10_intended_diff_only": 0.01799994707107544, - "tpp_threshold_10_unintended_diff_only": 0.0034999996423721313 + "tpp_threshold_2_total_metric": 0.0029999613761901855, + "tpp_threshold_2_intended_diff_only": 0.0029999613761901855, + "tpp_threshold_2_unintended_diff_only": 0.0, + "tpp_threshold_5_total_metric": 0.014750003814697266, + "tpp_threshold_5_intended_diff_only": 0.013999998569488525, + "tpp_threshold_5_unintended_diff_only": -0.0007500052452087402, + "tpp_threshold_10_total_metric": 0.011249974370002747, + "tpp_threshold_10_intended_diff_only": 0.01699995994567871, + "tpp_threshold_10_unintended_diff_only": 0.005749985575675964, + "tpp_threshold_20_total_metric": 0.028499960899353027, + "tpp_threshold_20_intended_diff_only": 0.030999958515167236, + "tpp_threshold_20_unintended_diff_only": 0.002499997615814209, + "tpp_threshold_50_total_metric": 0.10099998116493225, + "tpp_threshold_50_intended_diff_only": 0.10799998044967651, + "tpp_threshold_50_unintended_diff_only": 0.006999999284744263, + "tpp_threshold_100_total_metric": 0.23374995589256287, + "tpp_threshold_100_intended_diff_only": 0.2529999613761902, + "tpp_threshold_100_unintended_diff_only": 0.01925000548362732, + "tpp_threshold_500_total_metric": 0.37175001204013824, + "tpp_threshold_500_intended_diff_only": 0.4150000214576721, + "tpp_threshold_500_unintended_diff_only": 0.043250009417533875 }, "6": { - "tpp_threshold_2_total_metric": 0.030749961733818054, - "tpp_threshold_2_intended_diff_only": 0.030999958515167236, - "tpp_threshold_2_unintended_diff_only": 0.00024999678134918213, - "tpp_threshold_5_total_metric": 0.04474997520446777, - "tpp_threshold_5_intended_diff_only": 0.05199998617172241, - "tpp_threshold_5_unintended_diff_only": 0.007250010967254639, - "tpp_threshold_10_total_metric": 0.07150000333786011, - "tpp_threshold_10_intended_diff_only": 0.07400000095367432, - "tpp_threshold_10_unintended_diff_only": 0.002499997615814209 + "tpp_threshold_2_total_metric": 0.020999953150749207, + "tpp_threshold_2_intended_diff_only": 0.02599996328353882, + "tpp_threshold_2_unintended_diff_only": 0.005000010132789612, + "tpp_threshold_5_total_metric": 0.046999990940093994, + "tpp_threshold_5_intended_diff_only": 0.04799997806549072, + "tpp_threshold_5_unintended_diff_only": 0.0009999871253967285, + "tpp_threshold_10_total_metric": 0.0832500159740448, + "tpp_threshold_10_intended_diff_only": 0.08700001239776611, + "tpp_threshold_10_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_20_total_metric": 0.1159999817609787, + "tpp_threshold_20_intended_diff_only": 0.12699997425079346, + "tpp_threshold_20_unintended_diff_only": 0.010999992489814758, + "tpp_threshold_50_total_metric": 0.197999969124794, + "tpp_threshold_50_intended_diff_only": 0.20499998331069946, + "tpp_threshold_50_unintended_diff_only": 0.0070000141859054565, + "tpp_threshold_100_total_metric": 0.2735000252723694, + "tpp_threshold_100_intended_diff_only": 0.2850000262260437, + "tpp_threshold_100_unintended_diff_only": 0.011500000953674316, + "tpp_threshold_500_total_metric": 0.3395000100135803, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.03350001573562622 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-231933-tokens-950000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-231933-tokens-950000000/scr/saebench_scr_custom_sae_eval_results.json index 77cf60adc976003a48113749296be04a8acd1b8a..be35e52d7f2faed429ab1f1165ec4a74e59908d7 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-231933-tokens-950000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-231933-tokens-950000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "d0470eb0-5d97-4d7d-93f3-ec6aa1b0289d", - "datetime_epoch_millis": 1776885164730, + "eval_id": "59dc63a6-488a-4b00-88b6-92ffd0068930", + "datetime_epoch_millis": 1777316537790, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.28746353073212944, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.06195265936947533, "scr_metric_threshold_10": 0.2138710296363283, "scr_dir2_threshold_10": 0.20874631929499857, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.047000745989897424, + "scr_metric_threshold_20": 0.24324757819781082, + "scr_dir2_threshold_20": 0.23505808139056414, + "scr_dir1_threshold_50": 0.007289984864363129, + "scr_metric_threshold_50": 0.35559486332268514, + "scr_dir2_threshold_50": 0.34089950355370285, + "scr_dir1_threshold_100": -0.03458230235004653, + "scr_metric_threshold_100": 0.3201833541088444, + "scr_dir2_threshold_100": 0.30790828805513276, + "scr_dir1_threshold_500": -1.0669637122741167, + "scr_metric_threshold_500": 0.18804673121676252, + "scr_dir2_threshold_500": 0.17188868827086867 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.21794894327854547, "scr_metric_threshold_10": 0.015543994245836091, "scr_dir2_threshold_10": 0.015543994245836091, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2820514388024664, + "scr_metric_threshold_20": 0.08808283994795557, + "scr_dir2_threshold_20": 0.08808283994795557, + "scr_dir1_threshold_50": 0.03846180297916207, + "scr_metric_threshold_50": 0.24870468001419394, + "scr_dir2_threshold_50": 0.24870468001419394, + "scr_dir1_threshold_100": 0.37179539103317, + "scr_metric_threshold_100": 0.26683931423164214, + "scr_dir2_threshold_100": 0.26683931423164214, + "scr_dir1_threshold_500": -2.6410249552392093, + "scr_metric_threshold_500": -0.1398964114610147, + "scr_dir2_threshold_500": -0.1398964114610147 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": -0.07575748681586188, "scr_metric_threshold_10": 0.32721708301814567, "scr_dir2_threshold_10": 0.32721708301814567, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.06818182844432363, + "scr_metric_threshold_20": 0.11926615203740328, + "scr_dir2_threshold_20": 0.11926615203740328, + "scr_dir1_threshold_50": 0.5757577125909817, + "scr_metric_threshold_50": 0.35779827383502094, + "scr_dir2_threshold_50": 0.35779827383502094, + "scr_dir1_threshold_100": 0.03787874340793094, + "scr_metric_threshold_100": 0.535168196276077, + "scr_dir2_threshold_100": 0.535168196276077, + "scr_dir1_threshold_500": -1.9848482317066838, + "scr_metric_threshold_500": 0.10703371216609098, + "scr_dir2_threshold_500": 0.10703371216609098 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": -0.10937450523496334, "scr_metric_threshold_10": 0.10552761109862276, "scr_dir2_threshold_10": 0.10552761109862276, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.21874994179234863, + "scr_metric_threshold_20": 0.19597989197183965, + "scr_dir2_threshold_20": 0.19597989197183965, + "scr_dir1_threshold_50": -0.9062502910382568, + "scr_metric_threshold_50": 0.16331655168313458, + "scr_dir2_threshold_50": 0.16331655168313458, + "scr_dir1_threshold_100": -0.5156247962732202, + "scr_metric_threshold_100": 0.12060309108441948, + "scr_dir2_threshold_100": 0.12060309108441948, + "scr_dir1_threshold_500": -3.4062493597158348, + "scr_metric_threshold_500": -0.012562800147903362, + "scr_dir2_threshold_500": -0.012562800147903362 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": -0.47651036780868905, "scr_metric_threshold_10": 0.08598729016413297, "scr_dir2_threshold_10": 0.08598729016413297, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -1.2080538281985995, + "scr_metric_threshold_20": 0.08598729016413297, + "scr_dir2_threshold_20": 0.08598729016413297, + "scr_dir1_threshold_50": -0.8456379597608613, + "scr_metric_threshold_50": 0.2324841127051662, + "scr_dir2_threshold_50": 0.2324841127051662, + "scr_dir1_threshold_100": -1.3288592510215798, + "scr_metric_threshold_100": -0.09235663469908957, + "scr_dir2_threshold_100": -0.09235663469908957, + "scr_dir1_threshold_500": -1.7114098899714179, + "scr_metric_threshold_500": -0.5382164468571459, + "scr_dir2_threshold_500": -0.5382164468571459 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.08947378822971379, "scr_metric_threshold_10": 0.4166666863771947, "scr_dir2_threshold_10": 0.4166666863771947, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.09473687842942387, + "scr_metric_threshold_20": 0.4999998817368318, + "scr_dir2_threshold_20": 0.4999998817368318, + "scr_dir1_threshold_50": -0.12105264313663487, + "scr_metric_threshold_50": 0.6150794054390177, + "scr_dir2_threshold_50": 0.6150794054390177, + "scr_dir1_threshold_100": -0.16842108235134678, + "scr_metric_threshold_100": 0.654761901946115, + "scr_dir2_threshold_100": 0.654761901946115, + "scr_dir1_threshold_500": -0.14736840784384586, + "scr_metric_threshold_500": 0.7261904902694248, + "scr_dir2_threshold_500": 0.7261904902694248 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.3661201278080144, "scr_metric_threshold_10": 0.14122136275770192, "scr_dir2_threshold_10": 0.14122136275770192, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3497268934249226, + "scr_metric_threshold_20": 0.1908397224425799, + "scr_dir2_threshold_20": 0.1908397224425799, + "scr_dir1_threshold_50": 0.4426228653879351, + "scr_metric_threshold_50": 0.3091603913067316, + "scr_dir2_threshold_50": 0.3091603913067316, + "scr_dir1_threshold_100": 0.4754099855711136, + "scr_metric_threshold_100": 0.35496177909383203, + "scr_dir2_threshold_100": 0.35496177909383203, + "scr_dir1_threshold_500": 0.3551913048859532, + "scr_metric_threshold_500": 0.4274808326722602, + "scr_dir2_threshold_500": 0.4274808326722602 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.3069766255570261, "scr_metric_threshold_10": 0.44206005948697535, "scr_dir2_threshold_10": 0.44206005948697535, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.46046493833553914, + "scr_metric_threshold_20": 0.5193133135043415, + "scr_dir2_threshold_20": 0.5193133135043415, + "scr_dir1_threshold_50": 0.5441858866203395, + "scr_metric_threshold_50": 0.587982986325977, + "scr_dir2_threshold_50": 0.587982986325977, + "scr_dir1_threshold_100": 0.5674416747849105, + "scr_metric_threshold_100": 0.43776826888911, + "scr_dir2_threshold_100": 0.43776826888911, + "scr_dir1_threshold_500": 0.5720930542025149, + "scr_metric_threshold_500": 0.506437685896798, + "scr_dir2_threshold_500": 0.506437685896798 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.17674414994201715, "scr_metric_threshold_10": 0.17674414994201715, "scr_dir2_threshold_10": 0.13574646721137937, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.24651153377740184, + "scr_metric_threshold_20": 0.24651153377740184, + "scr_dir2_threshold_20": 0.1809955593194283, + "scr_dir1_threshold_50": 0.3302325052722394, + "scr_metric_threshold_50": 0.3302325052722394, + "scr_dir2_threshold_50": 0.21266962712038115, + "scr_dir1_threshold_100": 0.2837209160486496, + "scr_metric_threshold_100": 0.2837209160486496, + "scr_dir2_threshold_100": 0.18552038761895642, + "scr_dir1_threshold_500": 0.42790678719559005, + "scr_metric_threshold_500": 0.42790678719559005, + "scr_dir2_threshold_500": 0.2986424436284392 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-231933-tokens-950000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-231933-tokens-950000000/tpp/saebench_tpp_custom_sae_eval_results.json index e65a155e4abf0b04c4f89c08d4036d4244246bae..b8344732bf2e1c3a7c84add239418c33c2c8c02c 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-231933-tokens-950000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-231933-tokens-950000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "e1c16378-1481-4085-bdc1-6d9c2c28dc1d", - "datetime_epoch_millis": 1776885330410, + "eval_id": "41086c4e-63d7-4953-8646-1ffdb6f54620", + "datetime_epoch_millis": 1777316814695, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.024399994313716887, - "tpp_threshold_2_intended_diff_only": 0.025899994373321533, - "tpp_threshold_2_unintended_diff_only": 0.0015000000596046448, - "tpp_threshold_5_total_metric": 0.0909250020980835, - "tpp_threshold_5_intended_diff_only": 0.1090000033378601, - "tpp_threshold_5_unintended_diff_only": 0.01807500123977661, - "tpp_threshold_10_total_metric": 0.1162249967455864, - "tpp_threshold_10_intended_diff_only": 0.15770000219345093, - "tpp_threshold_10_unintended_diff_only": 0.04147500544786453, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.022574988007545472, + "tpp_threshold_2_intended_diff_only": 0.025399994850158692, + "tpp_threshold_2_unintended_diff_only": 0.00282500684261322, + "tpp_threshold_5_total_metric": 0.09110000878572465, + "tpp_threshold_5_intended_diff_only": 0.10790001153945923, + "tpp_threshold_5_unintended_diff_only": 0.016800002753734587, + "tpp_threshold_10_total_metric": 0.11690000593662261, + "tpp_threshold_10_intended_diff_only": 0.15910000801086424, + "tpp_threshold_10_unintended_diff_only": 0.042200002074241634, + "tpp_threshold_20_total_metric": 0.14222500771284105, + "tpp_threshold_20_intended_diff_only": 0.20410001873970032, + "tpp_threshold_20_unintended_diff_only": 0.061875011026859286, + "tpp_threshold_50_total_metric": 0.15880000591278076, + "tpp_threshold_50_intended_diff_only": 0.2608000159263611, + "tpp_threshold_50_unintended_diff_only": 0.10200001001358033, + "tpp_threshold_100_total_metric": 0.17900000810623168, + "tpp_threshold_100_intended_diff_only": 0.3216000199317932, + "tpp_threshold_100_unintended_diff_only": 0.14260001182556153, + "tpp_threshold_500_total_metric": 0.23692501187324522, + "tpp_threshold_500_intended_diff_only": 0.43640002608299255, + "tpp_threshold_500_unintended_diff_only": 0.1994750142097473 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.039299994707107544, - "tpp_threshold_2_intended_diff_only": 0.042200005054473876, - "tpp_threshold_2_unintended_diff_only": 0.002900010347366333, - "tpp_threshold_5_total_metric": 0.16690001487731934, - "tpp_threshold_5_intended_diff_only": 0.20100002288818358, - "tpp_threshold_5_unintended_diff_only": 0.034100008010864255, - "tpp_threshold_10_total_metric": 0.206850004196167, + "tpp_threshold_2_total_metric": 0.03794998526573181, + "tpp_threshold_2_intended_diff_only": 0.04259999990463257, + "tpp_threshold_2_unintended_diff_only": 0.004650014638900757, + "tpp_threshold_5_total_metric": 0.1678500145673752, + "tpp_threshold_5_intended_diff_only": 0.20080002546310424, + "tpp_threshold_5_unintended_diff_only": 0.03295001089572906, + "tpp_threshold_10_total_metric": 0.20570000410079955, "tpp_threshold_10_intended_diff_only": 0.2876000165939331, - "tpp_threshold_10_unintended_diff_only": 0.08075001239776611, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_10_unintended_diff_only": 0.08190001249313354, + "tpp_threshold_20_total_metric": 0.23370001912117006, + "tpp_threshold_20_intended_diff_only": 0.3536000370979309, + "tpp_threshold_20_unintended_diff_only": 0.11990001797676086, + "tpp_threshold_50_total_metric": 0.21265000998973846, + "tpp_threshold_50_intended_diff_only": 0.410200035572052, + "tpp_threshold_50_unintended_diff_only": 0.19755002558231355, + "tpp_threshold_100_total_metric": 0.1687000125646591, + "tpp_threshold_100_intended_diff_only": 0.44060003757476807, + "tpp_threshold_100_unintended_diff_only": 0.27190002501010896, + "tpp_threshold_500_total_metric": 0.09975000619888305, + "tpp_threshold_500_intended_diff_only": 0.46720004081726074, + "tpp_threshold_500_unintended_diff_only": 0.36745003461837766 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.009499993920326234, - "tpp_threshold_2_intended_diff_only": 0.00959998369216919, - "tpp_threshold_2_unintended_diff_only": 9.998977184295654e-05, - "tpp_threshold_5_total_metric": 0.014949989318847657, - "tpp_threshold_5_intended_diff_only": 0.016999983787536622, - "tpp_threshold_5_unintended_diff_only": 0.0020499944686889647, - "tpp_threshold_10_total_metric": 0.025599989295005798, - "tpp_threshold_10_intended_diff_only": 0.02779998779296875, - "tpp_threshold_10_unintended_diff_only": 0.0021999984979629516, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.007199990749359131, + "tpp_threshold_2_intended_diff_only": 0.008199989795684814, + "tpp_threshold_2_unintended_diff_only": 0.0009999990463256836, + "tpp_threshold_5_total_metric": 0.014350003004074097, + "tpp_threshold_5_intended_diff_only": 0.01499999761581421, + "tpp_threshold_5_unintended_diff_only": 0.0006499946117401123, + "tpp_threshold_10_total_metric": 0.02810000777244568, + "tpp_threshold_10_intended_diff_only": 0.03059999942779541, + "tpp_threshold_10_unintended_diff_only": 0.0024999916553497315, + "tpp_threshold_20_total_metric": 0.050749996304512025, + "tpp_threshold_20_intended_diff_only": 0.05460000038146973, + "tpp_threshold_20_unintended_diff_only": 0.0038500040769577025, + "tpp_threshold_50_total_metric": 0.10495000183582306, + "tpp_threshold_50_intended_diff_only": 0.11139999628067017, + "tpp_threshold_50_unintended_diff_only": 0.006449994444847107, + "tpp_threshold_100_total_metric": 0.18930000364780425, + "tpp_threshold_100_intended_diff_only": 0.20260000228881836, + "tpp_threshold_100_unintended_diff_only": 0.0132999986410141, + "tpp_threshold_500_total_metric": 0.3741000175476074, + "tpp_threshold_500_intended_diff_only": 0.40560001134872437, + "tpp_threshold_500_unintended_diff_only": 0.03149999380111694 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.018750011920928955, "tpp_threshold_10_total_metric": 0.17674997448921204, "tpp_threshold_10_intended_diff_only": 0.19599997997283936, - "tpp_threshold_10_unintended_diff_only": 0.01925000548362732 + "tpp_threshold_10_unintended_diff_only": 0.01925000548362732, + "tpp_threshold_20_total_metric": 0.25324998795986176, + "tpp_threshold_20_intended_diff_only": 0.3460000157356262, + "tpp_threshold_20_unintended_diff_only": 0.09275002777576447, + "tpp_threshold_50_total_metric": 0.23749999701976776, + "tpp_threshold_50_intended_diff_only": 0.4190000295639038, + "tpp_threshold_50_unintended_diff_only": 0.18150003254413605, + "tpp_threshold_100_total_metric": 0.2082499861717224, + "tpp_threshold_100_intended_diff_only": 0.44300001859664917, + "tpp_threshold_100_unintended_diff_only": 0.23475003242492676, + "tpp_threshold_500_total_metric": 0.10849998891353607, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.3475000411272049 }, "1": { - "tpp_threshold_2_total_metric": -0.0002500265836715698, - "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983, - "tpp_threshold_5_total_metric": 0.1860000342130661, + "tpp_threshold_2_total_metric": 0.004499971866607666, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 0.000500023365020752, + "tpp_threshold_5_total_metric": 0.19050002098083496, "tpp_threshold_5_intended_diff_only": 0.22400003671646118, - "tpp_threshold_5_unintended_diff_only": 0.03800000250339508, - "tpp_threshold_10_total_metric": 0.21649998426437378, + "tpp_threshold_5_unintended_diff_only": 0.03350001573562622, + "tpp_threshold_10_total_metric": 0.21424998342990875, "tpp_threshold_10_intended_diff_only": 0.29600000381469727, - "tpp_threshold_10_unintended_diff_only": 0.07950001955032349 + "tpp_threshold_10_unintended_diff_only": 0.08175002038478851, + "tpp_threshold_20_total_metric": 0.24300000071525574, + "tpp_threshold_20_intended_diff_only": 0.32600003480911255, + "tpp_threshold_20_unintended_diff_only": 0.08300003409385681, + "tpp_threshold_50_total_metric": 0.21724997460842133, + "tpp_threshold_50_intended_diff_only": 0.375, + "tpp_threshold_50_unintended_diff_only": 0.15775002539157867, + "tpp_threshold_100_total_metric": 0.16199998557567596, + "tpp_threshold_100_intended_diff_only": 0.42100000381469727, + "tpp_threshold_100_unintended_diff_only": 0.2590000182390213, + "tpp_threshold_500_total_metric": 0.08475001156330109, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.37425003945827484 }, "2": { - "tpp_threshold_2_total_metric": 0.06874999403953552, - "tpp_threshold_2_intended_diff_only": 0.07300001382827759, - "tpp_threshold_2_unintended_diff_only": 0.004250019788742065, - "tpp_threshold_5_total_metric": 0.21925002336502075, - "tpp_threshold_5_intended_diff_only": 0.25700002908706665, - "tpp_threshold_5_unintended_diff_only": 0.0377500057220459, - "tpp_threshold_10_total_metric": 0.18250000476837158, - "tpp_threshold_10_intended_diff_only": 0.33100003004074097, - "tpp_threshold_10_unintended_diff_only": 0.14850002527236938 + "tpp_threshold_2_total_metric": 0.056000009179115295, + "tpp_threshold_2_intended_diff_only": 0.06800001859664917, + "tpp_threshold_2_unintended_diff_only": 0.012000009417533875, + "tpp_threshold_5_total_metric": 0.218500018119812, + "tpp_threshold_5_intended_diff_only": 0.2560000419616699, + "tpp_threshold_5_unintended_diff_only": 0.03750002384185791, + "tpp_threshold_10_total_metric": 0.17949999868869781, + "tpp_threshold_10_intended_diff_only": 0.3320000171661377, + "tpp_threshold_10_unintended_diff_only": 0.15250001847743988, + "tpp_threshold_20_total_metric": 0.1472500115633011, + "tpp_threshold_20_intended_diff_only": 0.36000001430511475, + "tpp_threshold_20_unintended_diff_only": 0.21275000274181366, + "tpp_threshold_50_total_metric": 0.11400002241134644, + "tpp_threshold_50_intended_diff_only": 0.3800000548362732, + "tpp_threshold_50_unintended_diff_only": 0.26600003242492676, + "tpp_threshold_100_total_metric": 0.06650002300739288, + "tpp_threshold_100_intended_diff_only": 0.40800005197525024, + "tpp_threshold_100_unintended_diff_only": 0.34150002896785736, + "tpp_threshold_500_total_metric": 0.05399999022483826, + "tpp_threshold_500_intended_diff_only": 0.4520000219345093, + "tpp_threshold_500_unintended_diff_only": 0.398000031709671 }, "6": { - "tpp_threshold_2_total_metric": 0.011750012636184692, + "tpp_threshold_2_total_metric": 0.009499981999397278, "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": -0.002750009298324585, - "tpp_threshold_5_total_metric": 0.1184999942779541, + "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704, + "tpp_threshold_5_total_metric": 0.12399999797344208, "tpp_threshold_5_intended_diff_only": 0.1340000033378601, - "tpp_threshold_5_unintended_diff_only": 0.015500009059906006, - "tpp_threshold_10_total_metric": 0.2645000219345093, + "tpp_threshold_5_unintended_diff_only": 0.01000000536441803, + "tpp_threshold_10_total_metric": 0.26775000989437103, "tpp_threshold_10_intended_diff_only": 0.3500000238418579, - "tpp_threshold_10_unintended_diff_only": 0.08550000190734863 + "tpp_threshold_10_unintended_diff_only": 0.08225001394748688, + "tpp_threshold_20_total_metric": 0.29575005173683167, + "tpp_threshold_20_intended_diff_only": 0.43000006675720215, + "tpp_threshold_20_unintended_diff_only": 0.13425001502037048, + "tpp_threshold_50_total_metric": 0.2575000077486038, + "tpp_threshold_50_intended_diff_only": 0.4750000238418579, + "tpp_threshold_50_unintended_diff_only": 0.2175000160932541, + "tpp_threshold_100_total_metric": 0.20125003159046173, + "tpp_threshold_100_intended_diff_only": 0.4830000400543213, + "tpp_threshold_100_unintended_diff_only": 0.28175000846385956, + "tpp_threshold_500_total_metric": 0.1290000081062317, + "tpp_threshold_500_intended_diff_only": 0.4930000305175781, + "tpp_threshold_500_unintended_diff_only": 0.36400002241134644 }, "9": { - "tpp_threshold_2_total_metric": 0.0050000399351119995, - "tpp_threshold_2_intended_diff_only": 0.006000041961669922, - "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_5_total_metric": 0.15750004351139069, + "tpp_threshold_2_total_metric": 0.008500009775161743, + "tpp_threshold_2_intended_diff_only": 0.009000003337860107, + "tpp_threshold_2_unintended_diff_only": 0.0004999935626983643, + "tpp_threshold_5_total_metric": 0.15300005674362183, "tpp_threshold_5_intended_diff_only": 0.21800005435943604, - "tpp_threshold_5_unintended_diff_only": 0.06050001084804535, - "tpp_threshold_10_total_metric": 0.1940000355243683, - "tpp_threshold_10_intended_diff_only": 0.26500004529953003, - "tpp_threshold_10_unintended_diff_only": 0.07100000977516174 + "tpp_threshold_5_unintended_diff_only": 0.06499999761581421, + "tpp_threshold_10_total_metric": 0.19025005400180817, + "tpp_threshold_10_intended_diff_only": 0.2640000581741333, + "tpp_threshold_10_unintended_diff_only": 0.07375000417232513, + "tpp_threshold_20_total_metric": 0.22925004363059998, + "tpp_threshold_20_intended_diff_only": 0.3060000538825989, + "tpp_threshold_20_unintended_diff_only": 0.0767500102519989, + "tpp_threshold_50_total_metric": 0.23700004816055298, + "tpp_threshold_50_intended_diff_only": 0.4020000696182251, + "tpp_threshold_50_unintended_diff_only": 0.16500002145767212, + "tpp_threshold_100_total_metric": 0.2055000364780426, + "tpp_threshold_100_intended_diff_only": 0.44800007343292236, + "tpp_threshold_100_unintended_diff_only": 0.24250003695487976, + "tpp_threshold_500_total_metric": 0.12250003218650818, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.35350003838539124 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.00875002145767212, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": -0.00475001335144043, - "tpp_threshold_5_total_metric": 0.010499969124794006, - "tpp_threshold_5_intended_diff_only": 0.006999969482421875, - "tpp_threshold_5_unintended_diff_only": -0.0034999996423721313, - "tpp_threshold_10_total_metric": 0.0007499903440475464, - "tpp_threshold_10_intended_diff_only": 0.004999995231628418, - "tpp_threshold_10_unintended_diff_only": 0.004250004887580872 + "tpp_threshold_2_total_metric": 0.002249985933303833, + "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_5_total_metric": 0.004250004887580872, + "tpp_threshold_5_intended_diff_only": 0.004999995231628418, + "tpp_threshold_5_unintended_diff_only": 0.0007499903440475464, + "tpp_threshold_10_total_metric": 0.00449998676776886, + "tpp_threshold_10_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_10_unintended_diff_only": 0.0014999955892562866, + "tpp_threshold_20_total_metric": 0.017999976873397827, + "tpp_threshold_20_intended_diff_only": 0.019999980926513672, + "tpp_threshold_20_unintended_diff_only": 0.0020000040531158447, + "tpp_threshold_50_total_metric": 0.06425002217292786, + "tpp_threshold_50_intended_diff_only": 0.0690000057220459, + "tpp_threshold_50_unintended_diff_only": 0.004749983549118042, + "tpp_threshold_100_total_metric": 0.13375000655651093, + "tpp_threshold_100_intended_diff_only": 0.14300000667572021, + "tpp_threshold_100_unintended_diff_only": 0.00925000011920929, + "tpp_threshold_500_total_metric": 0.40700003504753113, + "tpp_threshold_500_intended_diff_only": 0.4230000376701355, + "tpp_threshold_500_unintended_diff_only": 0.01600000262260437 }, "2": { - "tpp_threshold_2_total_metric": 0.002750024199485779, - "tpp_threshold_2_intended_diff_only": 0.008000016212463379, - "tpp_threshold_2_unintended_diff_only": 0.0052499920129776, - "tpp_threshold_5_total_metric": 0.006750017404556274, - "tpp_threshold_5_intended_diff_only": 0.013999998569488525, - "tpp_threshold_5_unintended_diff_only": 0.007249981164932251, - "tpp_threshold_10_total_metric": 0.03975000977516174, - "tpp_threshold_10_intended_diff_only": 0.04100000858306885, - "tpp_threshold_10_unintended_diff_only": 0.0012499988079071045 + "tpp_threshold_2_total_metric": 0.012000054121017456, + "tpp_threshold_2_intended_diff_only": 0.010000050067901611, + "tpp_threshold_2_unintended_diff_only": -0.0020000040531158447, + "tpp_threshold_5_total_metric": 0.0025000572204589844, + "tpp_threshold_5_intended_diff_only": 0.010000050067901611, + "tpp_threshold_5_unintended_diff_only": 0.007499992847442627, + "tpp_threshold_10_total_metric": 0.04000005125999451, + "tpp_threshold_10_intended_diff_only": 0.04300004243850708, + "tpp_threshold_10_unintended_diff_only": 0.0029999911785125732, + "tpp_threshold_20_total_metric": 0.08475004136562347, + "tpp_threshold_20_intended_diff_only": 0.08900004625320435, + "tpp_threshold_20_unintended_diff_only": 0.004250004887580872, + "tpp_threshold_50_total_metric": 0.13425004482269287, + "tpp_threshold_50_intended_diff_only": 0.14500004053115845, + "tpp_threshold_50_unintended_diff_only": 0.010749995708465576, + "tpp_threshold_100_total_metric": 0.24800005555152893, + "tpp_threshold_100_intended_diff_only": 0.26500004529953003, + "tpp_threshold_100_unintended_diff_only": 0.0169999897480011, + "tpp_threshold_500_total_metric": 0.40800005197525024, + "tpp_threshold_500_intended_diff_only": 0.43800002336502075, + "tpp_threshold_500_unintended_diff_only": 0.029999971389770508 }, "3": { - "tpp_threshold_2_total_metric": 0.012500002980232239, - "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_2_unintended_diff_only": -0.006500020623207092, - "tpp_threshold_5_total_metric": 0.019000008702278137, - "tpp_threshold_5_intended_diff_only": 0.018000006675720215, - "tpp_threshold_5_unintended_diff_only": -0.0010000020265579224, - "tpp_threshold_10_total_metric": 0.01799999177455902, - "tpp_threshold_10_intended_diff_only": 0.019999980926513672, - "tpp_threshold_10_unintended_diff_only": 0.001999989151954651 + "tpp_threshold_2_total_metric": 0.0029999762773513794, + "tpp_threshold_2_intended_diff_only": 0.001999974250793457, + "tpp_threshold_2_unintended_diff_only": -0.0010000020265579224, + "tpp_threshold_5_total_metric": 0.016750037670135498, + "tpp_threshold_5_intended_diff_only": 0.012000024318695068, + "tpp_threshold_5_unintended_diff_only": -0.00475001335144043, + "tpp_threshold_10_total_metric": 0.019000038504600525, + "tpp_threshold_10_intended_diff_only": 0.017000019550323486, + "tpp_threshold_10_unintended_diff_only": -0.0020000189542770386, + "tpp_threshold_20_total_metric": 0.008249983191490173, + "tpp_threshold_20_intended_diff_only": 0.010999977588653564, + "tpp_threshold_20_unintended_diff_only": 0.002749994397163391, + "tpp_threshold_50_total_metric": 0.04150000214576721, + "tpp_threshold_50_intended_diff_only": 0.041999995708465576, + "tpp_threshold_50_unintended_diff_only": 0.0004999935626983643, + "tpp_threshold_100_total_metric": 0.10274998843669891, + "tpp_threshold_100_intended_diff_only": 0.11699998378753662, + "tpp_threshold_100_unintended_diff_only": 0.014249995350837708, + "tpp_threshold_500_total_metric": 0.35099999606609344, + "tpp_threshold_500_intended_diff_only": 0.38499999046325684, + "tpp_threshold_500_unintended_diff_only": 0.03399999439716339 }, "5": { - "tpp_threshold_2_total_metric": -0.00850003957748413, + "tpp_threshold_2_total_metric": -0.0012500584125518799, "tpp_threshold_2_intended_diff_only": -0.001000046730041504, - "tpp_threshold_2_unintended_diff_only": 0.007499992847442627, - "tpp_threshold_5_total_metric": 0.001999959349632263, - "tpp_threshold_5_intended_diff_only": 0.0029999613761901855, - "tpp_threshold_5_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_10_total_metric": 0.0054999589920043945, - "tpp_threshold_10_intended_diff_only": 0.007999956607818604, - "tpp_threshold_10_unintended_diff_only": 0.002499997615814209 + "tpp_threshold_2_unintended_diff_only": 0.000250011682510376, + "tpp_threshold_5_total_metric": 0.009249955415725708, + "tpp_threshold_5_intended_diff_only": 0.007999956607818604, + "tpp_threshold_5_unintended_diff_only": -0.0012499988079071045, + "tpp_threshold_10_total_metric": 0.004250004887580872, + "tpp_threshold_10_intended_diff_only": 0.009999990463256836, + "tpp_threshold_10_unintended_diff_only": 0.005749985575675964, + "tpp_threshold_20_total_metric": 0.02724999189376831, + "tpp_threshold_20_intended_diff_only": 0.02799999713897705, + "tpp_threshold_20_unintended_diff_only": 0.0007500052452087402, + "tpp_threshold_50_total_metric": 0.0849999487400055, + "tpp_threshold_50_intended_diff_only": 0.09199994802474976, + "tpp_threshold_50_unintended_diff_only": 0.006999999284744263, + "tpp_threshold_100_total_metric": 0.18324998021125793, + "tpp_threshold_100_intended_diff_only": 0.19499999284744263, + "tpp_threshold_100_unintended_diff_only": 0.011750012636184692, + "tpp_threshold_500_total_metric": 0.3662499785423279, + "tpp_threshold_500_intended_diff_only": 0.4089999794960022, + "tpp_threshold_500_unintended_diff_only": 0.042750000953674316 }, "6": { - "tpp_threshold_2_total_metric": 0.03199996054172516, - "tpp_threshold_2_intended_diff_only": 0.030999958515167236, - "tpp_threshold_2_unintended_diff_only": -0.0010000020265579224, - "tpp_threshold_5_total_metric": 0.0364999920129776, - "tpp_threshold_5_intended_diff_only": 0.042999982833862305, - "tpp_threshold_5_unintended_diff_only": 0.006499990820884705, - "tpp_threshold_10_total_metric": 0.06399999558925629, - "tpp_threshold_10_intended_diff_only": 0.06499999761581421, - "tpp_threshold_10_unintended_diff_only": 0.0010000020265579224 + "tpp_threshold_2_total_metric": 0.019999995827674866, + "tpp_threshold_2_intended_diff_only": 0.02399998903274536, + "tpp_threshold_2_unintended_diff_only": 0.003999993205070496, + "tpp_threshold_5_total_metric": 0.03899995982646942, + "tpp_threshold_5_intended_diff_only": 0.039999961853027344, + "tpp_threshold_5_unintended_diff_only": 0.0010000020265579224, + "tpp_threshold_10_total_metric": 0.07274995744228363, + "tpp_threshold_10_intended_diff_only": 0.0769999623298645, + "tpp_threshold_10_unintended_diff_only": 0.004250004887580872, + "tpp_threshold_20_total_metric": 0.11549998819828033, + "tpp_threshold_20_intended_diff_only": 0.125, + "tpp_threshold_20_unintended_diff_only": 0.009500011801719666, + "tpp_threshold_50_total_metric": 0.19974999129772186, + "tpp_threshold_50_intended_diff_only": 0.20899999141693115, + "tpp_threshold_50_unintended_diff_only": 0.00925000011920929, + "tpp_threshold_100_total_metric": 0.2787499874830246, + "tpp_threshold_100_intended_diff_only": 0.2929999828338623, + "tpp_threshold_100_unintended_diff_only": 0.014249995350837708, + "tpp_threshold_500_total_metric": 0.3382500261068344, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.03474999964237213 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-2441-tokens-10000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-2441-tokens-10000000/scr/saebench_scr_custom_sae_eval_results.json index ba18b823b5fad63c10315d636f2cff7ecbf6c19f..f1753fe80c951c52479500d59806424d19b7d47f 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-2441-tokens-10000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-2441-tokens-10000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "c60d67bf-9707-4e04-b483-16786886d228", - "datetime_epoch_millis": 1776885285314, + "eval_id": "40add3df-7a9a-44e4-94f1-06a20761cddb", + "datetime_epoch_millis": 1777316682993, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.20086747439820368, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.2522650669526282, "scr_metric_threshold_10": 0.2776267778909501, "scr_dir2_threshold_10": 0.2736648780725306, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2373239502783135, + "scr_metric_threshold_20": 0.3074054764488834, + "scr_dir2_threshold_20": 0.2952566874534856, + "scr_dir1_threshold_50": 0.23638686186846655, + "scr_metric_threshold_50": 0.3400536817425774, + "scr_dir2_threshold_50": 0.3278259698209423, + "scr_dir1_threshold_100": -0.19920822841766905, + "scr_metric_threshold_100": 0.3790442844503187, + "scr_dir2_threshold_100": 0.36452255493089136, + "scr_dir1_threshold_500": 0.06468838438344704, + "scr_metric_threshold_500": 0.25111199801456385, + "scr_dir2_threshold_500": 0.22106621753024489 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.5256410746257707, "scr_metric_threshold_10": 0.1968911085011348, "scr_dir2_threshold_10": 0.1968911085011348, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.48718003580863245, + "scr_metric_threshold_20": 0.16839375998107475, + "scr_dir2_threshold_20": 0.16839375998107475, + "scr_dir1_threshold_50": 0.48718003580863245, + "scr_metric_threshold_50": 0.23575132573996999, + "scr_dir2_threshold_50": 0.23575132573996999, + "scr_dir1_threshold_100": -1.730768907469913, + "scr_metric_threshold_100": 0.25906739431680575, + "scr_dir2_threshold_100": 0.25906739431680575, + "scr_dir1_threshold_500": -1.1153838806134386, + "scr_metric_threshold_500": 0.23834196571158212, + "scr_dir2_threshold_500": 0.23834196571158212 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.1439393152601855, "scr_metric_threshold_10": 0.321100954221084, "scr_dir2_threshold_10": 0.321100954221084, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.18181805866811646, + "scr_metric_threshold_20": 0.28440363460714707, + "scr_dir2_threshold_20": 0.28440363460714707, + "scr_dir1_threshold_50": 0.2045454853329709, + "scr_metric_threshold_50": 0.42813466638717496, + "scr_dir2_threshold_50": 0.42813466638717496, + "scr_dir1_threshold_100": 0.22727291199782534, + "scr_metric_threshold_100": 0.47094811479817356, + "scr_dir2_threshold_100": 0.47094811479817356, + "scr_dir1_threshold_500": 0.1439393152601855, + "scr_metric_threshold_500": 0.07645270362640467, + "scr_dir2_threshold_500": 0.07645270362640467 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.2656252619344312, "scr_metric_threshold_10": 0.13567842130982535, "scr_dir2_threshold_10": 0.13567842130982535, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.09374970896174313, + "scr_metric_threshold_20": 0.17336682175353543, + "scr_dir2_threshold_20": 0.17336682175353543, + "scr_dir1_threshold_50": -0.28125005820765137, + "scr_metric_threshold_50": 0.16331655168313458, + "scr_dir2_threshold_50": 0.16331655168313458, + "scr_dir1_threshold_100": -1.4374998835846973, + "scr_metric_threshold_100": 0.0050252099153958625, + "scr_dir2_threshold_100": 0.0050252099153958625, + "scr_dir1_threshold_500": 0.2968748544808716, + "scr_metric_threshold_500": 0.027638280133700087, + "scr_dir2_threshold_500": 0.027638280133700087 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": 0.14765071675590696, "scr_metric_threshold_10": 0.19426747602431718, "scr_dir2_threshold_10": 0.19426747602431718, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.10738237582531437, + "scr_metric_threshold_20": 0.2675158872948338, + "scr_dir2_threshold_20": 0.2675158872948338, + "scr_dir1_threshold_50": 0.5704696966363376, + "scr_metric_threshold_50": 0.31847140286929915, + "scr_dir2_threshold_50": 0.31847140286929915, + "scr_dir1_threshold_100": 0.5704696966363376, + "scr_metric_threshold_100": 0.3630573840851048, + "scr_dir2_threshold_100": 0.3630573840851048, + "scr_dir1_threshold_500": 0.30201315702624804, + "scr_metric_threshold_500": 0.07006373900303832, + "scr_dir2_threshold_500": 0.07006373900303832 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.13157913724471565, "scr_metric_threshold_10": 0.567460267714699, "scr_dir2_threshold_10": 0.567460267714699, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.12105264313663487, + "scr_metric_threshold_20": 0.6269840124753451, + "scr_dir2_threshold_20": 0.6269840124753451, + "scr_dir1_threshold_50": -0.01052618039942014, + "scr_metric_threshold_50": 0.6825395548905486, + "scr_dir2_threshold_50": 0.6825395548905486, + "scr_dir1_threshold_100": -0.09473687842942387, + "scr_metric_threshold_100": 0.7341268949603097, + "scr_dir2_threshold_100": 0.7341268949603097, + "scr_dir1_threshold_500": 0.03684225881529179, + "scr_metric_threshold_500": 0.7539681432138584, + "scr_dir2_threshold_500": 0.7539681432138584 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.3060109503196829, "scr_metric_threshold_10": 0.2519083153249215, "scr_dir2_threshold_10": 0.2519083153249215, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2841529787670631, + "scr_metric_threshold_20": 0.2671755204201627, + "scr_dir2_threshold_20": 0.2671755204201627, + "scr_dir1_threshold_50": 0.1857922696345225, + "scr_metric_threshold_50": 0.2748092367170948, + "scr_dir2_threshold_50": 0.2748092367170948, + "scr_dir1_threshold_100": 0.2622950072144432, + "scr_metric_threshold_100": 0.4618319872618971, + "scr_dir2_threshold_100": 0.4618319872618971, + "scr_dir1_threshold_500": 0.2950818016891243, + "scr_metric_threshold_500": 0.31297713570588614, + "scr_dir2_threshold_500": 0.31297713570588614 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.330232413721597, "scr_metric_threshold_10": 0.3862660142728834, "scr_dir2_threshold_10": 0.3862660142728834, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3767442672816017, + "scr_metric_threshold_20": 0.4248926412815665, + "scr_dir2_threshold_20": 0.4248926412815665, + "scr_dir1_threshold_50": 0.46511631775314366, + "scr_metric_threshold_50": 0.3476393872642003, + "scr_dir2_threshold_50": 0.3476393872642003, + "scr_dir1_threshold_100": 0.330232413721597, + "scr_metric_threshold_100": 0.45922747769238426, + "scr_dir2_threshold_100": 0.45922747769238426, + "scr_dir1_threshold_500": 0.20465126852525933, + "scr_metric_threshold_500": 0.17596571683800658, + "scr_dir2_threshold_500": 0.17596571683800658 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.16744166575873534, "scr_metric_threshold_10": 0.16744166575873534, "scr_dir2_threshold_10": 0.13574646721137937, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.24651153377740184, + "scr_metric_threshold_20": 0.24651153377740184, + "scr_dir2_threshold_20": 0.1493212218142196, + "scr_dir1_threshold_50": 0.26976732838919676, + "scr_metric_threshold_50": 0.26976732838919676, + "scr_dir2_threshold_50": 0.17194563301611618, + "scr_dir1_threshold_100": 0.27906981257247854, + "scr_metric_threshold_100": 0.27906981257247854, + "scr_dir2_threshold_100": 0.16289597641705988, + "scr_dir1_threshold_500": 0.3534882998840343, + "scr_metric_threshold_500": 0.3534882998840343, + "scr_dir2_threshold_500": 0.1131220560094828 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-2441-tokens-10000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-2441-tokens-10000000/tpp/saebench_tpp_custom_sae_eval_results.json index 7c923a31b9e494936cd5394c09ce90f6d0e1b262..9b9d0aa481ead47e785349b463474f69cfac0ece 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-2441-tokens-10000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-2441-tokens-10000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "b12321ac-7c3b-43f3-a124-23afb3858b0f", - "datetime_epoch_millis": 1776885438722, + "eval_id": "a20cdede-f11b-47e8-a1c0-b653a0d431e7", + "datetime_epoch_millis": 1777317083537, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.008525006473064423, - "tpp_threshold_2_intended_diff_only": 0.008600008487701417, - "tpp_threshold_2_unintended_diff_only": 7.50020146369934e-05, - "tpp_threshold_5_total_metric": 0.01054999977350235, - "tpp_threshold_5_intended_diff_only": 0.012900006771087647, - "tpp_threshold_5_unintended_diff_only": 0.0023500069975852965, - "tpp_threshold_10_total_metric": 0.019874994456768033, - "tpp_threshold_10_intended_diff_only": 0.02279999852180481, - "tpp_threshold_10_unintended_diff_only": 0.002925004065036774, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.007074996829032898, + "tpp_threshold_2_intended_diff_only": 0.00819999575614929, + "tpp_threshold_2_unintended_diff_only": 0.001124998927116394, + "tpp_threshold_5_total_metric": 0.009175001084804535, + "tpp_threshold_5_intended_diff_only": 0.01040000319480896, + "tpp_threshold_5_unintended_diff_only": 0.001225002110004425, + "tpp_threshold_10_total_metric": 0.022575002908706666, + "tpp_threshold_10_intended_diff_only": 0.02590000629425049, + "tpp_threshold_10_unintended_diff_only": 0.003325003385543823, + "tpp_threshold_20_total_metric": 0.039849989116191864, + "tpp_threshold_20_intended_diff_only": 0.04269999861717224, + "tpp_threshold_20_unintended_diff_only": 0.002850009500980377, + "tpp_threshold_50_total_metric": 0.07912500053644181, + "tpp_threshold_50_intended_diff_only": 0.08320000767707825, + "tpp_threshold_50_unintended_diff_only": 0.004075007140636444, + "tpp_threshold_100_total_metric": 0.14174999743700029, + "tpp_threshold_100_intended_diff_only": 0.14919999837875364, + "tpp_threshold_100_unintended_diff_only": 0.007450000941753387, + "tpp_threshold_500_total_metric": 0.31735001504421234, + "tpp_threshold_500_intended_diff_only": 0.3318000197410583, + "tpp_threshold_500_unintended_diff_only": 0.014450004696846007 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.007350000739097595, - "tpp_threshold_2_intended_diff_only": 0.006600010395050049, - "tpp_threshold_2_unintended_diff_only": -0.0007499903440475464, - "tpp_threshold_5_total_metric": 0.006949999928474426, - "tpp_threshold_5_intended_diff_only": 0.00860002040863037, - "tpp_threshold_5_unintended_diff_only": 0.0016500204801559448, - "tpp_threshold_10_total_metric": 0.02139999270439148, - "tpp_threshold_10_intended_diff_only": 0.023600006103515626, - "tpp_threshold_10_unintended_diff_only": 0.0022000133991241454, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.0064999938011169435, + "tpp_threshold_2_intended_diff_only": 0.007400000095367431, + "tpp_threshold_2_unintended_diff_only": 0.0009000062942504883, + "tpp_threshold_5_total_metric": 0.006599998474121094, + "tpp_threshold_5_intended_diff_only": 0.007400012016296387, + "tpp_threshold_5_unintended_diff_only": 0.000800013542175293, + "tpp_threshold_10_total_metric": 0.02254999577999115, + "tpp_threshold_10_intended_diff_only": 0.024600005149841307, + "tpp_threshold_10_unintended_diff_only": 0.0020500093698501585, + "tpp_threshold_20_total_metric": 0.043299993872642516, + "tpp_threshold_20_intended_diff_only": 0.04560000896453857, + "tpp_threshold_20_unintended_diff_only": 0.002300015091896057, + "tpp_threshold_50_total_metric": 0.07815000116825103, + "tpp_threshold_50_intended_diff_only": 0.08140001296997071, + "tpp_threshold_50_unintended_diff_only": 0.0032500118017196656, + "tpp_threshold_100_total_metric": 0.16464999914169312, + "tpp_threshold_100_intended_diff_only": 0.1680000066757202, + "tpp_threshold_100_unintended_diff_only": 0.0033500075340270998, + "tpp_threshold_500_total_metric": 0.36230001747608187, + "tpp_threshold_500_intended_diff_only": 0.37480002641677856, + "tpp_threshold_500_unintended_diff_only": 0.012500008940696717 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.00970001220703125, - "tpp_threshold_2_intended_diff_only": 0.010600006580352784, - "tpp_threshold_2_unintended_diff_only": 0.0008999943733215332, - "tpp_threshold_5_total_metric": 0.014149999618530274, - "tpp_threshold_5_intended_diff_only": 0.017199993133544922, - "tpp_threshold_5_unintended_diff_only": 0.0030499935150146485, - "tpp_threshold_10_total_metric": 0.01834999620914459, - "tpp_threshold_10_intended_diff_only": 0.021999990940093993, - "tpp_threshold_10_unintended_diff_only": 0.003649994730949402, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.007649999856948852, + "tpp_threshold_2_intended_diff_only": 0.008999991416931152, + "tpp_threshold_2_unintended_diff_only": 0.0013499915599822997, + "tpp_threshold_5_total_metric": 0.011750003695487976, + "tpp_threshold_5_intended_diff_only": 0.013399994373321534, + "tpp_threshold_5_unintended_diff_only": 0.0016499906778335571, + "tpp_threshold_10_total_metric": 0.02260001003742218, + "tpp_threshold_10_intended_diff_only": 0.02720000743865967, + "tpp_threshold_10_unintended_diff_only": 0.0045999974012374874, + "tpp_threshold_20_total_metric": 0.03639998435974121, + "tpp_threshold_20_intended_diff_only": 0.03979998826980591, + "tpp_threshold_20_unintended_diff_only": 0.003400003910064697, + "tpp_threshold_50_total_metric": 0.08009999990463257, + "tpp_threshold_50_intended_diff_only": 0.08500000238418579, + "tpp_threshold_50_unintended_diff_only": 0.004900002479553222, + "tpp_threshold_100_total_metric": 0.11884999573230744, + "tpp_threshold_100_intended_diff_only": 0.1303999900817871, + "tpp_threshold_100_unintended_diff_only": 0.011549994349479675, + "tpp_threshold_500_total_metric": 0.2724000126123428, + "tpp_threshold_500_intended_diff_only": 0.28880001306533815, + "tpp_threshold_500_unintended_diff_only": 0.0164000004529953 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.0015000253915786743, "tpp_threshold_10_total_metric": 0.008749961853027344, "tpp_threshold_10_intended_diff_only": 0.010999977588653564, - "tpp_threshold_10_unintended_diff_only": 0.0022500157356262207 + "tpp_threshold_10_unintended_diff_only": 0.0022500157356262207, + "tpp_threshold_20_total_metric": 0.02774997055530548, + "tpp_threshold_20_intended_diff_only": 0.03299999237060547, + "tpp_threshold_20_unintended_diff_only": 0.005250021815299988, + "tpp_threshold_50_total_metric": 0.05949994921684265, + "tpp_threshold_50_intended_diff_only": 0.06699997186660767, + "tpp_threshold_50_unintended_diff_only": 0.007500022649765015, + "tpp_threshold_100_total_metric": 0.16950000822544098, + "tpp_threshold_100_intended_diff_only": 0.17500001192092896, + "tpp_threshold_100_unintended_diff_only": 0.005500003695487976, + "tpp_threshold_500_total_metric": 0.4129999876022339, + "tpp_threshold_500_intended_diff_only": 0.42100000381469727, + "tpp_threshold_500_unintended_diff_only": 0.008000016212463379 }, "1": { - "tpp_threshold_2_total_metric": -1.4901161193847656e-08, - "tpp_threshold_2_intended_diff_only": 0.0, - "tpp_threshold_2_unintended_diff_only": 1.4901161193847656e-08, - "tpp_threshold_5_total_metric": 0.002250000834465027, - "tpp_threshold_5_intended_diff_only": 0.003000020980834961, - "tpp_threshold_5_unintended_diff_only": 0.0007500201463699341, - "tpp_threshold_10_total_metric": -0.0022500157356262207, - "tpp_threshold_10_intended_diff_only": 0.0, - "tpp_threshold_10_unintended_diff_only": 0.0022500157356262207 + "tpp_threshold_2_total_metric": 0.0052499920129776, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": -0.00024999678134918213, + "tpp_threshold_5_total_metric": -0.003750026226043701, + "tpp_threshold_5_intended_diff_only": -0.003000020980834961, + "tpp_threshold_5_unintended_diff_only": 0.0007500052452087402, + "tpp_threshold_10_total_metric": 0.005249977111816406, + "tpp_threshold_10_intended_diff_only": 0.004999995231628418, + "tpp_threshold_10_unintended_diff_only": -0.0002499818801879883, + "tpp_threshold_20_total_metric": 0.0352499783039093, + "tpp_threshold_20_intended_diff_only": 0.03700000047683716, + "tpp_threshold_20_unintended_diff_only": 0.0017500221729278564, + "tpp_threshold_50_total_metric": 0.09299996495246887, + "tpp_threshold_50_intended_diff_only": 0.09799998998641968, + "tpp_threshold_50_unintended_diff_only": 0.005000025033950806, + "tpp_threshold_100_total_metric": 0.22074998915195465, + "tpp_threshold_100_intended_diff_only": 0.22699999809265137, + "tpp_threshold_100_unintended_diff_only": 0.006250008940696716, + "tpp_threshold_500_total_metric": 0.4177500158548355, + "tpp_threshold_500_intended_diff_only": 0.4230000376701355, + "tpp_threshold_500_unintended_diff_only": 0.005250021815299988 }, "2": { - "tpp_threshold_2_total_metric": 0.015250012278556824, - "tpp_threshold_2_intended_diff_only": 0.012000024318695068, - "tpp_threshold_2_unintended_diff_only": -0.0032499879598617554, - "tpp_threshold_5_total_metric": 0.012499988079071045, - "tpp_threshold_5_intended_diff_only": 0.013000011444091797, - "tpp_threshold_5_unintended_diff_only": 0.000500023365020752, - "tpp_threshold_10_total_metric": 0.016500011086463928, - "tpp_threshold_10_intended_diff_only": 0.017000019550323486, - "tpp_threshold_10_unintended_diff_only": 0.0005000084638595581 + "tpp_threshold_2_total_metric": 0.006000012159347534, + "tpp_threshold_2_intended_diff_only": 0.009000003337860107, + "tpp_threshold_2_unintended_diff_only": 0.0029999911785125732, + "tpp_threshold_5_total_metric": 0.009250029921531677, + "tpp_threshold_5_intended_diff_only": 0.01100003719329834, + "tpp_threshold_5_unintended_diff_only": 0.0017500072717666626, + "tpp_threshold_10_total_metric": 0.015499994158744812, + "tpp_threshold_10_intended_diff_only": 0.018000006675720215, + "tpp_threshold_10_unintended_diff_only": 0.002500012516975403, + "tpp_threshold_20_total_metric": 0.032500043511390686, + "tpp_threshold_20_intended_diff_only": 0.033000051975250244, + "tpp_threshold_20_unintended_diff_only": 0.0005000084638595581, + "tpp_threshold_50_total_metric": 0.05475005507469177, + "tpp_threshold_50_intended_diff_only": 0.05600005388259888, + "tpp_threshold_50_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_100_total_metric": 0.09600000083446503, + "tpp_threshold_100_intended_diff_only": 0.09600001573562622, + "tpp_threshold_100_unintended_diff_only": 1.4901161193847656e-08, + "tpp_threshold_500_total_metric": 0.31650005280971527, + "tpp_threshold_500_intended_diff_only": 0.35200005769729614, + "tpp_threshold_500_unintended_diff_only": 0.03550000488758087 }, "6": { - "tpp_threshold_2_total_metric": 0.0102500319480896, + "tpp_threshold_2_total_metric": 0.007750019431114197, "tpp_threshold_2_intended_diff_only": 0.00700002908706665, - "tpp_threshold_2_unintended_diff_only": -0.0032500028610229492, - "tpp_threshold_5_total_metric": 0.000250011682510376, - "tpp_threshold_5_intended_diff_only": 0.003000020980834961, - "tpp_threshold_5_unintended_diff_only": 0.002750009298324585, - "tpp_threshold_10_total_metric": 0.006499990820884705, - "tpp_threshold_10_intended_diff_only": 0.009000003337860107, - "tpp_threshold_10_unintended_diff_only": 0.002500012516975403 + "tpp_threshold_2_unintended_diff_only": -0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.006749987602233887, + "tpp_threshold_5_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_5_unintended_diff_only": -0.0027499794960021973, + "tpp_threshold_10_total_metric": 0.008999988436698914, + "tpp_threshold_10_intended_diff_only": 0.009999990463256836, + "tpp_threshold_10_unintended_diff_only": 0.0010000020265579224, + "tpp_threshold_20_total_metric": 0.01124998927116394, + "tpp_threshold_20_intended_diff_only": 0.009999990463256836, + "tpp_threshold_20_unintended_diff_only": -0.0012499988079071045, + "tpp_threshold_50_total_metric": 0.01600000262260437, + "tpp_threshold_50_intended_diff_only": 0.013999998569488525, + "tpp_threshold_50_unintended_diff_only": -0.0020000040531158447, + "tpp_threshold_100_total_metric": 0.01774999499320984, + "tpp_threshold_100_intended_diff_only": 0.018999993801116943, + "tpp_threshold_100_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_500_total_metric": 0.20600000023841858, + "tpp_threshold_500_intended_diff_only": 0.20800000429153442, + "tpp_threshold_500_unintended_diff_only": 0.0020000040531158447 }, "9": { - "tpp_threshold_2_total_metric": 0.006000027060508728, - "tpp_threshold_2_intended_diff_only": 0.00700002908706665, - "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_5_total_metric": 0.008250012993812561, - "tpp_threshold_5_intended_diff_only": 0.01100003719329834, - "tpp_threshold_5_unintended_diff_only": 0.002750024199485779, - "tpp_threshold_10_total_metric": 0.07750001549720764, - "tpp_threshold_10_intended_diff_only": 0.08100003004074097, - "tpp_threshold_10_unintended_diff_only": 0.003500014543533325 + "tpp_threshold_2_total_metric": 0.008249998092651367, + "tpp_threshold_2_intended_diff_only": 0.009000003337860107, + "tpp_threshold_2_unintended_diff_only": 0.0007500052452087402, + "tpp_threshold_5_total_metric": 0.009250015020370483, + "tpp_threshold_5_intended_diff_only": 0.012000024318695068, + "tpp_threshold_5_unintended_diff_only": 0.002750009298324585, + "tpp_threshold_10_total_metric": 0.07425005733966827, + "tpp_threshold_10_intended_diff_only": 0.07900005578994751, + "tpp_threshold_10_unintended_diff_only": 0.004749998450279236, + "tpp_threshold_20_total_metric": 0.10974998772144318, + "tpp_threshold_20_intended_diff_only": 0.11500000953674316, + "tpp_threshold_20_unintended_diff_only": 0.005250021815299988, + "tpp_threshold_50_total_metric": 0.16750003397464752, + "tpp_threshold_50_intended_diff_only": 0.17200005054473877, + "tpp_threshold_50_unintended_diff_only": 0.0045000165700912476, + "tpp_threshold_100_total_metric": 0.3192500025033951, + "tpp_threshold_100_intended_diff_only": 0.3230000138282776, + "tpp_threshold_100_unintended_diff_only": 0.0037500113248825073, + "tpp_threshold_500_total_metric": 0.458250030875206, + "tpp_threshold_500_intended_diff_only": 0.4700000286102295, + "tpp_threshold_500_unintended_diff_only": 0.011749997735023499 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.007500022649765015, - "tpp_threshold_2_intended_diff_only": 0.003000020980834961, - "tpp_threshold_2_unintended_diff_only": -0.004500001668930054, - "tpp_threshold_5_total_metric": 0.00599999725818634, + "tpp_threshold_2_total_metric": 0.0020000040531158447, + "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_2_unintended_diff_only": 0.003999978303909302, + "tpp_threshold_5_total_metric": 0.0007500052452087402, "tpp_threshold_5_intended_diff_only": 0.004999995231628418, - "tpp_threshold_5_unintended_diff_only": -0.0010000020265579224, - "tpp_threshold_10_total_metric": -0.003250017762184143, - "tpp_threshold_10_intended_diff_only": 0.001999974250793457, - "tpp_threshold_10_unintended_diff_only": 0.0052499920129776 + "tpp_threshold_5_unintended_diff_only": 0.004249989986419678, + "tpp_threshold_10_total_metric": -0.0007499754428863525, + "tpp_threshold_10_intended_diff_only": 0.003000020980834961, + "tpp_threshold_10_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_20_total_metric": 0.007250010967254639, + "tpp_threshold_20_intended_diff_only": 0.009000003337860107, + "tpp_threshold_20_unintended_diff_only": 0.0017499923706054688, + "tpp_threshold_50_total_metric": 0.027499988675117493, + "tpp_threshold_50_intended_diff_only": 0.02399998903274536, + "tpp_threshold_50_unintended_diff_only": -0.0034999996423721313, + "tpp_threshold_100_total_metric": 0.051249995827674866, + "tpp_threshold_100_intended_diff_only": 0.0559999942779541, + "tpp_threshold_100_unintended_diff_only": 0.004749998450279236, + "tpp_threshold_500_total_metric": 0.2067500203847885, + "tpp_threshold_500_intended_diff_only": 0.20600003004074097, + "tpp_threshold_500_unintended_diff_only": -0.0007499903440475464 }, "2": { - "tpp_threshold_2_total_metric": -0.0004999488592147827, - "tpp_threshold_2_intended_diff_only": 0.006000041961669922, - "tpp_threshold_2_unintended_diff_only": 0.006499990820884705, - "tpp_threshold_5_total_metric": 0.002750024199485779, - "tpp_threshold_5_intended_diff_only": 0.009000003337860107, - "tpp_threshold_5_unintended_diff_only": 0.006249979138374329, - "tpp_threshold_10_total_metric": 0.010500013828277588, - "tpp_threshold_10_intended_diff_only": 0.013000011444091797, - "tpp_threshold_10_unintended_diff_only": 0.002499997615814209 + "tpp_threshold_2_total_metric": 0.00850003957748413, + "tpp_threshold_2_intended_diff_only": 0.00700002908706665, + "tpp_threshold_2_unintended_diff_only": -0.0015000104904174805, + "tpp_threshold_5_total_metric": -0.0009999722242355347, + "tpp_threshold_5_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_5_unintended_diff_only": 0.004999980330467224, + "tpp_threshold_10_total_metric": 0.013750061392784119, + "tpp_threshold_10_intended_diff_only": 0.01900005340576172, + "tpp_threshold_10_unintended_diff_only": 0.0052499920129776, + "tpp_threshold_20_total_metric": 0.025499999523162842, + "tpp_threshold_20_intended_diff_only": 0.02799999713897705, + "tpp_threshold_20_unintended_diff_only": 0.002499997615814209, + "tpp_threshold_50_total_metric": 0.06825001537799835, + "tpp_threshold_50_intended_diff_only": 0.07700002193450928, + "tpp_threshold_50_unintended_diff_only": 0.008750006556510925, + "tpp_threshold_100_total_metric": 0.10425005853176117, + "tpp_threshold_100_intended_diff_only": 0.12200003862380981, + "tpp_threshold_100_unintended_diff_only": 0.017749980092048645, + "tpp_threshold_500_total_metric": 0.25950005650520325, + "tpp_threshold_500_intended_diff_only": 0.27500003576278687, + "tpp_threshold_500_unintended_diff_only": 0.015499979257583618 }, "3": { - "tpp_threshold_2_total_metric": 0.018750011920928955, - "tpp_threshold_2_intended_diff_only": 0.013999998569488525, - "tpp_threshold_2_unintended_diff_only": -0.00475001335144043, - "tpp_threshold_5_total_metric": 0.016000017523765564, - "tpp_threshold_5_intended_diff_only": 0.018000006675720215, - "tpp_threshold_5_unintended_diff_only": 0.001999989151954651, - "tpp_threshold_10_total_metric": 0.013999998569488525, + "tpp_threshold_2_total_metric": 0.009249985218048096, + "tpp_threshold_2_intended_diff_only": 0.010999977588653564, + "tpp_threshold_2_unintended_diff_only": 0.0017499923706054688, + "tpp_threshold_5_total_metric": 0.006749987602233887, + "tpp_threshold_5_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_5_unintended_diff_only": -0.0007500052452087402, + "tpp_threshold_10_total_metric": 0.016749978065490723, "tpp_threshold_10_intended_diff_only": 0.018999993801116943, - "tpp_threshold_10_unintended_diff_only": 0.004999995231628418 + "tpp_threshold_10_unintended_diff_only": 0.0022500157356262207, + "tpp_threshold_20_total_metric": 0.010499969124794006, + "tpp_threshold_20_intended_diff_only": 0.014999985694885254, + "tpp_threshold_20_unintended_diff_only": 0.0045000165700912476, + "tpp_threshold_50_total_metric": 0.054000020027160645, + "tpp_threshold_50_intended_diff_only": 0.054000020027160645, + "tpp_threshold_50_unintended_diff_only": 0.0, + "tpp_threshold_100_total_metric": 0.08474999666213989, + "tpp_threshold_100_intended_diff_only": 0.09399998188018799, + "tpp_threshold_100_unintended_diff_only": 0.009249985218048096, + "tpp_threshold_500_total_metric": 0.2877500206232071, + "tpp_threshold_500_intended_diff_only": 0.30900001525878906, + "tpp_threshold_500_unintended_diff_only": 0.02124999463558197 }, "5": { - "tpp_threshold_2_total_metric": -0.0025000274181365967, - "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_2_unintended_diff_only": 0.008500009775161743, - "tpp_threshold_5_total_metric": 0.008000001311302185, - "tpp_threshold_5_intended_diff_only": 0.009999990463256836, - "tpp_threshold_5_unintended_diff_only": 0.001999989151954651, - "tpp_threshold_10_total_metric": 0.01649998128414154, - "tpp_threshold_10_intended_diff_only": 0.019999980926513672, - "tpp_threshold_10_unintended_diff_only": 0.0034999996423721313 + "tpp_threshold_2_total_metric": 0.005249977111816406, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": -0.0002499818801879883, + "tpp_threshold_5_total_metric": 0.013999998569488525, + "tpp_threshold_5_intended_diff_only": 0.013999998569488525, + "tpp_threshold_5_unintended_diff_only": 0.0, + "tpp_threshold_10_total_metric": 0.017499983310699463, + "tpp_threshold_10_intended_diff_only": 0.02499997615814209, + "tpp_threshold_10_unintended_diff_only": 0.007499992847442627, + "tpp_threshold_20_total_metric": 0.027249976992607117, + "tpp_threshold_20_intended_diff_only": 0.02899998426437378, + "tpp_threshold_20_unintended_diff_only": 0.0017500072717666626, + "tpp_threshold_50_total_metric": 0.037249982357025146, + "tpp_threshold_50_intended_diff_only": 0.042999982833862305, + "tpp_threshold_50_unintended_diff_only": 0.005750000476837158, + "tpp_threshold_100_total_metric": 0.088749960064888, + "tpp_threshold_100_intended_diff_only": 0.09999996423721313, + "tpp_threshold_100_unintended_diff_only": 0.011250004172325134, + "tpp_threshold_500_total_metric": 0.26224999129772186, + "tpp_threshold_500_intended_diff_only": 0.28200000524520874, + "tpp_threshold_500_unintended_diff_only": 0.019750013947486877 }, "6": { - "tpp_threshold_2_total_metric": 0.02525000274181366, - "tpp_threshold_2_intended_diff_only": 0.02399998903274536, - "tpp_threshold_2_unintended_diff_only": -0.0012500137090682983, - "tpp_threshold_5_total_metric": 0.0379999577999115, - "tpp_threshold_5_intended_diff_only": 0.04399996995925903, - "tpp_threshold_5_unintended_diff_only": 0.006000012159347534, - "tpp_threshold_10_total_metric": 0.05400000512599945, - "tpp_threshold_10_intended_diff_only": 0.0559999942779541, - "tpp_threshold_10_unintended_diff_only": 0.001999989151954651 + "tpp_threshold_2_total_metric": 0.013249993324279785, + "tpp_threshold_2_intended_diff_only": 0.015999972820281982, + "tpp_threshold_2_unintended_diff_only": 0.0027499794960021973, + "tpp_threshold_5_total_metric": 0.03824999928474426, + "tpp_threshold_5_intended_diff_only": 0.03799998760223389, + "tpp_threshold_5_unintended_diff_only": -0.000250011682510376, + "tpp_threshold_10_total_metric": 0.06575000286102295, + "tpp_threshold_10_intended_diff_only": 0.06999999284744263, + "tpp_threshold_10_unintended_diff_only": 0.004249989986419678, + "tpp_threshold_20_total_metric": 0.11149996519088745, + "tpp_threshold_20_intended_diff_only": 0.11799997091293335, + "tpp_threshold_20_unintended_diff_only": 0.0065000057220458984, + "tpp_threshold_50_total_metric": 0.2134999930858612, + "tpp_threshold_50_intended_diff_only": 0.22699999809265137, + "tpp_threshold_50_unintended_diff_only": 0.013500005006790161, + "tpp_threshold_100_total_metric": 0.26524996757507324, + "tpp_threshold_100_intended_diff_only": 0.2799999713897705, + "tpp_threshold_100_unintended_diff_only": 0.014750003814697266, + "tpp_threshold_500_total_metric": 0.34574997425079346, + "tpp_threshold_500_intended_diff_only": 0.37199997901916504, + "tpp_threshold_500_unintended_diff_only": 0.026250004768371582 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-24414-tokens-100000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-24414-tokens-100000000/scr/saebench_scr_custom_sae_eval_results.json index c865ca0b8249f9c520e09f342b7f806a39f7c62a..639937f87a8a27dd22550e7d6b73d9cd822e454d 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-24414-tokens-100000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-24414-tokens-100000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "6983652a-37fc-44b7-b030-946aed57cff6", - "datetime_epoch_millis": 1776885496694, + "eval_id": "c9a879e4-cf40-4841-b84b-c5f618d46953", + "datetime_epoch_millis": 1777316667993, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.21195278907653772, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.2732106130356535, "scr_metric_threshold_10": 0.2643277530959982, "scr_dir2_threshold_10": 0.27417727552003496, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.254990915046506, + "scr_metric_threshold_20": 0.3350846355160032, + "scr_dir2_threshold_20": 0.3471492516707594, + "scr_dir1_threshold_50": 0.1993347158036075, + "scr_metric_threshold_50": 0.41213895336215817, + "scr_dir2_threshold_50": 0.41827912914356763, + "scr_dir1_threshold_100": 0.23447883885121396, + "scr_metric_threshold_100": 0.43812244114237103, + "scr_dir2_threshold_100": 0.45387801191240573, + "scr_dir1_threshold_500": 0.008625756773020434, + "scr_metric_threshold_500": 0.330668632184563, + "scr_dir2_threshold_500": 0.34271482165810296 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.6153850268564743, "scr_metric_threshold_10": 0.12176162282740319, "scr_dir2_threshold_10": 0.12176162282740319, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.6153850268564743, + "scr_metric_threshold_20": 0.21761653710635845, + "scr_dir2_threshold_20": 0.21761653710635845, + "scr_dir1_threshold_50": 0.5897435701496917, + "scr_metric_threshold_50": 0.26683931423164214, + "scr_dir2_threshold_50": 0.26683931423164214, + "scr_dir1_threshold_100": 0.5384614208981502, + "scr_metric_threshold_100": 0.27461138856264183, + "scr_dir2_threshold_100": 0.27461138856264183, + "scr_dir1_threshold_500": 0.24359039998532808, + "scr_metric_threshold_500": 0.28497410286525365, + "scr_dir2_threshold_500": 0.28497410286525365 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.32575782547854165, "scr_metric_threshold_10": 0.4128439798401429, "scr_dir2_threshold_10": 0.4128439798401429, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3484848005931564, + "scr_metric_threshold_20": 0.47706424359523525, + "scr_dir2_threshold_20": 0.47706424359523525, + "scr_dir1_threshold_50": 0.2045454853329709, + "scr_metric_threshold_50": 0.5412845073503276, + "scr_dir2_threshold_50": 0.5412845073503276, + "scr_dir1_threshold_100": 0.18939416858989438, + "scr_metric_threshold_100": 0.7064219899200714, + "scr_dir2_threshold_100": 0.7064219899200714, + "scr_dir1_threshold_500": -0.09090880355893836, + "scr_metric_threshold_500": 0.27522944141155453, + "scr_dir2_threshold_500": 0.27522944141155453 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.5156247962732202, "scr_metric_threshold_10": 0.10301508102112025, "scr_dir2_threshold_10": 0.10301508102112025, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.42187508731147705, + "scr_metric_threshold_20": 0.15829149152812957, + "scr_dir2_threshold_20": 0.15829149152812957, + "scr_dir1_threshold_50": 0.14062502910382568, + "scr_metric_threshold_50": 0.24874377240134646, + "scr_dir2_threshold_50": 0.24874377240134646, + "scr_dir1_threshold_100": 0.3593749708961743, + "scr_metric_threshold_100": 0.04522614043660845, + "scr_dir2_threshold_100": 0.04522614043660845, + "scr_dir1_threshold_500": -0.42187508731147705, + "scr_metric_threshold_500": 0.03517587036620758, + "scr_dir2_threshold_500": 0.03517587036620758 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": 0.22147647516506289, "scr_metric_threshold_10": 0.31847140286929915, "scr_dir2_threshold_10": 0.31847140286929915, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.053690987897055974, + "scr_metric_threshold_20": 0.4267515887294834, + "scr_dir2_threshold_20": 0.4267515887294834, + "scr_dir1_threshold_50": 0.03355701744736089, + "scr_metric_threshold_50": 0.5541401878419436, + "scr_dir2_threshold_50": 0.5541401878419436, + "scr_dir1_threshold_100": 0.25503349261242375, + "scr_metric_threshold_100": 0.6369428057385983, + "scr_dir2_threshold_100": 0.6369428057385983, + "scr_dir1_threshold_500": -0.38926196243306976, + "scr_metric_threshold_500": 0.14649682254103324, + "scr_dir2_threshold_500": 0.14649682254103324 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.08421069803000372, "scr_metric_threshold_10": 0.4999998817368318, "scr_dir2_threshold_10": 0.4999998817368318, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.08947378822971379, + "scr_metric_threshold_20": 0.5992063595309115, + "scr_dir2_threshold_20": 0.5992063595309115, + "scr_dir1_threshold_50": -0.12631573333634494, + "scr_metric_threshold_50": 0.6865079937623274, + "scr_dir2_threshold_50": 0.6865079937623274, + "scr_dir1_threshold_100": -0.12105264313663487, + "scr_metric_threshold_100": 0.6904761961077699, + "scr_dir2_threshold_100": 0.6904761961077699, + "scr_dir1_threshold_500": -0.06842111372221286, + "scr_metric_threshold_500": 0.6190476077844602, + "scr_dir2_threshold_500": 0.6190476077844602 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.21857906410920355, "scr_metric_threshold_10": 0.11450392446499717, "scr_dir2_threshold_10": 0.11450392446499717, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.245901447122854, + "scr_metric_threshold_20": 0.17938926174649328, + "scr_dir2_threshold_20": 0.17938926174649328, + "scr_dir1_threshold_50": 0.3060109503196829, + "scr_metric_threshold_50": 0.25954203162185363, + "scr_dir2_threshold_50": 0.25954203162185363, + "scr_dir1_threshold_100": 0.2732241558450018, + "scr_metric_threshold_100": 0.38549618928431434, + "scr_dir2_threshold_100": 0.38549618928431434, + "scr_dir1_threshold_500": 0.4098360709132541, + "scr_metric_threshold_500": 0.47709919235713827, + "scr_dir2_threshold_500": 0.47709919235713827 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.10697673644937127, "scr_metric_threshold_10": 0.4463518500848407, "scr_dir2_threshold_10": 0.4463518500848407, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.15348831277851305, + "scr_metric_threshold_20": 0.5107297323086109, + "scr_dir2_threshold_20": 0.5107297323086109, + "scr_dir1_threshold_50": 0.25581394704114274, + "scr_metric_threshold_50": 0.549356359317294, + "scr_dir2_threshold_50": 0.549356359317294, + "scr_dir1_threshold_100": 0.19069768473417165, + "scr_metric_threshold_100": 0.5751073587184335, + "scr_dir2_threshold_100": 0.5751073587184335, + "scr_dir1_threshold_500": 0.10232563426262967, + "scr_metric_threshold_500": 0.5236051041022068, + "scr_dir2_threshold_500": 0.5236051041022068 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.09767428192335063, "scr_metric_threshold_10": 0.09767428192335063, "scr_dir2_threshold_10": 0.17647046131564434, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.11162786958280348, + "scr_metric_threshold_20": 0.11162786958280348, + "scr_dir2_threshold_20": 0.208144798820853, + "scr_dir1_threshold_50": 0.19069746037053023, + "scr_metric_threshold_50": 0.19069746037053023, + "scr_dir2_threshold_50": 0.23981886662180585, + "scr_dir1_threshold_100": 0.19069746037053023, + "scr_metric_threshold_100": 0.19069746037053023, + "scr_dir2_threshold_100": 0.31674202653080763, + "scr_dir1_threshold_500": 0.2837209160486496, + "scr_metric_threshold_500": 0.2837209160486496, + "scr_dir2_threshold_500": 0.38009043183696917 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-24414-tokens-100000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-24414-tokens-100000000/tpp/saebench_tpp_custom_sae_eval_results.json index c93daebd04c014eefc7821e586ec54891e9bba1a..53200cf81b1d535d9017b05436b33af5cba8dd31 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-24414-tokens-100000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-24414-tokens-100000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "5fca8cee-c948-46e9-a4d9-ffb5369182a7", - "datetime_epoch_millis": 1776885656359, + "eval_id": "96d34a9f-9a36-44b1-920b-b6eab8594ada", + "datetime_epoch_millis": 1777316957490, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.011775010824203493, - "tpp_threshold_2_intended_diff_only": 0.01190001368522644, - "tpp_threshold_2_unintended_diff_only": 0.0001250028610229492, - "tpp_threshold_5_total_metric": 0.016574998199939725, - "tpp_threshold_5_intended_diff_only": 0.019100004434585573, - "tpp_threshold_5_unintended_diff_only": 0.0025250062346458433, - "tpp_threshold_10_total_metric": 0.027750010788440707, - "tpp_threshold_10_intended_diff_only": 0.03140001893043518, - "tpp_threshold_10_unintended_diff_only": 0.0036500081419944762, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.010125000774860383, + "tpp_threshold_2_intended_diff_only": 0.011500006914138794, + "tpp_threshold_2_unintended_diff_only": 0.001375006139278412, + "tpp_threshold_5_total_metric": 0.015449994802474977, + "tpp_threshold_5_intended_diff_only": 0.01690000295639038, + "tpp_threshold_5_unintended_diff_only": 0.0014500081539154052, + "tpp_threshold_10_total_metric": 0.02767498642206192, + "tpp_threshold_10_intended_diff_only": 0.031699997186660764, + "tpp_threshold_10_unintended_diff_only": 0.004025010764598847, + "tpp_threshold_20_total_metric": 0.05050000697374344, + "tpp_threshold_20_intended_diff_only": 0.05460001230239868, + "tpp_threshold_20_unintended_diff_only": 0.004100005328655242, + "tpp_threshold_50_total_metric": 0.11895001530647278, + "tpp_threshold_50_intended_diff_only": 0.12460001111030579, + "tpp_threshold_50_unintended_diff_only": 0.005649995803833008, + "tpp_threshold_100_total_metric": 0.20270000994205473, + "tpp_threshold_100_intended_diff_only": 0.21150001287460327, + "tpp_threshold_100_unintended_diff_only": 0.008800002932548522, + "tpp_threshold_500_total_metric": 0.3938000306487084, + "tpp_threshold_500_intended_diff_only": 0.4196000337600708, + "tpp_threshold_500_unintended_diff_only": 0.025800003111362456 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.01425001621246338, - "tpp_threshold_2_intended_diff_only": 0.01380002498626709, - "tpp_threshold_2_unintended_diff_only": -0.0004499912261962891, - "tpp_threshold_5_total_metric": 0.017799994349479674, - "tpp_threshold_5_intended_diff_only": 0.02040001153945923, - "tpp_threshold_5_unintended_diff_only": 0.0026000171899795534, - "tpp_threshold_10_total_metric": 0.03145001232624054, - "tpp_threshold_10_intended_diff_only": 0.0350000262260437, - "tpp_threshold_10_unintended_diff_only": 0.0035500138998031615, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.012350007891654968, + "tpp_threshold_2_intended_diff_only": 0.013400018215179443, + "tpp_threshold_2_unintended_diff_only": 0.001050010323524475, + "tpp_threshold_5_total_metric": 0.018299993872642518, + "tpp_threshold_5_intended_diff_only": 0.01960000991821289, + "tpp_threshold_5_unintended_diff_only": 0.0013000160455703736, + "tpp_threshold_10_total_metric": 0.031249985098838806, + "tpp_threshold_10_intended_diff_only": 0.0346000075340271, + "tpp_threshold_10_unintended_diff_only": 0.0033500224351882936, + "tpp_threshold_20_total_metric": 0.05729999542236328, + "tpp_threshold_20_intended_diff_only": 0.06020001173019409, + "tpp_threshold_20_unintended_diff_only": 0.0029000163078308104, + "tpp_threshold_50_total_metric": 0.13355001509189607, + "tpp_threshold_50_intended_diff_only": 0.13940001726150514, + "tpp_threshold_50_unintended_diff_only": 0.00585000216960907, + "tpp_threshold_100_total_metric": 0.22655001580715178, + "tpp_threshold_100_intended_diff_only": 0.232200026512146, + "tpp_threshold_100_unintended_diff_only": 0.0056500107049942015, + "tpp_threshold_500_total_metric": 0.4307000309228897, + "tpp_threshold_500_intended_diff_only": 0.4522000432014465, + "tpp_threshold_500_unintended_diff_only": 0.021500012278556822 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.009300005435943604, - "tpp_threshold_2_intended_diff_only": 0.010000002384185792, - "tpp_threshold_2_unintended_diff_only": 0.0006999969482421875, - "tpp_threshold_5_total_metric": 0.01535000205039978, - "tpp_threshold_5_intended_diff_only": 0.017799997329711915, - "tpp_threshold_5_unintended_diff_only": 0.0024499952793121337, - "tpp_threshold_10_total_metric": 0.02405000925064087, - "tpp_threshold_10_intended_diff_only": 0.02780001163482666, - "tpp_threshold_10_unintended_diff_only": 0.003750002384185791, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.007899993658065796, + "tpp_threshold_2_intended_diff_only": 0.009599995613098145, + "tpp_threshold_2_unintended_diff_only": 0.0017000019550323486, + "tpp_threshold_5_total_metric": 0.012599995732307434, + "tpp_threshold_5_intended_diff_only": 0.01419999599456787, + "tpp_threshold_5_unintended_diff_only": 0.001600000262260437, + "tpp_threshold_10_total_metric": 0.024099987745285035, + "tpp_threshold_10_intended_diff_only": 0.028799986839294432, + "tpp_threshold_10_unintended_diff_only": 0.004699999094009399, + "tpp_threshold_20_total_metric": 0.0437000185251236, + "tpp_threshold_20_intended_diff_only": 0.049000012874603274, + "tpp_threshold_20_unintended_diff_only": 0.005299994349479675, + "tpp_threshold_50_total_metric": 0.1043500155210495, + "tpp_threshold_50_intended_diff_only": 0.10980000495910644, + "tpp_threshold_50_unintended_diff_only": 0.0054499894380569455, + "tpp_threshold_100_total_metric": 0.1788500040769577, + "tpp_threshold_100_intended_diff_only": 0.19079999923706054, + "tpp_threshold_100_unintended_diff_only": 0.011949995160102844, + "tpp_threshold_500_total_metric": 0.356900030374527, + "tpp_threshold_500_intended_diff_only": 0.38700002431869507, + "tpp_threshold_500_unintended_diff_only": 0.03009999394416809 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.0022500157356262207, "tpp_threshold_10_total_metric": 0.004999950528144836, "tpp_threshold_10_intended_diff_only": 0.006999969482421875, - "tpp_threshold_10_unintended_diff_only": 0.0020000189542770386 + "tpp_threshold_10_unintended_diff_only": 0.0020000189542770386, + "tpp_threshold_20_total_metric": 0.017499983310699463, + "tpp_threshold_20_intended_diff_only": 0.018000006675720215, + "tpp_threshold_20_unintended_diff_only": 0.000500023365020752, + "tpp_threshold_50_total_metric": 0.07524995505809784, + "tpp_threshold_50_intended_diff_only": 0.08099997043609619, + "tpp_threshold_50_unintended_diff_only": 0.005750015377998352, + "tpp_threshold_100_total_metric": 0.15574996173381805, + "tpp_threshold_100_intended_diff_only": 0.1589999794960022, + "tpp_threshold_100_unintended_diff_only": 0.003250017762184143, + "tpp_threshold_500_total_metric": 0.4347500056028366, + "tpp_threshold_500_intended_diff_only": 0.44700002670288086, + "tpp_threshold_500_unintended_diff_only": 0.01225002110004425 }, "1": { - "tpp_threshold_2_total_metric": 0.006000012159347534, - "tpp_threshold_2_intended_diff_only": 0.00700002908706665, - "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162, - "tpp_threshold_5_total_metric": 0.009500011801719666, - "tpp_threshold_5_intended_diff_only": 0.012000024318695068, - "tpp_threshold_5_unintended_diff_only": 0.002500012516975403, - "tpp_threshold_10_total_metric": 0.009749993681907654, - "tpp_threshold_10_intended_diff_only": 0.013000011444091797, - "tpp_threshold_10_unintended_diff_only": 0.003250017762184143 + "tpp_threshold_2_total_metric": 0.009499981999397278, + "tpp_threshold_2_intended_diff_only": 0.009999990463256836, + "tpp_threshold_2_unintended_diff_only": 0.0005000084638595581, + "tpp_threshold_5_total_metric": 0.006250008940696716, + "tpp_threshold_5_intended_diff_only": 0.00700002908706665, + "tpp_threshold_5_unintended_diff_only": 0.0007500201463699341, + "tpp_threshold_10_total_metric": 0.015249952673912048, + "tpp_threshold_10_intended_diff_only": 0.014999985694885254, + "tpp_threshold_10_unintended_diff_only": -0.00024996697902679443, + "tpp_threshold_20_total_metric": 0.024749964475631714, + "tpp_threshold_20_intended_diff_only": 0.02899998426437378, + "tpp_threshold_20_unintended_diff_only": 0.004250019788742065, + "tpp_threshold_50_total_metric": 0.11525003612041473, + "tpp_threshold_50_intended_diff_only": 0.12200003862380981, + "tpp_threshold_50_unintended_diff_only": 0.006750002503395081, + "tpp_threshold_100_total_metric": 0.2240000069141388, + "tpp_threshold_100_intended_diff_only": 0.23000001907348633, + "tpp_threshold_100_unintended_diff_only": 0.006000012159347534, + "tpp_threshold_500_total_metric": 0.434750035405159, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.02425001561641693 }, "2": { - "tpp_threshold_2_total_metric": 0.0467500239610672, - "tpp_threshold_2_intended_diff_only": 0.04400002956390381, - "tpp_threshold_2_unintended_diff_only": -0.002749994397163391, - "tpp_threshold_5_total_metric": 0.05849997699260712, - "tpp_threshold_5_intended_diff_only": 0.06000000238418579, - "tpp_threshold_5_unintended_diff_only": 0.0015000253915786743, - "tpp_threshold_10_total_metric": 0.08375003933906555, - "tpp_threshold_10_intended_diff_only": 0.08400005102157593, - "tpp_threshold_10_unintended_diff_only": 0.000250011682510376 + "tpp_threshold_2_total_metric": 0.03475002944469452, + "tpp_threshold_2_intended_diff_only": 0.03800004720687866, + "tpp_threshold_2_unintended_diff_only": 0.003250017762184143, + "tpp_threshold_5_total_metric": 0.05400003492832184, + "tpp_threshold_5_intended_diff_only": 0.05600005388259888, + "tpp_threshold_5_unintended_diff_only": 0.0020000189542770386, + "tpp_threshold_10_total_metric": 0.08175002038478851, + "tpp_threshold_10_intended_diff_only": 0.08500003814697266, + "tpp_threshold_10_unintended_diff_only": 0.003250017762184143, + "tpp_threshold_20_total_metric": 0.12000003457069397, + "tpp_threshold_20_intended_diff_only": 0.12100005149841309, + "tpp_threshold_20_unintended_diff_only": 0.0010000169277191162, + "tpp_threshold_50_total_metric": 0.20500002801418304, + "tpp_threshold_50_intended_diff_only": 0.2070000171661377, + "tpp_threshold_50_unintended_diff_only": 0.001999989151954651, + "tpp_threshold_100_total_metric": 0.2952500432729721, + "tpp_threshold_100_intended_diff_only": 0.29700005054473877, + "tpp_threshold_100_unintended_diff_only": 0.0017500072717666626, + "tpp_threshold_500_total_metric": 0.4150000214576721, + "tpp_threshold_500_intended_diff_only": 0.44300001859664917, + "tpp_threshold_500_unintended_diff_only": 0.02799999713897705 }, "6": { - "tpp_threshold_2_total_metric": 0.010000035166740417, + "tpp_threshold_2_total_metric": 0.007750019431114197, "tpp_threshold_2_intended_diff_only": 0.00700002908706665, - "tpp_threshold_2_unintended_diff_only": -0.003000006079673767, - "tpp_threshold_5_total_metric": 0.004249989986419678, - "tpp_threshold_5_intended_diff_only": 0.009000003337860107, - "tpp_threshold_5_unintended_diff_only": 0.00475001335144043, - "tpp_threshold_10_total_metric": 0.0102500319480896, - "tpp_threshold_10_intended_diff_only": 0.01500004529953003, - "tpp_threshold_10_unintended_diff_only": 0.00475001335144043 + "tpp_threshold_2_unintended_diff_only": -0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.010749980807304382, + "tpp_threshold_5_intended_diff_only": 0.009999990463256836, + "tpp_threshold_5_unintended_diff_only": -0.0007499903440475464, + "tpp_threshold_10_total_metric": 0.013750001788139343, + "tpp_threshold_10_intended_diff_only": 0.016000032424926758, + "tpp_threshold_10_unintended_diff_only": 0.0022500306367874146, + "tpp_threshold_20_total_metric": 0.01374998688697815, + "tpp_threshold_20_intended_diff_only": 0.013999998569488525, + "tpp_threshold_20_unintended_diff_only": 0.000250011682510376, + "tpp_threshold_50_total_metric": 0.03700004518032074, + "tpp_threshold_50_intended_diff_only": 0.03800004720687866, + "tpp_threshold_50_unintended_diff_only": 0.0010000020265579224, + "tpp_threshold_100_total_metric": 0.07475000619888306, + "tpp_threshold_100_intended_diff_only": 0.07700002193450928, + "tpp_threshold_100_unintended_diff_only": 0.0022500157356262207, + "tpp_threshold_500_total_metric": 0.42750002443790436, + "tpp_threshold_500_intended_diff_only": 0.4360000491142273, + "tpp_threshold_500_unintended_diff_only": 0.008500024676322937 }, "9": { - "tpp_threshold_2_total_metric": 0.00025004148483276367, - "tpp_threshold_2_intended_diff_only": 0.001000046730041504, - "tpp_threshold_2_unintended_diff_only": 0.0007500052452087402, - "tpp_threshold_5_total_metric": 0.008000031113624573, - "tpp_threshold_5_intended_diff_only": 0.010000050067901611, - "tpp_threshold_5_unintended_diff_only": 0.0020000189542770386, - "tpp_threshold_10_total_metric": 0.048500046133995056, - "tpp_threshold_10_intended_diff_only": 0.05600005388259888, - "tpp_threshold_10_unintended_diff_only": 0.007500007748603821 + "tpp_threshold_2_total_metric": 0.0015000402927398682, + "tpp_threshold_2_intended_diff_only": 0.0020000338554382324, + "tpp_threshold_2_unintended_diff_only": 0.0004999935626983643, + "tpp_threshold_5_total_metric": 0.011749982833862305, + "tpp_threshold_5_intended_diff_only": 0.013999998569488525, + "tpp_threshold_5_unintended_diff_only": 0.0022500157356262207, + "tpp_threshold_10_total_metric": 0.04050000011920929, + "tpp_threshold_10_intended_diff_only": 0.050000011920928955, + "tpp_threshold_10_unintended_diff_only": 0.009500011801719666, + "tpp_threshold_20_total_metric": 0.11050000786781311, + "tpp_threshold_20_intended_diff_only": 0.11900001764297485, + "tpp_threshold_20_unintended_diff_only": 0.008500009775161743, + "tpp_threshold_50_total_metric": 0.23525001108646393, + "tpp_threshold_50_intended_diff_only": 0.24900001287460327, + "tpp_threshold_50_unintended_diff_only": 0.013750001788139343, + "tpp_threshold_100_total_metric": 0.38300006091594696, + "tpp_threshold_100_intended_diff_only": 0.3980000615119934, + "tpp_threshold_100_unintended_diff_only": 0.015000000596046448, + "tpp_threshold_500_total_metric": 0.44150006771087646, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.03450000286102295 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.010249987244606018, - "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_2_unintended_diff_only": -0.004250004887580872, - "tpp_threshold_5_total_metric": 0.016500011086463928, - "tpp_threshold_5_intended_diff_only": 0.013000011444091797, - "tpp_threshold_5_unintended_diff_only": -0.0034999996423721313, - "tpp_threshold_10_total_metric": 0.00700002908706665, - "tpp_threshold_10_intended_diff_only": 0.012000024318695068, - "tpp_threshold_10_unintended_diff_only": 0.004999995231628418 + "tpp_threshold_2_total_metric": 0.004000023007392883, + "tpp_threshold_2_intended_diff_only": 0.008000016212463379, + "tpp_threshold_2_unintended_diff_only": 0.003999993205070496, + "tpp_threshold_5_total_metric": 0.008249998092651367, + "tpp_threshold_5_intended_diff_only": 0.009000003337860107, + "tpp_threshold_5_unintended_diff_only": 0.0007500052452087402, + "tpp_threshold_10_total_metric": 0.008999988436698914, + "tpp_threshold_10_intended_diff_only": 0.010999977588653564, + "tpp_threshold_10_unintended_diff_only": 0.001999989151954651, + "tpp_threshold_20_total_metric": 0.015750035643577576, + "tpp_threshold_20_intended_diff_only": 0.021000027656555176, + "tpp_threshold_20_unintended_diff_only": 0.0052499920129776, + "tpp_threshold_50_total_metric": 0.0805000513792038, + "tpp_threshold_50_intended_diff_only": 0.08100003004074097, + "tpp_threshold_50_unintended_diff_only": 0.0004999786615371704, + "tpp_threshold_100_total_metric": 0.13700000941753387, + "tpp_threshold_100_intended_diff_only": 0.14300000667572021, + "tpp_threshold_100_unintended_diff_only": 0.00599999725818634, + "tpp_threshold_500_total_metric": 0.3920000493526459, + "tpp_threshold_500_intended_diff_only": 0.39500004053115845, + "tpp_threshold_500_unintended_diff_only": 0.0029999911785125732 }, "2": { - "tpp_threshold_2_total_metric": 0.004000052809715271, - "tpp_threshold_2_intended_diff_only": 0.010000050067901611, - "tpp_threshold_2_unintended_diff_only": 0.00599999725818634, - "tpp_threshold_5_total_metric": 0.018000036478042603, - "tpp_threshold_5_intended_diff_only": 0.026000022888183594, - "tpp_threshold_5_unintended_diff_only": 0.007999986410140991, - "tpp_threshold_10_total_metric": 0.044250041246414185, - "tpp_threshold_10_intended_diff_only": 0.04700005054473877, - "tpp_threshold_10_unintended_diff_only": 0.002750009298324585 + "tpp_threshold_2_total_metric": 0.012500032782554626, + "tpp_threshold_2_intended_diff_only": 0.012000024318695068, + "tpp_threshold_2_unintended_diff_only": -0.0005000084638595581, + "tpp_threshold_5_total_metric": 0.013250023126602173, + "tpp_threshold_5_intended_diff_only": 0.022000014781951904, + "tpp_threshold_5_unintended_diff_only": 0.008749991655349731, + "tpp_threshold_10_total_metric": 0.03925003111362457, + "tpp_threshold_10_intended_diff_only": 0.04900002479553223, + "tpp_threshold_10_unintended_diff_only": 0.009749993681907654, + "tpp_threshold_20_total_metric": 0.067500039935112, + "tpp_threshold_20_intended_diff_only": 0.07100003957748413, + "tpp_threshold_20_unintended_diff_only": 0.0034999996423721313, + "tpp_threshold_50_total_metric": 0.12450003623962402, + "tpp_threshold_50_intended_diff_only": 0.13200002908706665, + "tpp_threshold_50_unintended_diff_only": 0.007499992847442627, + "tpp_threshold_100_total_metric": 0.28025004267692566, + "tpp_threshold_100_intended_diff_only": 0.30800002813339233, + "tpp_threshold_100_unintended_diff_only": 0.027749985456466675, + "tpp_threshold_500_total_metric": 0.3997500538825989, + "tpp_threshold_500_intended_diff_only": 0.4360000491142273, + "tpp_threshold_500_unintended_diff_only": 0.03624999523162842 }, "3": { - "tpp_threshold_2_total_metric": 0.016249999403953552, - "tpp_threshold_2_intended_diff_only": 0.009999990463256836, - "tpp_threshold_2_unintended_diff_only": -0.006250008940696716, - "tpp_threshold_5_total_metric": 0.01524999737739563, - "tpp_threshold_5_intended_diff_only": 0.013999998569488525, - "tpp_threshold_5_unintended_diff_only": -0.0012499988079071045, - "tpp_threshold_10_total_metric": 0.01975002884864807, - "tpp_threshold_10_intended_diff_only": 0.021000027656555176, - "tpp_threshold_10_unintended_diff_only": 0.0012499988079071045 + "tpp_threshold_2_total_metric": 0.005499988794326782, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": -0.0004999935626983643, + "tpp_threshold_5_total_metric": 0.00849999487400055, + "tpp_threshold_5_intended_diff_only": 0.004999995231628418, + "tpp_threshold_5_unintended_diff_only": -0.0034999996423721313, + "tpp_threshold_10_total_metric": 0.021249979734420776, + "tpp_threshold_10_intended_diff_only": 0.018999993801116943, + "tpp_threshold_10_unintended_diff_only": -0.002249985933303833, + "tpp_threshold_20_total_metric": 0.013999998569488525, + "tpp_threshold_20_intended_diff_only": 0.014999985694885254, + "tpp_threshold_20_unintended_diff_only": 0.0009999871253967285, + "tpp_threshold_50_total_metric": 0.056250035762786865, + "tpp_threshold_50_intended_diff_only": 0.054000020027160645, + "tpp_threshold_50_unintended_diff_only": -0.0022500157356262207, + "tpp_threshold_100_total_metric": 0.09549999237060547, + "tpp_threshold_100_intended_diff_only": 0.09799998998641968, + "tpp_threshold_100_unintended_diff_only": 0.002499997615814209, + "tpp_threshold_500_total_metric": 0.33650003373622894, + "tpp_threshold_500_intended_diff_only": 0.3450000286102295, + "tpp_threshold_500_unintended_diff_only": 0.00849999487400055 }, "5": { - "tpp_threshold_2_total_metric": -0.006750017404556274, - "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 0.007750004529953003, - "tpp_threshold_5_total_metric": 0.0029999762773513794, - "tpp_threshold_5_intended_diff_only": 0.006999969482421875, - "tpp_threshold_5_unintended_diff_only": 0.003999993205070496, - "tpp_threshold_10_total_metric": 0.005249962210655212, - "tpp_threshold_10_intended_diff_only": 0.010999977588653564, - "tpp_threshold_10_unintended_diff_only": 0.005750015377998352 + "tpp_threshold_2_total_metric": 0.005499973893165588, + "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_2_unintended_diff_only": 0.0005000084638595581, + "tpp_threshold_5_total_metric": 0.01199999451637268, + "tpp_threshold_5_intended_diff_only": 0.013999998569488525, + "tpp_threshold_5_unintended_diff_only": 0.0020000040531158447, + "tpp_threshold_10_total_metric": -0.005500033497810364, + "tpp_threshold_10_intended_diff_only": 0.0029999613761901855, + "tpp_threshold_10_unintended_diff_only": 0.00849999487400055, + "tpp_threshold_20_total_metric": 0.02449999749660492, + "tpp_threshold_20_intended_diff_only": 0.02799999713897705, + "tpp_threshold_20_unintended_diff_only": 0.0034999996423721313, + "tpp_threshold_50_total_metric": 0.08624997735023499, + "tpp_threshold_50_intended_diff_only": 0.09399998188018799, + "tpp_threshold_50_unintended_diff_only": 0.007750004529953003, + "tpp_threshold_100_total_metric": 0.1289999783039093, + "tpp_threshold_100_intended_diff_only": 0.13599997758865356, + "tpp_threshold_100_unintended_diff_only": 0.006999999284744263, + "tpp_threshold_500_total_metric": 0.31325002014636993, + "tpp_threshold_500_intended_diff_only": 0.3880000114440918, + "tpp_threshold_500_unintended_diff_only": 0.07474999129772186 }, "6": { - "tpp_threshold_2_total_metric": 0.02275000512599945, - "tpp_threshold_2_intended_diff_only": 0.023000001907348633, - "tpp_threshold_2_unintended_diff_only": 0.00024999678134918213, - "tpp_threshold_5_total_metric": 0.02399998903274536, - "tpp_threshold_5_intended_diff_only": 0.02899998426437378, - "tpp_threshold_5_unintended_diff_only": 0.004999995231628418, - "tpp_threshold_10_total_metric": 0.04399998486042023, - "tpp_threshold_10_intended_diff_only": 0.04799997806549072, - "tpp_threshold_10_unintended_diff_only": 0.003999993205070496 + "tpp_threshold_2_total_metric": 0.011999949812889099, + "tpp_threshold_2_intended_diff_only": 0.01699995994567871, + "tpp_threshold_2_unintended_diff_only": 0.005000010132789612, + "tpp_threshold_5_total_metric": 0.0209999680519104, + "tpp_threshold_5_intended_diff_only": 0.0209999680519104, + "tpp_threshold_5_unintended_diff_only": 0.0, + "tpp_threshold_10_total_metric": 0.05649997293949127, + "tpp_threshold_10_intended_diff_only": 0.06199997663497925, + "tpp_threshold_10_unintended_diff_only": 0.005500003695487976, + "tpp_threshold_20_total_metric": 0.09675002098083496, + "tpp_threshold_20_intended_diff_only": 0.11000001430511475, + "tpp_threshold_20_unintended_diff_only": 0.013249993324279785, + "tpp_threshold_50_total_metric": 0.17424997687339783, + "tpp_threshold_50_intended_diff_only": 0.18799996376037598, + "tpp_threshold_50_unintended_diff_only": 0.01374998688697815, + "tpp_threshold_100_total_metric": 0.2524999976158142, + "tpp_threshold_100_intended_diff_only": 0.26899999380111694, + "tpp_threshold_100_unintended_diff_only": 0.016499996185302734, + "tpp_threshold_500_total_metric": 0.34299999475479126, + "tpp_threshold_500_intended_diff_only": 0.3709999918937683, + "tpp_threshold_500_unintended_diff_only": 0.02799999713897705 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-244140-tokens-1000000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-244140-tokens-1000000000/scr/saebench_scr_custom_sae_eval_results.json index f54cf81748b850826bacde1d927b0849f84f508a..95f2fa137755be767cb523bcda66426ac7c8c15d 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-244140-tokens-1000000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-244140-tokens-1000000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "7b769476-9026-4538-89a9-a67496f617b5", - "datetime_epoch_millis": 1776885520377, + "eval_id": "e4e83998-fec8-45e5-bdbc-3730c9454491", + "datetime_epoch_millis": 1777317034894, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.2840545467631945, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.21232264677068308, "scr_metric_threshold_10": 0.20882376972801514, "scr_dir2_threshold_10": 0.20197067998020166, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.13545519102935433, + "scr_metric_threshold_20": 0.23108328808010634, + "scr_dir2_threshold_20": 0.22685304974797876, + "scr_dir1_threshold_50": 0.13788349135316502, + "scr_metric_threshold_50": 0.3072251125637197, + "scr_dir2_threshold_50": 0.2971335714465668, + "scr_dir1_threshold_100": -0.09793706365514628, + "scr_metric_threshold_100": 0.30912104268525553, + "scr_dir2_threshold_100": 0.3019522602916574, + "scr_dir1_threshold_500": -1.0374360799353397, + "scr_metric_threshold_500": 0.18726127052226596, + "scr_dir2_threshold_500": 0.17635157269187982 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.24359039998532808, "scr_metric_threshold_10": 0.010362714302611827, "scr_dir2_threshold_10": 0.010362714302611827, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.15384644775462453, + "scr_metric_threshold_20": 0.08290156000473131, + "scr_dir2_threshold_20": 0.08290156000473131, + "scr_dir1_threshold_50": 0.38461573730554943, + "scr_metric_threshold_50": 0.17616583431207444, + "scr_dir2_threshold_50": 0.17616583431207444, + "scr_dir1_threshold_100": 0.11538464477546245, + "scr_metric_threshold_100": 0.2797926685058661, + "scr_dir2_threshold_100": 0.2797926685058661, + "scr_dir1_threshold_500": -2.1538456835926008, + "scr_metric_threshold_500": -0.06735756575889522, + "scr_dir2_threshold_500": -0.06735756575889522 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.31060605718522544, "scr_metric_threshold_10": 0.29969413887699026, "scr_dir2_threshold_10": 0.29969413887699026, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3484848005931564, + "scr_metric_threshold_20": 0.22018355312924365, + "scr_dir2_threshold_20": 0.22018355312924365, + "scr_dir1_threshold_50": 0.5530302859261272, + "scr_metric_threshold_50": 0.3241590186196148, + "scr_dir2_threshold_50": 0.3241590186196148, + "scr_dir1_threshold_100": 0.3181817155567637, + "scr_metric_threshold_100": 0.30581044995124085, + "scr_dir2_threshold_100": 0.30581044995124085, + "scr_dir1_threshold_500": -1.8560602331895748, + "scr_metric_threshold_500": -0.006116128797061692, + "scr_dir2_threshold_500": -0.006116128797061692 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.1250002328306055, "scr_metric_threshold_10": 0.14070348146483033, "scr_dir2_threshold_10": 0.14070348146483033, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.28125005820765137, + "scr_metric_threshold_20": 0.15577896145062706, + "scr_dir2_threshold_20": 0.15577896145062706, + "scr_dir1_threshold_50": -0.7968748544808716, + "scr_metric_threshold_50": 0.15577896145062706, + "scr_dir2_threshold_50": 0.15577896145062706, + "scr_dir1_threshold_100": -0.9374998835846973, + "scr_metric_threshold_100": 0.027638280133700087, + "scr_dir2_threshold_100": 0.027638280133700087, + "scr_dir1_threshold_500": -3.9218741559890553, + "scr_metric_threshold_500": 0.07537695064781104, + "scr_dir2_threshold_500": 0.07537695064781104 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": -0.0805370818923876, "scr_metric_threshold_10": -0.009554016802434907, "scr_dir2_threshold_10": -0.009554016802434907, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.9395976886197124, + "scr_metric_threshold_20": -0.12420373702127886, + "scr_dir2_threshold_20": -0.12420373702127886, + "scr_dir1_threshold_50": -0.1610741637847752, + "scr_metric_threshold_50": 0.006369534358659739, + "scr_dir2_threshold_50": 0.006369534358659739, + "scr_dir1_threshold_100": -1.3355709745360138, + "scr_metric_threshold_100": -0.14649682254103324, + "scr_dir2_threshold_100": -0.14649682254103324, + "scr_dir1_threshold_500": -1.51006738525605, + "scr_metric_threshold_500": -0.5414013089483273, + "scr_dir2_threshold_500": -0.5414013089483273 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.09473687842942387, "scr_metric_threshold_10": 0.4285715299398585, "scr_dir2_threshold_10": 0.4285715299398585, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.08947378822971379, + "scr_metric_threshold_20": 0.5396826147702655, + "scr_dir2_threshold_20": 0.5396826147702655, + "scr_dir1_threshold_50": -0.13157882353605502, + "scr_metric_threshold_50": 0.6269840124753451, + "scr_dir2_threshold_50": 0.6269840124753451, + "scr_dir1_threshold_100": -0.15789458824326602, + "scr_metric_threshold_100": 0.6587301042915575, + "scr_dir2_threshold_100": 0.6587301042915575, + "scr_dir1_threshold_500": -0.12631573333634494, + "scr_metric_threshold_500": 0.773809391467407, + "scr_dir2_threshold_500": 0.773809391467407 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.4098360709132541, "scr_metric_threshold_10": 0.16412205665125212, "scr_dir2_threshold_10": 0.16412205665125212, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.4153004823742847, + "scr_metric_threshold_20": 0.22137413263306224, + "scr_dir2_threshold_20": 0.22137413263306224, + "scr_dir1_threshold_50": 0.4316940424658739, + "scr_metric_threshold_50": 0.28625946991455836, + "scr_dir2_threshold_50": 0.28625946991455836, + "scr_dir1_threshold_100": 0.4371584539269045, + "scr_metric_threshold_100": 0.48473290865407037, + "scr_dir2_threshold_100": 0.48473290865407037, + "scr_dir1_threshold_500": 0.3989072479911928, + "scr_metric_threshold_500": 0.35877852349298656, + "scr_dir2_threshold_500": 0.35877852349298656 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.4093022598196558, "scr_metric_threshold_10": 0.4506438964966536, "scr_dir2_threshold_10": 0.4506438964966536, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.48837210591771457, + "scr_metric_threshold_20": 0.506437685896798, + "scr_dir2_threshold_20": 0.506437685896798, + "scr_dir1_threshold_50": 0.5162789962690271, + "scr_metric_threshold_50": 0.5751073587184335, + "scr_dir2_threshold_50": 0.5751073587184335, + "scr_dir1_threshold_100": 0.49767431029119774, + "scr_metric_threshold_100": 0.5836909399141642, + "scr_dir2_threshold_100": 0.5836909399141642, + "scr_dir1_threshold_500": 0.48837210591771457, + "scr_metric_threshold_500": 0.5236051041022068, + "scr_dir2_threshold_500": 0.5236051041022068 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.18604635689435922, "scr_metric_threshold_10": 0.18604635689435922, "scr_dir2_threshold_10": 0.1312216389118512, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.24651153377740184, + "scr_metric_threshold_20": 0.24651153377740184, + "scr_dir2_threshold_20": 0.21266962712038115, + "scr_dir1_threshold_50": 0.3069767106604445, + "scr_metric_threshold_50": 0.3069767106604445, + "scr_dir2_threshold_50": 0.2262443817232214, + "scr_dir1_threshold_100": 0.27906981257247854, + "scr_metric_threshold_100": 0.27906981257247854, + "scr_dir2_threshold_100": 0.22171955342369326, + "scr_dir1_threshold_500": 0.3813951979720002, + "scr_metric_threshold_500": 0.3813951979720002, + "scr_dir2_threshold_500": 0.29411761532891106 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-244140-tokens-1000000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-244140-tokens-1000000000/tpp/saebench_tpp_custom_sae_eval_results.json index 496e0f1552cd968d623b59d37b14111f87cabfcf..9eb6c48b7c678d290883f8fc5bcc3c60f15b452c 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-244140-tokens-1000000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-244140-tokens-1000000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "f9684801-165e-4650-91a4-1f8bd9b21619", - "datetime_epoch_millis": 1776885682543, + "eval_id": "194fbc19-bfbb-4a57-9342-0a022d09d4d0", + "datetime_epoch_millis": 1777317310395, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.02392500936985016, - "tpp_threshold_2_intended_diff_only": 0.02620001435279846, - "tpp_threshold_2_unintended_diff_only": 0.0022750049829483032, - "tpp_threshold_5_total_metric": 0.10072501301765441, - "tpp_threshold_5_intended_diff_only": 0.13040001392364503, - "tpp_threshold_5_unintended_diff_only": 0.0296750009059906, - "tpp_threshold_10_total_metric": 0.10860000550746918, - "tpp_threshold_10_intended_diff_only": 0.1636000096797943, - "tpp_threshold_10_unintended_diff_only": 0.05500000417232514, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.02252500206232071, + "tpp_threshold_2_intended_diff_only": 0.025500005483627318, + "tpp_threshold_2_unintended_diff_only": 0.0029750034213066098, + "tpp_threshold_5_total_metric": 0.10015000998973846, + "tpp_threshold_5_intended_diff_only": 0.12900001406669617, + "tpp_threshold_5_unintended_diff_only": 0.028850004076957703, + "tpp_threshold_10_total_metric": 0.1085500031709671, + "tpp_threshold_10_intended_diff_only": 0.16440001130104065, + "tpp_threshold_10_unintended_diff_only": 0.05585000813007354, + "tpp_threshold_20_total_metric": 0.13464999347925186, + "tpp_threshold_20_intended_diff_only": 0.21290000081062316, + "tpp_threshold_20_unintended_diff_only": 0.07825000733137132, + "tpp_threshold_50_total_metric": 0.14915000647306442, + "tpp_threshold_50_intended_diff_only": 0.26150001883506774, + "tpp_threshold_50_unintended_diff_only": 0.11235001236200333, + "tpp_threshold_100_total_metric": 0.1778000056743622, + "tpp_threshold_100_intended_diff_only": 0.32670001983642577, + "tpp_threshold_100_unintended_diff_only": 0.1489000141620636, + "tpp_threshold_500_total_metric": 0.2337000250816345, + "tpp_threshold_500_intended_diff_only": 0.43680003881454466, + "tpp_threshold_500_unintended_diff_only": 0.20310001373291015 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.03755001127719879, - "tpp_threshold_2_intended_diff_only": 0.04180002212524414, - "tpp_threshold_2_unintended_diff_only": 0.0042500108480453495, - "tpp_threshold_5_total_metric": 0.18545002043247222, - "tpp_threshold_5_intended_diff_only": 0.24300003051757812, - "tpp_threshold_5_unintended_diff_only": 0.05755001008510589, - "tpp_threshold_10_total_metric": 0.19265002012252808, + "tpp_threshold_2_total_metric": 0.03645000159740448, + "tpp_threshold_2_intended_diff_only": 0.041400015354156494, + "tpp_threshold_2_unintended_diff_only": 0.004950013756752014, + "tpp_threshold_5_total_metric": 0.1860500156879425, + "tpp_threshold_5_intended_diff_only": 0.24320002794265747, + "tpp_threshold_5_unintended_diff_only": 0.057150012254714964, + "tpp_threshold_10_total_metric": 0.19125000536441802, "tpp_threshold_10_intended_diff_only": 0.29980002641677855, - "tpp_threshold_10_unintended_diff_only": 0.10715000629425049, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_10_unintended_diff_only": 0.10855002105236053, + "tpp_threshold_20_total_metric": 0.21470000445842743, + "tpp_threshold_20_intended_diff_only": 0.3666000247001648, + "tpp_threshold_20_unintended_diff_only": 0.15190002024173738, + "tpp_threshold_50_total_metric": 0.19535000622272491, + "tpp_threshold_50_intended_diff_only": 0.4150000333786011, + "tpp_threshold_50_unintended_diff_only": 0.21965002715587617, + "tpp_threshold_100_total_metric": 0.1574999988079071, + "tpp_threshold_100_intended_diff_only": 0.4402000308036804, + "tpp_threshold_100_unintended_diff_only": 0.2827000319957733, + "tpp_threshold_500_total_metric": 0.09405001103878022, + "tpp_threshold_500_intended_diff_only": 0.4670000433921814, + "tpp_threshold_500_unintended_diff_only": 0.3729500323534012 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.010300007462501527, - "tpp_threshold_2_intended_diff_only": 0.010600006580352784, - "tpp_threshold_2_unintended_diff_only": 0.0002999991178512573, - "tpp_threshold_5_total_metric": 0.016000005602836608, - "tpp_threshold_5_intended_diff_only": 0.017799997329711915, - "tpp_threshold_5_unintended_diff_only": 0.001799991726875305, - "tpp_threshold_10_total_metric": 0.02454999089241028, - "tpp_threshold_10_intended_diff_only": 0.027399992942810057, - "tpp_threshold_10_unintended_diff_only": 0.00285000205039978, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.008600002527236939, + "tpp_threshold_2_intended_diff_only": 0.009599995613098145, + "tpp_threshold_2_unintended_diff_only": 0.000999993085861206, + "tpp_threshold_5_total_metric": 0.014250004291534423, + "tpp_threshold_5_intended_diff_only": 0.014800000190734863, + "tpp_threshold_5_unintended_diff_only": 0.0005499958992004394, + "tpp_threshold_10_total_metric": 0.025850000977516174, + "tpp_threshold_10_intended_diff_only": 0.028999996185302735, + "tpp_threshold_10_unintended_diff_only": 0.00314999520778656, + "tpp_threshold_20_total_metric": 0.05459998250007629, + "tpp_threshold_20_intended_diff_only": 0.059199976921081546, + "tpp_threshold_20_unintended_diff_only": 0.004599994421005249, + "tpp_threshold_50_total_metric": 0.10295000672340393, + "tpp_threshold_50_intended_diff_only": 0.10800000429153442, + "tpp_threshold_50_unintended_diff_only": 0.005049997568130493, + "tpp_threshold_100_total_metric": 0.19810001254081727, + "tpp_threshold_100_intended_diff_only": 0.21320000886917115, + "tpp_threshold_100_unintended_diff_only": 0.015099996328353881, + "tpp_threshold_500_total_metric": 0.37335003912448883, + "tpp_threshold_500_intended_diff_only": 0.40660003423690794, + "tpp_threshold_500_unintended_diff_only": 0.03324999511241913 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.019500017166137695, "tpp_threshold_10_total_metric": 0.1617499738931656, "tpp_threshold_10_intended_diff_only": 0.1809999942779541, - "tpp_threshold_10_unintended_diff_only": 0.019250020384788513 + "tpp_threshold_10_unintended_diff_only": 0.019250020384788513, + "tpp_threshold_20_total_metric": 0.21324995160102844, + "tpp_threshold_20_intended_diff_only": 0.36299997568130493, + "tpp_threshold_20_unintended_diff_only": 0.1497500240802765, + "tpp_threshold_50_total_metric": 0.21424999833106995, + "tpp_threshold_50_intended_diff_only": 0.4190000295639038, + "tpp_threshold_50_unintended_diff_only": 0.20475003123283386, + "tpp_threshold_100_total_metric": 0.17949995398521423, + "tpp_threshold_100_intended_diff_only": 0.44099998474121094, + "tpp_threshold_100_unintended_diff_only": 0.2615000307559967, + "tpp_threshold_500_total_metric": 0.08949998021125793, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.36650004982948303 }, "1": { - "tpp_threshold_2_total_metric": 0.04900000989437103, + "tpp_threshold_2_total_metric": 0.05224999785423279, "tpp_threshold_2_intended_diff_only": 0.05900001525878906, - "tpp_threshold_2_unintended_diff_only": 0.01000000536441803, - "tpp_threshold_5_total_metric": 0.1902499943971634, + "tpp_threshold_2_unintended_diff_only": 0.006750017404556274, + "tpp_threshold_5_total_metric": 0.19199998676776886, "tpp_threshold_5_intended_diff_only": 0.2630000114440918, - "tpp_threshold_5_unintended_diff_only": 0.0727500170469284, - "tpp_threshold_10_total_metric": 0.20275001227855682, + "tpp_threshold_5_unintended_diff_only": 0.07100002467632294, + "tpp_threshold_10_total_metric": 0.1952500194311142, "tpp_threshold_10_intended_diff_only": 0.32200002670288086, - "tpp_threshold_10_unintended_diff_only": 0.11925001442432404 + "tpp_threshold_10_unintended_diff_only": 0.12675000727176666, + "tpp_threshold_20_total_metric": 0.2110000103712082, + "tpp_threshold_20_intended_diff_only": 0.3540000319480896, + "tpp_threshold_20_unintended_diff_only": 0.1430000215768814, + "tpp_threshold_50_total_metric": 0.16900001466274261, + "tpp_threshold_50_intended_diff_only": 0.3940000534057617, + "tpp_threshold_50_unintended_diff_only": 0.2250000387430191, + "tpp_threshold_100_total_metric": 0.14649996161460876, + "tpp_threshold_100_intended_diff_only": 0.4259999990463257, + "tpp_threshold_100_unintended_diff_only": 0.2795000374317169, + "tpp_threshold_500_total_metric": 0.08175002038478851, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.3772500306367874 }, "2": { - "tpp_threshold_2_total_metric": 0.022750020027160645, - "tpp_threshold_2_intended_diff_only": 0.025000035762786865, - "tpp_threshold_2_unintended_diff_only": 0.0022500157356262207, - "tpp_threshold_5_total_metric": 0.19025000929832458, - "tpp_threshold_5_intended_diff_only": 0.24800002574920654, - "tpp_threshold_5_unintended_diff_only": 0.05775001645088196, - "tpp_threshold_10_total_metric": 0.130000039935112, - "tpp_threshold_10_intended_diff_only": 0.33900004625320435, - "tpp_threshold_10_unintended_diff_only": 0.20900000631809235 + "tpp_threshold_2_total_metric": 0.01400001347064972, + "tpp_threshold_2_intended_diff_only": 0.021000027656555176, + "tpp_threshold_2_unintended_diff_only": 0.0070000141859054565, + "tpp_threshold_5_total_metric": 0.19050003588199615, + "tpp_threshold_5_intended_diff_only": 0.24700003862380981, + "tpp_threshold_5_unintended_diff_only": 0.05650000274181366, + "tpp_threshold_10_total_metric": 0.132750004529953, + "tpp_threshold_10_intended_diff_only": 0.3400000333786011, + "tpp_threshold_10_unintended_diff_only": 0.20725002884864807, + "tpp_threshold_20_total_metric": 0.12600001692771912, + "tpp_threshold_20_intended_diff_only": 0.3580000400543213, + "tpp_threshold_20_unintended_diff_only": 0.23200002312660217, + "tpp_threshold_50_total_metric": 0.10300001502037048, + "tpp_threshold_50_intended_diff_only": 0.37700003385543823, + "tpp_threshold_50_unintended_diff_only": 0.27400001883506775, + "tpp_threshold_100_total_metric": 0.05925004184246063, + "tpp_threshold_100_intended_diff_only": 0.4020000696182251, + "tpp_threshold_100_unintended_diff_only": 0.34275002777576447, + "tpp_threshold_500_total_metric": 0.05025000870227814, + "tpp_threshold_500_intended_diff_only": 0.45100003480911255, + "tpp_threshold_500_unintended_diff_only": 0.4007500261068344 }, "6": { - "tpp_threshold_2_total_metric": 0.011500000953674316, + "tpp_threshold_2_total_metric": 0.009499981999397278, "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": -0.002499997615814209, - "tpp_threshold_5_total_metric": 0.25700004398822784, - "tpp_threshold_5_intended_diff_only": 0.33000004291534424, - "tpp_threshold_5_unintended_diff_only": 0.0729999989271164, - "tpp_threshold_10_total_metric": 0.2820000648498535, + "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704, + "tpp_threshold_5_total_metric": 0.26100002229213715, + "tpp_threshold_5_intended_diff_only": 0.33100003004074097, + "tpp_threshold_5_unintended_diff_only": 0.07000000774860382, + "tpp_threshold_10_total_metric": 0.2827500253915787, "tpp_threshold_10_intended_diff_only": 0.3940000534057617, - "tpp_threshold_10_unintended_diff_only": 0.1119999885559082 + "tpp_threshold_10_unintended_diff_only": 0.11125002801418304, + "tpp_threshold_20_total_metric": 0.30000002682209015, + "tpp_threshold_20_intended_diff_only": 0.4410000443458557, + "tpp_threshold_20_unintended_diff_only": 0.14100001752376556, + "tpp_threshold_50_total_metric": 0.265500009059906, + "tpp_threshold_50_intended_diff_only": 0.4790000319480896, + "tpp_threshold_50_unintended_diff_only": 0.2135000228881836, + "tpp_threshold_100_total_metric": 0.20400001108646393, + "tpp_threshold_100_intended_diff_only": 0.4830000400543213, + "tpp_threshold_100_unintended_diff_only": 0.27900002896785736, + "tpp_threshold_500_total_metric": 0.1287499964237213, + "tpp_threshold_500_intended_diff_only": 0.4930000305175781, + "tpp_threshold_500_unintended_diff_only": 0.3642500340938568 }, "9": { - "tpp_threshold_2_total_metric": 0.005250036716461182, - "tpp_threshold_2_intended_diff_only": 0.006000041961669922, - "tpp_threshold_2_unintended_diff_only": 0.0007500052452087402, - "tpp_threshold_5_total_metric": 0.153250053524971, - "tpp_threshold_5_intended_diff_only": 0.21800005435943604, - "tpp_threshold_5_unintended_diff_only": 0.06475000083446503, - "tpp_threshold_10_total_metric": 0.18675000965595245, - "tpp_threshold_10_intended_diff_only": 0.2630000114440918, - "tpp_threshold_10_unintended_diff_only": 0.07625000178813934 + "tpp_threshold_2_total_metric": 0.0072500258684158325, + "tpp_threshold_2_intended_diff_only": 0.008000016212463379, + "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.15025003254413605, + "tpp_threshold_5_intended_diff_only": 0.21900004148483276, + "tpp_threshold_5_unintended_diff_only": 0.06875000894069672, + "tpp_threshold_10_total_metric": 0.1837500035762787, + "tpp_threshold_10_intended_diff_only": 0.26200002431869507, + "tpp_threshold_10_unintended_diff_only": 0.07825002074241638, + "tpp_threshold_20_total_metric": 0.22325001657009125, + "tpp_threshold_20_intended_diff_only": 0.31700003147125244, + "tpp_threshold_20_unintended_diff_only": 0.0937500149011612, + "tpp_threshold_50_total_metric": 0.22499999403953552, + "tpp_threshold_50_intended_diff_only": 0.406000018119812, + "tpp_threshold_50_unintended_diff_only": 0.1810000240802765, + "tpp_threshold_100_total_metric": 0.19825002551078796, + "tpp_threshold_100_intended_diff_only": 0.4490000605583191, + "tpp_threshold_100_unintended_diff_only": 0.25075003504753113, + "tpp_threshold_500_total_metric": 0.12000004947185516, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.35600002110004425 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.008000016212463379, - "tpp_threshold_2_intended_diff_only": 0.003000020980834961, - "tpp_threshold_2_unintended_diff_only": -0.004999995231628418, - "tpp_threshold_5_total_metric": 0.00974997878074646, - "tpp_threshold_5_intended_diff_only": 0.006999969482421875, - "tpp_threshold_5_unintended_diff_only": -0.002750009298324585, - "tpp_threshold_10_total_metric": -0.001749977469444275, - "tpp_threshold_10_intended_diff_only": 0.003000020980834961, - "tpp_threshold_10_unintended_diff_only": 0.004749998450279236 + "tpp_threshold_2_total_metric": 0.0015000104904174805, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 0.0034999847412109375, + "tpp_threshold_5_total_metric": 0.002750009298324585, + "tpp_threshold_5_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_10_total_metric": -0.0007500350475311279, + "tpp_threshold_10_intended_diff_only": 0.001999974250793457, + "tpp_threshold_10_unintended_diff_only": 0.002750009298324585, + "tpp_threshold_20_total_metric": 0.011749997735023499, + "tpp_threshold_20_intended_diff_only": 0.014999985694885254, + "tpp_threshold_20_unintended_diff_only": 0.0032499879598617554, + "tpp_threshold_50_total_metric": 0.04874999821186066, + "tpp_threshold_50_intended_diff_only": 0.046000003814697266, + "tpp_threshold_50_unintended_diff_only": -0.002749994397163391, + "tpp_threshold_100_total_metric": 0.11000002920627594, + "tpp_threshold_100_intended_diff_only": 0.11900001764297485, + "tpp_threshold_100_unintended_diff_only": 0.008999988436698914, + "tpp_threshold_500_total_metric": 0.3965000510215759, + "tpp_threshold_500_intended_diff_only": 0.409000039100647, + "tpp_threshold_500_unintended_diff_only": 0.012499988079071045 }, "2": { - "tpp_threshold_2_total_metric": 0.004250004887580872, - "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": 0.004749998450279236, - "tpp_threshold_5_total_metric": 0.013250067830085754, - "tpp_threshold_5_intended_diff_only": 0.01900005340576172, - "tpp_threshold_5_unintended_diff_only": 0.005749985575675964, - "tpp_threshold_10_total_metric": 0.03425005078315735, + "tpp_threshold_2_total_metric": 0.013250038027763367, + "tpp_threshold_2_intended_diff_only": 0.01100003719329834, + "tpp_threshold_2_unintended_diff_only": -0.002250000834465027, + "tpp_threshold_5_total_metric": 0.007500007748603821, + "tpp_threshold_5_intended_diff_only": 0.013999998569488525, + "tpp_threshold_5_unintended_diff_only": 0.006499990820884705, + "tpp_threshold_10_total_metric": 0.03250005841255188, "tpp_threshold_10_intended_diff_only": 0.03800004720687866, - "tpp_threshold_10_unintended_diff_only": 0.0037499964237213135 + "tpp_threshold_10_unintended_diff_only": 0.005499988794326782, + "tpp_threshold_20_total_metric": 0.09225001931190491, + "tpp_threshold_20_intended_diff_only": 0.09700000286102295, + "tpp_threshold_20_unintended_diff_only": 0.004749983549118042, + "tpp_threshold_50_total_metric": 0.1572500616312027, + "tpp_threshold_50_intended_diff_only": 0.16800004243850708, + "tpp_threshold_50_unintended_diff_only": 0.010749980807304382, + "tpp_threshold_100_total_metric": 0.27675004303455353, + "tpp_threshold_100_intended_diff_only": 0.2980000376701355, + "tpp_threshold_100_unintended_diff_only": 0.02124999463558197, + "tpp_threshold_500_total_metric": 0.403000071644783, + "tpp_threshold_500_intended_diff_only": 0.43900007009506226, + "tpp_threshold_500_unintended_diff_only": 0.035999998450279236 }, "3": { - "tpp_threshold_2_total_metric": 0.01000000536441803, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": -0.00599999725818634, - "tpp_threshold_5_total_metric": 0.01875002682209015, - "tpp_threshold_5_intended_diff_only": 0.017000019550323486, - "tpp_threshold_5_unintended_diff_only": -0.0017500072717666626, - "tpp_threshold_10_total_metric": 0.014749974012374878, - "tpp_threshold_10_intended_diff_only": 0.015999972820281982, - "tpp_threshold_10_unintended_diff_only": 0.0012499988079071045 + "tpp_threshold_2_total_metric": 0.0014999955892562866, + "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, + "tpp_threshold_2_unintended_diff_only": -0.0005000084638595581, + "tpp_threshold_5_total_metric": 0.012000039219856262, + "tpp_threshold_5_intended_diff_only": 0.00700002908706665, + "tpp_threshold_5_unintended_diff_only": -0.005000010132789612, + "tpp_threshold_10_total_metric": 0.015499994158744812, + "tpp_threshold_10_intended_diff_only": 0.013999998569488525, + "tpp_threshold_10_unintended_diff_only": -0.0014999955892562866, + "tpp_threshold_20_total_metric": 0.00849999487400055, + "tpp_threshold_20_intended_diff_only": 0.010999977588653564, + "tpp_threshold_20_unintended_diff_only": 0.002499982714653015, + "tpp_threshold_50_total_metric": 0.036500006914138794, + "tpp_threshold_50_intended_diff_only": 0.03299999237060547, + "tpp_threshold_50_unintended_diff_only": -0.003500014543533325, + "tpp_threshold_100_total_metric": 0.10525000095367432, + "tpp_threshold_100_intended_diff_only": 0.12099999189376831, + "tpp_threshold_100_unintended_diff_only": 0.015749990940093994, + "tpp_threshold_500_total_metric": 0.3632500320672989, + "tpp_threshold_500_intended_diff_only": 0.3960000276565552, + "tpp_threshold_500_unintended_diff_only": 0.03274999558925629 }, "5": { - "tpp_threshold_2_total_metric": -0.007500007748603821, - "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 0.00849999487400055, - "tpp_threshold_5_total_metric": 0.0029999464750289917, - "tpp_threshold_5_intended_diff_only": 0.003999948501586914, - "tpp_threshold_5_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_10_total_metric": 0.010249942541122437, - "tpp_threshold_10_intended_diff_only": 0.012999951839447021, - "tpp_threshold_10_unintended_diff_only": 0.002750009298324585 + "tpp_threshold_2_total_metric": 0.001749977469444275, + "tpp_threshold_2_intended_diff_only": 0.001999974250793457, + "tpp_threshold_2_unintended_diff_only": 0.00024999678134918213, + "tpp_threshold_5_total_metric": 0.010499998927116394, + "tpp_threshold_5_intended_diff_only": 0.009999990463256836, + "tpp_threshold_5_unintended_diff_only": -0.0005000084638595581, + "tpp_threshold_10_total_metric": 0.0032499581575393677, + "tpp_threshold_10_intended_diff_only": 0.008999943733215332, + "tpp_threshold_10_unintended_diff_only": 0.005749985575675964, + "tpp_threshold_20_total_metric": 0.02599993348121643, + "tpp_threshold_20_intended_diff_only": 0.026999950408935547, + "tpp_threshold_20_unintended_diff_only": 0.0010000169277191162, + "tpp_threshold_50_total_metric": 0.05699998140335083, + "tpp_threshold_50_intended_diff_only": 0.06599998474121094, + "tpp_threshold_50_unintended_diff_only": 0.009000003337860107, + "tpp_threshold_100_total_metric": 0.19049999117851257, + "tpp_threshold_100_intended_diff_only": 0.20399999618530273, + "tpp_threshold_100_unintended_diff_only": 0.013500005006790161, + "tpp_threshold_500_total_metric": 0.375, + "tpp_threshold_500_intended_diff_only": 0.41600000858306885, + "tpp_threshold_500_unintended_diff_only": 0.04100000858306885 }, "6": { - "tpp_threshold_2_total_metric": 0.03675001859664917, - "tpp_threshold_2_intended_diff_only": 0.03600001335144043, - "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402, - "tpp_threshold_5_total_metric": 0.03525000810623169, - "tpp_threshold_5_intended_diff_only": 0.041999995708465576, - "tpp_threshold_5_unintended_diff_only": 0.006749987602233887, - "tpp_threshold_10_total_metric": 0.065249964594841, - "tpp_threshold_10_intended_diff_only": 0.06699997186660767, - "tpp_threshold_10_unintended_diff_only": 0.0017500072717666626 + "tpp_threshold_2_total_metric": 0.024999991059303284, + "tpp_threshold_2_intended_diff_only": 0.02899998426437378, + "tpp_threshold_2_unintended_diff_only": 0.003999993205070496, + "tpp_threshold_5_total_metric": 0.03849996626377106, + "tpp_threshold_5_intended_diff_only": 0.038999974727630615, + "tpp_threshold_5_unintended_diff_only": 0.0005000084638595581, + "tpp_threshold_10_total_metric": 0.07875002920627594, + "tpp_threshold_10_intended_diff_only": 0.0820000171661377, + "tpp_threshold_10_unintended_diff_only": 0.0032499879598617554, + "tpp_threshold_20_total_metric": 0.13449996709823608, + "tpp_threshold_20_intended_diff_only": 0.1459999680519104, + "tpp_threshold_20_unintended_diff_only": 0.011500000953674316, + "tpp_threshold_50_total_metric": 0.21524998545646667, + "tpp_threshold_50_intended_diff_only": 0.22699999809265137, + "tpp_threshold_50_unintended_diff_only": 0.011750012636184692, + "tpp_threshold_100_total_metric": 0.30799999833106995, + "tpp_threshold_100_intended_diff_only": 0.3240000009536743, + "tpp_threshold_100_unintended_diff_only": 0.01600000262260437, + "tpp_threshold_500_total_metric": 0.3290000408887863, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.04399998486042023 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-268554-tokens-1100000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-268554-tokens-1100000000/scr/saebench_scr_custom_sae_eval_results.json index 4a1d43428c57712eb374c48f81024708f9975b95..d17292d5e2488ab20d30b363fb939e2e147db091 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-268554-tokens-1100000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-268554-tokens-1100000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "ae563e9f-2c3f-40c3-8da8-85ecdc76f324", - "datetime_epoch_millis": 1776885575680, + "eval_id": "8490fe9e-8ae3-4400-9250-cd9ba5aa8d6e", + "datetime_epoch_millis": 1777317133212, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.27799565814935673, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.09746747674747715, "scr_metric_threshold_10": 0.20395231432564953, "scr_dir2_threshold_10": 0.19763325838028084, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.06295881921345468, + "scr_metric_threshold_20": 0.22838102912176494, + "scr_dir2_threshold_20": 0.21502211012524783, + "scr_dir1_threshold_50": -0.05904648623138321, + "scr_metric_threshold_50": 0.3063793988798543, + "scr_dir2_threshold_50": 0.29803205622013307, + "scr_dir1_threshold_100": -0.10039104355690062, + "scr_metric_threshold_100": 0.31131066188410583, + "scr_dir2_threshold_100": 0.30588607794793926, + "scr_dir1_threshold_500": -0.9007161627968477, + "scr_metric_threshold_500": 0.2019885197553268, + "scr_dir2_threshold_500": 0.19177070157986006 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.3589742805987668, "scr_metric_threshold_10": 0.046632137153671566, "scr_dir2_threshold_10": 0.046632137153671566, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.07692284179630039, + "scr_metric_threshold_20": 0.14248705143262683, + "scr_dir2_threshold_20": 0.14248705143262683, + "scr_dir1_threshold_50": 0.39743608357792887, + "scr_metric_threshold_50": 0.2720207485910297, + "scr_dir2_threshold_50": 0.2720207485910297, + "scr_dir1_threshold_100": 0.20512859700616604, + "scr_metric_threshold_100": 0.2461140400425818, + "scr_dir2_threshold_100": 0.2461140400425818, + "scr_dir1_threshold_500": -2.589742805987668, + "scr_metric_threshold_500": -0.12953369715840288, + "scr_dir2_threshold_500": -0.12953369715840288 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.15151542518196345, "scr_metric_threshold_10": 0.14984716057708958, "scr_dir2_threshold_10": 0.14984716057708958, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.03030308503639269, + "scr_metric_threshold_20": 0.030581008539686302, + "scr_dir2_threshold_20": 0.030581008539686302, + "scr_dir1_threshold_50": 0.03030308503639269, + "scr_metric_threshold_50": 0.3700305314291443, + "scr_dir2_threshold_50": 0.3700305314291443, + "scr_dir1_threshold_100": 0.0, + "scr_metric_threshold_100": 0.5993884600311694, + "scr_dir2_threshold_100": 0.5993884600311694, + "scr_dir1_threshold_500": -1.4924236643031021, + "scr_metric_threshold_500": 0.1743120403197142, + "scr_dir2_threshold_500": 0.1743120403197142 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.1093754365573853, "scr_metric_threshold_10": 0.10804029093651613, "scr_dir2_threshold_10": 0.10804029093651613, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.17187462165026607, + "scr_metric_threshold_20": 0.17085429167603294, + "scr_dir2_threshold_20": 0.17085429167603294, + "scr_dir1_threshold_50": -1.6874994179234863, + "scr_metric_threshold_50": 0.16080402160563206, + "scr_dir2_threshold_50": 0.16080402160563206, + "scr_dir1_threshold_100": -0.8906245634426148, + "scr_metric_threshold_100": 0.03266334028870509, + "scr_dir2_threshold_100": 0.03266334028870509, + "scr_dir1_threshold_500": -2.5, + "scr_metric_threshold_500": 0.08291454088031854, + "scr_dir2_threshold_500": 0.08291454088031854 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": -0.9127519946555831, "scr_metric_threshold_10": 0.06369439446808171, "scr_dir2_threshold_10": 0.06369439446808171, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.8322149127631955, + "scr_metric_threshold_20": -0.05413999801824054, + "scr_dir2_threshold_20": -0.05413999801824054, + "scr_dir1_threshold_50": -0.2953022335742188, + "scr_metric_threshold_50": -0.05732486010942198, + "scr_dir2_threshold_50": -0.05732486010942198, + "scr_dir1_threshold_100": -1.2080538281985995, + "scr_metric_threshold_100": -0.2070063549179335, + "scr_dir2_threshold_100": -0.2070063549179335, + "scr_dir1_threshold_500": -1.8053696188302688, + "scr_metric_threshold_500": -0.5254777577872327, + "scr_dir2_threshold_500": -0.5254777577872327 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.09999996862913393, "scr_metric_threshold_10": 0.4761904311378407, "scr_dir2_threshold_10": 0.4761904311378407, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.12105264313663487, + "scr_metric_threshold_20": 0.5277777712076017, + "scr_dir2_threshold_20": 0.5277777712076017, + "scr_dir1_threshold_50": -0.16842108235134678, + "scr_metric_threshold_50": 0.6507936996006726, + "scr_dir2_threshold_50": 0.6507936996006726, + "scr_dir1_threshold_100": -0.15789458824326602, + "scr_metric_threshold_100": 0.714285646706761, + "scr_dir2_threshold_100": 0.714285646706761, + "scr_dir1_threshold_500": -0.12105264313663487, + "scr_metric_threshold_500": 0.7857142350300708, + "scr_dir2_threshold_500": 0.7857142350300708 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.4098360709132541, "scr_metric_threshold_10": 0.14122136275770192, "scr_dir2_threshold_10": 0.14122136275770192, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.4153004823742847, + "scr_metric_threshold_20": 0.19465646684173443, + "scr_dir2_threshold_20": 0.19465646684173443, + "scr_dir1_threshold_50": 0.4371584539269045, + "scr_metric_threshold_50": 0.25954203162185363, + "scr_dir2_threshold_50": 0.25954203162185363, + "scr_dir1_threshold_100": 0.4808743970321442, + "scr_metric_threshold_100": 0.3206106245041952, + "scr_dir2_threshold_100": 0.3206106245041952, + "scr_dir1_threshold_500": 0.4098360709132541, + "scr_metric_threshold_500": 0.31297713570588614, + "scr_dir2_threshold_500": 0.31297713570588614 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.3674417856772556, "scr_metric_threshold_10": 0.4506438964966536, "scr_dir2_threshold_10": 0.4506438964966536, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.46511631775314366, + "scr_metric_threshold_20": 0.5450643129054811, + "scr_dir2_threshold_20": 0.5450643129054811, + "scr_dir1_threshold_50": 0.5209300984557687, + "scr_metric_threshold_50": 0.5021458952989327, + "scr_dir2_threshold_50": 0.5021458952989327, + "scr_dir1_threshold_100": 0.5023254124779394, + "scr_metric_threshold_100": 0.5193133135043415, + "scr_dir2_threshold_100": 0.5193133135043415, + "scr_dir1_threshold_500": 0.548837266037944, + "scr_metric_threshold_500": 0.5708155681205682, + "scr_dir2_threshold_500": 0.5708155681205682 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.19534884107764103, "scr_metric_threshold_10": 0.19534884107764103, "scr_dir2_threshold_10": 0.14479639351469145, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.26976732838919676, + "scr_metric_threshold_20": 0.26976732838919676, + "scr_dir2_threshold_20": 0.16289597641705988, + "scr_dir1_threshold_50": 0.29302312300099165, + "scr_metric_threshold_50": 0.29302312300099165, + "scr_dir2_threshold_50": 0.2262443817232214, + "scr_dir1_threshold_100": 0.2651162249130257, + "scr_metric_threshold_100": 0.2651162249130257, + "scr_dir2_threshold_100": 0.22171955342369326, + "scr_dir1_threshold_500": 0.3441860929316922, + "scr_metric_threshold_500": 0.3441860929316922, + "scr_dir2_threshold_500": 0.26244354752795823 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-268554-tokens-1100000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-268554-tokens-1100000000/tpp/saebench_tpp_custom_sae_eval_results.json index 4b08496e6763a1719842acb2834d79eef6138014..f89465f7d4cdf9e690d56c8c0ab603d000ada679 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-268554-tokens-1100000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-268554-tokens-1100000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "22271bd6-3e5d-48e4-ad56-7bdadb5c08f7", - "datetime_epoch_millis": 1776885726256, + "eval_id": "ca94a942-63bd-47ee-a0d0-ad74ff107731", + "datetime_epoch_millis": 1777317407490, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.04750001132488251, - "tpp_threshold_2_intended_diff_only": 0.05780001282691956, - "tpp_threshold_2_unintended_diff_only": 0.010300001502037049, - "tpp_threshold_5_total_metric": 0.099875009059906, - "tpp_threshold_5_intended_diff_only": 0.13090001344680785, - "tpp_threshold_5_unintended_diff_only": 0.031025004386901853, - "tpp_threshold_10_total_metric": 0.11487500071525573, - "tpp_threshold_10_intended_diff_only": 0.17110000848770143, - "tpp_threshold_10_unintended_diff_only": 0.05622500777244568, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.046325007081031794, + "tpp_threshold_2_intended_diff_only": 0.05740000605583191, + "tpp_threshold_2_unintended_diff_only": 0.01107499897480011, + "tpp_threshold_5_total_metric": 0.0999000072479248, + "tpp_threshold_5_intended_diff_only": 0.13030001521110535, + "tpp_threshold_5_unintended_diff_only": 0.030400007963180542, + "tpp_threshold_10_total_metric": 0.115425007045269, + "tpp_threshold_10_intended_diff_only": 0.17250001430511475, + "tpp_threshold_10_unintended_diff_only": 0.05707500725984573, + "tpp_threshold_20_total_metric": 0.13175000250339508, + "tpp_threshold_20_intended_diff_only": 0.2121000111103058, + "tpp_threshold_20_unintended_diff_only": 0.0803500086069107, + "tpp_threshold_50_total_metric": 0.15572500377893447, + "tpp_threshold_50_intended_diff_only": 0.27340001463890073, + "tpp_threshold_50_unintended_diff_only": 0.11767501085996628, + "tpp_threshold_100_total_metric": 0.17822501808404922, + "tpp_threshold_100_intended_diff_only": 0.32950002551078794, + "tpp_threshold_100_unintended_diff_only": 0.15127500742673874, + "tpp_threshold_500_total_metric": 0.23397501409053803, + "tpp_threshold_500_intended_diff_only": 0.43740002512931825, + "tpp_threshold_500_unintended_diff_only": 0.20342501103878022 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.08625001311302186, + "tpp_threshold_2_total_metric": 0.08535001575946807, "tpp_threshold_2_intended_diff_only": 0.10660002231597901, - "tpp_threshold_2_unintended_diff_only": 0.020350009202957153, - "tpp_threshold_5_total_metric": 0.1866000175476074, - "tpp_threshold_5_intended_diff_only": 0.24660003185272217, - "tpp_threshold_5_unintended_diff_only": 0.06000001430511474, - "tpp_threshold_10_total_metric": 0.20535000264644623, + "tpp_threshold_2_unintended_diff_only": 0.021250006556510926, + "tpp_threshold_5_total_metric": 0.1865000069141388, + "tpp_threshold_5_intended_diff_only": 0.24720002412796022, + "tpp_threshold_5_unintended_diff_only": 0.060700017213821414, + "tpp_threshold_10_total_metric": 0.20345000326633453, "tpp_threshold_10_intended_diff_only": 0.31500002145767214, - "tpp_threshold_10_unintended_diff_only": 0.1096500188112259, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_10_unintended_diff_only": 0.11155001819133759, + "tpp_threshold_20_total_metric": 0.2124500036239624, + "tpp_threshold_20_intended_diff_only": 0.3694000244140625, + "tpp_threshold_20_unintended_diff_only": 0.1569500207901001, + "tpp_threshold_50_total_metric": 0.1884000062942505, + "tpp_threshold_50_intended_diff_only": 0.4172000288963318, + "tpp_threshold_50_unintended_diff_only": 0.2288000226020813, + "tpp_threshold_100_total_metric": 0.15235000848770142, + "tpp_threshold_100_intended_diff_only": 0.44200003147125244, + "tpp_threshold_100_unintended_diff_only": 0.289650022983551, + "tpp_threshold_500_total_metric": 0.0913000077009201, + "tpp_threshold_500_intended_diff_only": 0.4674000382423401, + "tpp_threshold_500_unintended_diff_only": 0.37610003054142 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.008750009536743163, - "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": 0.00024999380111694335, - "tpp_threshold_5_total_metric": 0.01315000057220459, - "tpp_threshold_5_intended_diff_only": 0.015199995040893555, - "tpp_threshold_5_unintended_diff_only": 0.0020499944686889647, - "tpp_threshold_10_total_metric": 0.024399998784065246, - "tpp_threshold_10_intended_diff_only": 0.027199995517730714, - "tpp_threshold_10_unintended_diff_only": 0.0027999967336654664, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.00729999840259552, + "tpp_threshold_2_intended_diff_only": 0.008199989795684814, + "tpp_threshold_2_unintended_diff_only": 0.0008999913930892945, + "tpp_threshold_5_total_metric": 0.013300007581710816, + "tpp_threshold_5_intended_diff_only": 0.013400006294250488, + "tpp_threshold_5_unintended_diff_only": 9.999871253967285e-05, + "tpp_threshold_10_total_metric": 0.027400010824203493, + "tpp_threshold_10_intended_diff_only": 0.03000000715255737, + "tpp_threshold_10_unintended_diff_only": 0.0025999963283538817, + "tpp_threshold_20_total_metric": 0.05105000138282776, + "tpp_threshold_20_intended_diff_only": 0.054799997806549074, + "tpp_threshold_20_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_50_total_metric": 0.12305000126361847, + "tpp_threshold_50_intended_diff_only": 0.12960000038146974, + "tpp_threshold_50_unintended_diff_only": 0.006549999117851257, + "tpp_threshold_100_total_metric": 0.20410002768039703, + "tpp_threshold_100_intended_diff_only": 0.2170000195503235, + "tpp_threshold_100_unintended_diff_only": 0.012899991869926453, + "tpp_threshold_500_total_metric": 0.37665002048015594, + "tpp_threshold_500_intended_diff_only": 0.4074000120162964, + "tpp_threshold_500_unintended_diff_only": 0.030749991536140442 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.018750011920928955, "tpp_threshold_10_total_metric": 0.18799994885921478, "tpp_threshold_10_intended_diff_only": 0.21999996900558472, - "tpp_threshold_10_unintended_diff_only": 0.032000020146369934 + "tpp_threshold_10_unintended_diff_only": 0.032000020146369934, + "tpp_threshold_20_total_metric": 0.21149998903274536, + "tpp_threshold_20_intended_diff_only": 0.35500001907348633, + "tpp_threshold_20_unintended_diff_only": 0.14350003004074097, + "tpp_threshold_50_total_metric": 0.18724992871284485, + "tpp_threshold_50_intended_diff_only": 0.4139999747276306, + "tpp_threshold_50_unintended_diff_only": 0.22675004601478577, + "tpp_threshold_100_total_metric": 0.14524996280670166, + "tpp_threshold_100_intended_diff_only": 0.4359999895095825, + "tpp_threshold_100_unintended_diff_only": 0.29075002670288086, + "tpp_threshold_500_total_metric": 0.08874998986721039, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.3672500401735306 }, "1": { - "tpp_threshold_2_total_metric": 0.1820000261068344, - "tpp_threshold_2_intended_diff_only": 0.2510000467300415, - "tpp_threshold_2_unintended_diff_only": 0.06900002062320709, - "tpp_threshold_5_total_metric": 0.1885000318288803, + "tpp_threshold_2_total_metric": 0.18300001323223114, + "tpp_threshold_2_intended_diff_only": 0.24800002574920654, + "tpp_threshold_2_unintended_diff_only": 0.0650000125169754, + "tpp_threshold_5_total_metric": 0.18825002014636993, "tpp_threshold_5_intended_diff_only": 0.26500004529953003, - "tpp_threshold_5_unintended_diff_only": 0.07650001347064972, - "tpp_threshold_10_total_metric": 0.21524998545646667, + "tpp_threshold_5_unintended_diff_only": 0.0767500251531601, + "tpp_threshold_10_total_metric": 0.2095000147819519, "tpp_threshold_10_intended_diff_only": 0.3320000171661377, - "tpp_threshold_10_unintended_diff_only": 0.11675003170967102 + "tpp_threshold_10_unintended_diff_only": 0.12250000238418579, + "tpp_threshold_20_total_metric": 0.20500001311302185, + "tpp_threshold_20_intended_diff_only": 0.3540000319480896, + "tpp_threshold_20_unintended_diff_only": 0.14900001883506775, + "tpp_threshold_50_total_metric": 0.19350001215934753, + "tpp_threshold_50_intended_diff_only": 0.3960000276565552, + "tpp_threshold_50_unintended_diff_only": 0.20250001549720764, + "tpp_threshold_100_total_metric": 0.1745000034570694, + "tpp_threshold_100_intended_diff_only": 0.437000036239624, + "tpp_threshold_100_unintended_diff_only": 0.2625000327825546, + "tpp_threshold_500_total_metric": 0.08075001835823059, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.37825003266334534 }, "2": { - "tpp_threshold_2_total_metric": 0.1340000480413437, - "tpp_threshold_2_intended_diff_only": 0.15800005197525024, - "tpp_threshold_2_unintended_diff_only": 0.024000003933906555, - "tpp_threshold_5_total_metric": 0.19050003588199615, - "tpp_threshold_5_intended_diff_only": 0.23200005292892456, - "tpp_threshold_5_unintended_diff_only": 0.041500017046928406, - "tpp_threshold_10_total_metric": 0.1575000137090683, - "tpp_threshold_10_intended_diff_only": 0.3400000333786011, - "tpp_threshold_10_unintended_diff_only": 0.18250001966953278 + "tpp_threshold_2_total_metric": 0.12850002944469452, + "tpp_threshold_2_intended_diff_only": 0.15900003910064697, + "tpp_threshold_2_unintended_diff_only": 0.030500009655952454, + "tpp_threshold_5_total_metric": 0.1912500262260437, + "tpp_threshold_5_intended_diff_only": 0.2330000400543213, + "tpp_threshold_5_unintended_diff_only": 0.04175001382827759, + "tpp_threshold_10_total_metric": 0.15574999153614044, + "tpp_threshold_10_intended_diff_only": 0.3410000205039978, + "tpp_threshold_10_unintended_diff_only": 0.18525002896785736, + "tpp_threshold_20_total_metric": 0.13650000095367432, + "tpp_threshold_20_intended_diff_only": 0.36000001430511475, + "tpp_threshold_20_unintended_diff_only": 0.22350001335144043, + "tpp_threshold_50_total_metric": 0.09500002861022949, + "tpp_threshold_50_intended_diff_only": 0.3850000500679016, + "tpp_threshold_50_unintended_diff_only": 0.2900000214576721, + "tpp_threshold_100_total_metric": 0.0700000524520874, + "tpp_threshold_100_intended_diff_only": 0.4070000648498535, + "tpp_threshold_100_unintended_diff_only": 0.3370000123977661, + "tpp_threshold_500_total_metric": 0.0495000034570694, + "tpp_threshold_500_intended_diff_only": 0.4520000219345093, + "tpp_threshold_500_unintended_diff_only": 0.4025000184774399 }, "6": { - "tpp_threshold_2_total_metric": 0.011749997735023499, + "tpp_threshold_2_total_metric": 0.009749993681907654, "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": -0.002749994397163391, - "tpp_threshold_5_total_metric": 0.2600000500679016, - "tpp_threshold_5_intended_diff_only": 0.3380000591278076, - "tpp_threshold_5_unintended_diff_only": 0.078000009059906, - "tpp_threshold_10_total_metric": 0.27775004506111145, + "tpp_threshold_2_unintended_diff_only": -0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.26350001990795135, + "tpp_threshold_5_intended_diff_only": 0.33900004625320435, + "tpp_threshold_5_unintended_diff_only": 0.07550002634525299, + "tpp_threshold_10_total_metric": 0.27900004386901855, "tpp_threshold_10_intended_diff_only": 0.3980000615119934, - "tpp_threshold_10_unintended_diff_only": 0.12025001645088196 + "tpp_threshold_10_unintended_diff_only": 0.11900001764297485, + "tpp_threshold_20_total_metric": 0.28725001215934753, + "tpp_threshold_20_intended_diff_only": 0.45100003480911255, + "tpp_threshold_20_unintended_diff_only": 0.16375002264976501, + "tpp_threshold_50_total_metric": 0.24950005114078522, + "tpp_threshold_50_intended_diff_only": 0.48100006580352783, + "tpp_threshold_50_unintended_diff_only": 0.23150001466274261, + "tpp_threshold_100_total_metric": 0.19350002706050873, + "tpp_threshold_100_intended_diff_only": 0.4830000400543213, + "tpp_threshold_100_unintended_diff_only": 0.28950001299381256, + "tpp_threshold_500_total_metric": 0.12299998104572296, + "tpp_threshold_500_intended_diff_only": 0.49400001764297485, + "tpp_threshold_500_unintended_diff_only": 0.3710000365972519 }, "9": { - "tpp_threshold_2_total_metric": 0.003999993205070496, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_5_total_metric": 0.16175001859664917, - "tpp_threshold_5_intended_diff_only": 0.24700003862380981, - "tpp_threshold_5_unintended_diff_only": 0.08525002002716064, - "tpp_threshold_10_total_metric": 0.18825002014636993, - "tpp_threshold_10_intended_diff_only": 0.2850000262260437, - "tpp_threshold_10_unintended_diff_only": 0.09675000607967377 + "tpp_threshold_2_total_metric": 0.006000041961669922, + "tpp_threshold_2_intended_diff_only": 0.00700002908706665, + "tpp_threshold_2_unintended_diff_only": 0.0009999871253967285, + "tpp_threshold_5_total_metric": 0.15725001692771912, + "tpp_threshold_5_intended_diff_only": 0.24800002574920654, + "tpp_threshold_5_unintended_diff_only": 0.09075000882148743, + "tpp_threshold_10_total_metric": 0.18500001728534698, + "tpp_threshold_10_intended_diff_only": 0.284000039100647, + "tpp_threshold_10_unintended_diff_only": 0.09900002181529999, + "tpp_threshold_20_total_metric": 0.22200000286102295, + "tpp_threshold_20_intended_diff_only": 0.3270000219345093, + "tpp_threshold_20_unintended_diff_only": 0.10500001907348633, + "tpp_threshold_50_total_metric": 0.21675001084804535, + "tpp_threshold_50_intended_diff_only": 0.4100000262260437, + "tpp_threshold_50_unintended_diff_only": 0.19325001537799835, + "tpp_threshold_100_total_metric": 0.1784999966621399, + "tpp_threshold_100_intended_diff_only": 0.44700002670288086, + "tpp_threshold_100_unintended_diff_only": 0.26850003004074097, + "tpp_threshold_500_total_metric": 0.11450004577636719, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.3615000247955322 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.008000031113624573, - "tpp_threshold_2_intended_diff_only": 0.003000020980834961, - "tpp_threshold_2_unintended_diff_only": -0.005000010132789612, - "tpp_threshold_5_total_metric": 0.008499979972839355, - "tpp_threshold_5_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_5_unintended_diff_only": -0.002499997615814209, - "tpp_threshold_10_total_metric": 0.0014999806880950928, - "tpp_threshold_10_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_10_unintended_diff_only": 0.004500001668930054 + "tpp_threshold_2_total_metric": 0.002499997615814209, + "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_2_unintended_diff_only": 0.0034999847412109375, + "tpp_threshold_5_total_metric": 0.0022500157356262207, + "tpp_threshold_5_intended_diff_only": 0.003000020980834961, + "tpp_threshold_5_unintended_diff_only": 0.0007500052452087402, + "tpp_threshold_10_total_metric": 0.0029999911785125732, + "tpp_threshold_10_intended_diff_only": 0.004999995231628418, + "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447, + "tpp_threshold_20_total_metric": 0.01199999451637268, + "tpp_threshold_20_intended_diff_only": 0.013999998569488525, + "tpp_threshold_20_unintended_diff_only": 0.0020000040531158447, + "tpp_threshold_50_total_metric": 0.036500006914138794, + "tpp_threshold_50_intended_diff_only": 0.03600001335144043, + "tpp_threshold_50_unintended_diff_only": -0.0004999935626983643, + "tpp_threshold_100_total_metric": 0.09850002825260162, + "tpp_threshold_100_intended_diff_only": 0.1040000319480896, + "tpp_threshold_100_unintended_diff_only": 0.005500003695487976, + "tpp_threshold_500_total_metric": 0.4007500559091568, + "tpp_threshold_500_intended_diff_only": 0.4140000343322754, + "tpp_threshold_500_unintended_diff_only": 0.013249978423118591 }, "2": { - "tpp_threshold_2_total_metric": 0.0022500455379486084, - "tpp_threshold_2_intended_diff_only": 0.00700002908706665, - "tpp_threshold_2_unintended_diff_only": 0.004749983549118042, - "tpp_threshold_5_total_metric": 0.004500061273574829, - "tpp_threshold_5_intended_diff_only": 0.01100003719329834, - "tpp_threshold_5_unintended_diff_only": 0.006499975919723511, - "tpp_threshold_10_total_metric": 0.027250036597251892, - "tpp_threshold_10_intended_diff_only": 0.030000030994415283, - "tpp_threshold_10_unintended_diff_only": 0.002749994397163391 + "tpp_threshold_2_total_metric": 0.011250019073486328, + "tpp_threshold_2_intended_diff_only": 0.009000003337860107, + "tpp_threshold_2_unintended_diff_only": -0.0022500157356262207, + "tpp_threshold_5_total_metric": 0.006750032305717468, + "tpp_threshold_5_intended_diff_only": 0.013000011444091797, + "tpp_threshold_5_unintended_diff_only": 0.006249979138374329, + "tpp_threshold_10_total_metric": 0.028500065207481384, + "tpp_threshold_10_intended_diff_only": 0.033000051975250244, + "tpp_threshold_10_unintended_diff_only": 0.00449998676776886, + "tpp_threshold_20_total_metric": 0.08800001442432404, + "tpp_threshold_20_intended_diff_only": 0.09200000762939453, + "tpp_threshold_20_unintended_diff_only": 0.003999993205070496, + "tpp_threshold_50_total_metric": 0.16875003278255463, + "tpp_threshold_50_intended_diff_only": 0.18000000715255737, + "tpp_threshold_50_unintended_diff_only": 0.011249974370002747, + "tpp_threshold_100_total_metric": 0.26725006103515625, + "tpp_threshold_100_intended_diff_only": 0.28300005197525024, + "tpp_threshold_100_unintended_diff_only": 0.015749990940093994, + "tpp_threshold_500_total_metric": 0.4047500491142273, + "tpp_threshold_500_intended_diff_only": 0.437000036239624, + "tpp_threshold_500_unintended_diff_only": 0.03224998712539673 }, "3": { - "tpp_threshold_2_total_metric": 0.01400001347064972, - "tpp_threshold_2_intended_diff_only": 0.008000016212463379, - "tpp_threshold_2_unintended_diff_only": -0.00599999725818634, - "tpp_threshold_5_total_metric": 0.017999976873397827, - "tpp_threshold_5_intended_diff_only": 0.015999972820281982, - "tpp_threshold_5_unintended_diff_only": -0.0020000040531158447, - "tpp_threshold_10_total_metric": 0.020750001072883606, - "tpp_threshold_10_intended_diff_only": 0.023000001907348633, - "tpp_threshold_10_unintended_diff_only": 0.002250000834465027 + "tpp_threshold_2_total_metric": 0.003750026226043701, + "tpp_threshold_2_intended_diff_only": 0.003000020980834961, + "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402, + "tpp_threshold_5_total_metric": 0.013500005006790161, + "tpp_threshold_5_intended_diff_only": 0.008000016212463379, + "tpp_threshold_5_unintended_diff_only": -0.005499988794326782, + "tpp_threshold_10_total_metric": 0.02124999463558197, + "tpp_threshold_10_intended_diff_only": 0.018999993801116943, + "tpp_threshold_10_unintended_diff_only": -0.002250000834465027, + "tpp_threshold_20_total_metric": 0.010000035166740417, + "tpp_threshold_20_intended_diff_only": 0.012000024318695068, + "tpp_threshold_20_unintended_diff_only": 0.001999989151954651, + "tpp_threshold_50_total_metric": 0.06225000321865082, + "tpp_threshold_50_intended_diff_only": 0.06400001049041748, + "tpp_threshold_50_unintended_diff_only": 0.0017500072717666626, + "tpp_threshold_100_total_metric": 0.12650001049041748, + "tpp_threshold_100_intended_diff_only": 0.14300000667572021, + "tpp_threshold_100_unintended_diff_only": 0.016499996185302734, + "tpp_threshold_500_total_metric": 0.3659999966621399, + "tpp_threshold_500_intended_diff_only": 0.39399999380111694, + "tpp_threshold_500_unintended_diff_only": 0.02799999713897705 }, "5": { - "tpp_threshold_2_total_metric": -0.003750041127204895, + "tpp_threshold_2_total_metric": 0.004249945282936096, "tpp_threshold_2_intended_diff_only": 0.003999948501586914, - "tpp_threshold_2_unintended_diff_only": 0.007749989628791809, - "tpp_threshold_5_total_metric": 0.0057499706745147705, - "tpp_threshold_5_intended_diff_only": 0.006999969482421875, - "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045, - "tpp_threshold_10_total_metric": 0.018249958753585815, - "tpp_threshold_10_intended_diff_only": 0.02199995517730713, - "tpp_threshold_10_unintended_diff_only": 0.0037499964237213135 + "tpp_threshold_2_unintended_diff_only": -0.00024999678134918213, + "tpp_threshold_5_total_metric": 0.011499971151351929, + "tpp_threshold_5_intended_diff_only": 0.010999977588653564, + "tpp_threshold_5_unintended_diff_only": -0.0004999935626983643, + "tpp_threshold_10_total_metric": 0.016250014305114746, + "tpp_threshold_10_intended_diff_only": 0.023000001907348633, + "tpp_threshold_10_unintended_diff_only": 0.006749987602233887, + "tpp_threshold_20_total_metric": 0.04149998724460602, + "tpp_threshold_20_intended_diff_only": 0.042999982833862305, + "tpp_threshold_20_unintended_diff_only": 0.0014999955892562866, + "tpp_threshold_50_total_metric": 0.15074999630451202, + "tpp_threshold_50_intended_diff_only": 0.16200000047683716, + "tpp_threshold_50_unintended_diff_only": 0.011250004172325134, + "tpp_threshold_100_total_metric": 0.24775002896785736, + "tpp_threshold_100_intended_diff_only": 0.2630000114440918, + "tpp_threshold_100_unintended_diff_only": 0.015249982476234436, + "tpp_threshold_500_total_metric": 0.37599997222423553, + "tpp_threshold_500_intended_diff_only": 0.41899996995925903, + "tpp_threshold_500_unintended_diff_only": 0.0429999977350235 }, "6": { - "tpp_threshold_2_total_metric": 0.023249998688697815, - "tpp_threshold_2_intended_diff_only": 0.023000001907348633, - "tpp_threshold_2_unintended_diff_only": -0.00024999678134918213, - "tpp_threshold_5_total_metric": 0.029000014066696167, - "tpp_threshold_5_intended_diff_only": 0.03600001335144043, - "tpp_threshold_5_unintended_diff_only": 0.006999999284744263, - "tpp_threshold_10_total_metric": 0.05425001680850983, - "tpp_threshold_10_intended_diff_only": 0.05500000715255737, - "tpp_threshold_10_unintended_diff_only": 0.0007499903440475464 + "tpp_threshold_2_total_metric": 0.014750003814697266, + "tpp_threshold_2_intended_diff_only": 0.018999993801116943, + "tpp_threshold_2_unintended_diff_only": 0.004249989986419678, + "tpp_threshold_5_total_metric": 0.0325000137090683, + "tpp_threshold_5_intended_diff_only": 0.03200000524520874, + "tpp_threshold_5_unintended_diff_only": -0.0005000084638595581, + "tpp_threshold_10_total_metric": 0.06799998879432678, + "tpp_threshold_10_intended_diff_only": 0.06999999284744263, + "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447, + "tpp_threshold_20_total_metric": 0.10374997556209564, + "tpp_threshold_20_intended_diff_only": 0.11299997568130493, + "tpp_threshold_20_unintended_diff_only": 0.00925000011920929, + "tpp_threshold_50_total_metric": 0.19699996709823608, + "tpp_threshold_50_intended_diff_only": 0.2059999704360962, + "tpp_threshold_50_unintended_diff_only": 0.009000003337860107, + "tpp_threshold_100_total_metric": 0.28050000965595245, + "tpp_threshold_100_intended_diff_only": 0.2919999957084656, + "tpp_threshold_100_unintended_diff_only": 0.011499986052513123, + "tpp_threshold_500_total_metric": 0.3357500284910202, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.03724999725818634 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-292968-tokens-1200000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-292968-tokens-1200000000/scr/saebench_scr_custom_sae_eval_results.json index a73be9c74df0fe0e27d30d8b9f4f7860e4a8339e..a6ba13fb58372be49565b19214034a2c93e83c8f 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-292968-tokens-1200000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-292968-tokens-1200000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "0bf9147d-f57c-4585-96a0-148907394839", - "datetime_epoch_millis": 1776885956411, + "eval_id": "a5f0acc4-c047-4749-a48d-7e4d190221e6", + "datetime_epoch_millis": 1777317150738, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.2829853198662318, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.27287744671552183, "scr_metric_threshold_10": 0.19882804859098052, "scr_dir2_threshold_10": 0.18966515684829438, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2991269431541925, + "scr_metric_threshold_20": 0.21957360880872895, + "scr_dir2_threshold_20": 0.20910587974160583, + "scr_dir1_threshold_50": 0.15900130660025805, + "scr_metric_threshold_50": 0.26660734469158326, + "scr_dir2_threshold_50": 0.253608829247956, + "scr_dir1_threshold_100": -0.04134676619549471, + "scr_metric_threshold_100": 0.26784659140862055, + "scr_dir2_threshold_100": 0.25048102609187584, + "scr_dir1_threshold_500": -1.076120057194125, + "scr_metric_threshold_500": 0.2294901639413055, + "scr_dir2_threshold_500": 0.227914380472892 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.43589788655709094, "scr_metric_threshold_10": 0.03367878287944761, "scr_dir2_threshold_10": 0.03367878287944761, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.24359039998532808, + "scr_metric_threshold_20": 0.16580312000946262, + "scr_dir2_threshold_20": 0.16580312000946262, + "scr_dir1_threshold_50": 0.6025646805840948, + "scr_metric_threshold_50": 0.25906739431680575, + "scr_dir2_threshold_50": 0.25906739431680575, + "scr_dir1_threshold_100": 0.0, + "scr_metric_threshold_100": 0.2461140400425818, + "scr_dir2_threshold_100": 0.2461140400425818, + "scr_dir1_threshold_500": -3.0384610388171382, + "scr_metric_threshold_500": 0.3108808114137016, + "scr_dir2_threshold_500": 0.3108808114137016 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.3484848005931564, "scr_metric_threshold_10": 0.25688069046599166, "scr_dir2_threshold_10": 0.25688069046599166, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.38636354400108736, + "scr_metric_threshold_20": 0.14373084950283896, + "scr_dir2_threshold_20": 0.14373084950283896, + "scr_dir1_threshold_50": 0.1439393152601855, + "scr_metric_threshold_50": 0.21406724205499303, + "scr_dir2_threshold_50": 0.21406724205499303, + "scr_dir1_threshold_100": -0.07575748681586188, + "scr_metric_threshold_100": 0.3944954111717689, + "scr_dir2_threshold_100": 0.3944954111717689, + "scr_dir1_threshold_500": -0.8484845748180365, + "scr_metric_threshold_500": 0.13761472070577727, + "scr_dir2_threshold_500": 0.13761472070577727 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.5, "scr_metric_threshold_10": 0.11055282101401862, "scr_dir2_threshold_10": 0.11055282101401862, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.4062502910382569, + "scr_metric_threshold_20": 0.17085429167603294, + "scr_dir2_threshold_20": 0.17085429167603294, + "scr_dir1_threshold_50": -0.7187499417923486, + "scr_metric_threshold_50": 0.16582923152102794, + "scr_dir2_threshold_50": 0.16582923152102794, + "scr_dir1_threshold_100": -0.3437501746229541, + "scr_metric_threshold_100": 0.05276388042950681, + "scr_dir2_threshold_100": 0.05276388042950681, + "scr_dir1_threshold_500": -4.124999301508184, + "scr_metric_threshold_500": 0.09799002086611526, + "scr_dir2_threshold_500": 0.09799002086611526 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": -0.1543624402703411, "scr_metric_threshold_10": 0.06687906673556002, "scr_dir2_threshold_10": 0.06687906673556002, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.11409369930854607, + "scr_metric_threshold_20": -0.10191084132522761, + "scr_dir2_threshold_20": -0.10191084132522761, + "scr_dir1_threshold_50": 0.17449641072003616, + "scr_metric_threshold_50": -0.11464972021884395, + "scr_dir2_threshold_50": -0.11464972021884395, + "scr_dir1_threshold_100": -1.0536913879282583, + "scr_metric_threshold_100": -0.2388534572401228, + "scr_dir2_threshold_100": -0.2388534572401228, + "scr_dir1_threshold_500": -1.8389266362776298, + "scr_metric_threshold_500": -0.5222928956960512, + "scr_dir2_threshold_500": -0.5222928956960512 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.08421069803000372, "scr_metric_threshold_10": 0.45238098053884956, "scr_dir2_threshold_10": 0.45238098053884956, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.11578955293692479, + "scr_metric_threshold_20": 0.543650817115708, + "scr_dir2_threshold_20": 0.543650817115708, + "scr_dir1_threshold_50": -0.15263149804355594, + "scr_metric_threshold_50": 0.6587301042915575, + "scr_dir2_threshold_50": 0.6587301042915575, + "scr_dir1_threshold_100": -0.15263149804355594, + "scr_metric_threshold_100": 0.6785713525451061, + "scr_dir2_threshold_100": 0.6785713525451061, + "scr_dir1_threshold_500": -0.19999993725826787, + "scr_metric_threshold_500": 0.773809391467407, + "scr_dir2_threshold_500": 0.773809391467407 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.4153004823742847, "scr_metric_threshold_10": 0.1183206688641517, "scr_dir2_threshold_10": 0.1183206688641517, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.4153004823742847, + "scr_metric_threshold_20": 0.17557251734733875, + "scr_dir2_threshold_20": 0.17557251734733875, + "scr_dir1_threshold_50": 0.4316940424658739, + "scr_metric_threshold_50": 0.26335877602100816, + "scr_dir2_threshold_50": 0.26335877602100816, + "scr_dir1_threshold_100": 0.5136611915068252, + "scr_metric_threshold_100": 0.3396945739985909, + "scr_dir2_threshold_100": 0.3396945739985909, + "scr_dir1_threshold_500": 0.5628415460730956, + "scr_metric_threshold_500": 0.29389318621149046, + "scr_dir2_threshold_500": 0.29389318621149046 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.3534882018861679, "scr_metric_threshold_10": 0.35193143367601315, "scr_dir2_threshold_10": 0.35193143367601315, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.46046493833553914, + "scr_metric_threshold_20": 0.40772547889010513, + "scr_dir2_threshold_20": 0.40772547889010513, + "scr_dir1_threshold_50": 0.46046493833553914, + "scr_metric_threshold_50": 0.3562232242738785, + "scr_dir2_threshold_50": 0.3562232242738785, + "scr_dir1_threshold_100": 0.49767431029119774, + "scr_metric_threshold_100": 0.3862660142728834, + "scr_dir2_threshold_100": 0.3862660142728834, + "scr_dir1_threshold_500": 0.5813952585759982, + "scr_metric_threshold_500": 0.4463518500848407, + "scr_dir2_threshold_500": 0.4463518500848407 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.19999994455381204, "scr_metric_threshold_10": 0.19999994455381204, "scr_dir2_threshold_10": 0.12669681061232305, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2511626372535729, + "scr_metric_threshold_20": 0.2511626372535729, + "scr_dir2_threshold_20": 0.16742080471658802, + "scr_dir1_threshold_50": 0.3302325052722394, + "scr_metric_threshold_50": 0.3302325052722394, + "scr_dir2_threshold_50": 0.2262443817232214, + "scr_dir1_threshold_100": 0.2837209160486496, + "scr_metric_threshold_100": 0.2837209160486496, + "scr_dir2_threshold_100": 0.14479639351469145, + "scr_dir1_threshold_500": 0.2976742264771627, + "scr_metric_threshold_500": 0.2976742264771627, + "scr_dir2_threshold_500": 0.2850679587298548 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-292968-tokens-1200000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-292968-tokens-1200000000/tpp/saebench_tpp_custom_sae_eval_results.json index 4321305156445cf05281884f9036753c5bb7469a..e9f88c504fcb29ef9b3185299ac3b25ceeea37ae 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-292968-tokens-1200000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-292968-tokens-1200000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "dfa66f69-74fd-4bcc-9a66-73be067e304e", - "datetime_epoch_millis": 1776886123512, + "eval_id": "7b752ed9-5a56-4940-9c78-a1d00014088e", + "datetime_epoch_millis": 1777317425795, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.034550003707408905, - "tpp_threshold_2_intended_diff_only": 0.039000004529953, - "tpp_threshold_2_unintended_diff_only": 0.004450000822544098, - "tpp_threshold_5_total_metric": 0.07345001846551895, - "tpp_threshold_5_intended_diff_only": 0.09270002245903015, - "tpp_threshold_5_unintended_diff_only": 0.019250003993511202, - "tpp_threshold_10_total_metric": 0.10297500342130661, - "tpp_threshold_10_intended_diff_only": 0.15680001378059386, - "tpp_threshold_10_unintended_diff_only": 0.053825010359287266, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.03327499777078629, + "tpp_threshold_2_intended_diff_only": 0.03860000371932983, + "tpp_threshold_2_unintended_diff_only": 0.005325005948543548, + "tpp_threshold_5_total_metric": 0.07347500324249268, + "tpp_threshold_5_intended_diff_only": 0.09190000891685485, + "tpp_threshold_5_unintended_diff_only": 0.018425005674362182, + "tpp_threshold_10_total_metric": 0.10349999815225601, + "tpp_threshold_10_intended_diff_only": 0.15780000686645507, + "tpp_threshold_10_unintended_diff_only": 0.054300008714199065, + "tpp_threshold_20_total_metric": 0.1278250113129616, + "tpp_threshold_20_intended_diff_only": 0.20930001735687256, + "tpp_threshold_20_unintended_diff_only": 0.08147500604391099, + "tpp_threshold_50_total_metric": 0.14660000950098037, + "tpp_threshold_50_intended_diff_only": 0.2668000221252442, + "tpp_threshold_50_unintended_diff_only": 0.12020001262426376, + "tpp_threshold_100_total_metric": 0.16400000900030137, + "tpp_threshold_100_intended_diff_only": 0.31930001974105837, + "tpp_threshold_100_unintended_diff_only": 0.15530001074075697, + "tpp_threshold_500_total_metric": 0.23262501657009124, + "tpp_threshold_500_intended_diff_only": 0.4371000289916992, + "tpp_threshold_500_unintended_diff_only": 0.20447501242160798 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.061400017142295836, - "tpp_threshold_2_intended_diff_only": 0.07020002603530884, - "tpp_threshold_2_unintended_diff_only": 0.008800008893013, - "tpp_threshold_5_total_metric": 0.1337500274181366, - "tpp_threshold_5_intended_diff_only": 0.17000004053115844, - "tpp_threshold_5_unintended_diff_only": 0.03625001311302185, - "tpp_threshold_10_total_metric": 0.1850000023841858, + "tpp_threshold_2_total_metric": 0.06065000891685486, + "tpp_threshold_2_intended_diff_only": 0.07060002088546753, + "tpp_threshold_2_unintended_diff_only": 0.009950011968612671, + "tpp_threshold_5_total_metric": 0.1334500193595886, + "tpp_threshold_5_intended_diff_only": 0.16980003118515014, + "tpp_threshold_5_unintended_diff_only": 0.03635001182556152, + "tpp_threshold_10_total_metric": 0.18390000164508818, "tpp_threshold_10_intended_diff_only": 0.28980002403259275, - "tpp_threshold_10_unintended_diff_only": 0.10480002164840699, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_10_unintended_diff_only": 0.10590002238750458, + "tpp_threshold_20_total_metric": 0.209250009059906, + "tpp_threshold_20_intended_diff_only": 0.3684000253677368, + "tpp_threshold_20_unintended_diff_only": 0.15915001630783082, + "tpp_threshold_50_total_metric": 0.18035001158714295, + "tpp_threshold_50_intended_diff_only": 0.4146000385284424, + "tpp_threshold_50_unintended_diff_only": 0.23425002694129943, + "tpp_threshold_100_total_metric": 0.14240001142024994, + "tpp_threshold_100_intended_diff_only": 0.44000003337860105, + "tpp_threshold_100_unintended_diff_only": 0.2976000219583511, + "tpp_threshold_500_total_metric": 0.09025001227855682, + "tpp_threshold_500_intended_diff_only": 0.4674000382423401, + "tpp_threshold_500_unintended_diff_only": 0.3771500259637833 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.007699990272521972, - "tpp_threshold_2_intended_diff_only": 0.007799983024597168, - "tpp_threshold_2_unintended_diff_only": 9.999275207519531e-05, - "tpp_threshold_5_total_metric": 0.013150009512901305, - "tpp_threshold_5_intended_diff_only": 0.015400004386901856, - "tpp_threshold_5_unintended_diff_only": 0.0022499948740005494, - "tpp_threshold_10_total_metric": 0.02095000445842743, - "tpp_threshold_10_intended_diff_only": 0.02380000352859497, - "tpp_threshold_10_unintended_diff_only": 0.0028499990701675417, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.005899986624717713, + "tpp_threshold_2_intended_diff_only": 0.006599986553192138, + "tpp_threshold_2_unintended_diff_only": 0.0006999999284744262, + "tpp_threshold_5_total_metric": 0.01349998712539673, + "tpp_threshold_5_intended_diff_only": 0.01399998664855957, + "tpp_threshold_5_unintended_diff_only": 0.0004999995231628418, + "tpp_threshold_10_total_metric": 0.023099994659423827, + "tpp_threshold_10_intended_diff_only": 0.025799989700317383, + "tpp_threshold_10_unintended_diff_only": 0.0026999950408935548, + "tpp_threshold_20_total_metric": 0.046400013566017154, + "tpp_threshold_20_intended_diff_only": 0.0502000093460083, + "tpp_threshold_20_unintended_diff_only": 0.00379999577999115, + "tpp_threshold_50_total_metric": 0.11285000741481781, + "tpp_threshold_50_intended_diff_only": 0.1190000057220459, + "tpp_threshold_50_unintended_diff_only": 0.006149998307228089, + "tpp_threshold_100_total_metric": 0.18560000658035278, + "tpp_threshold_100_intended_diff_only": 0.19860000610351564, + "tpp_threshold_100_unintended_diff_only": 0.012999999523162841, + "tpp_threshold_500_total_metric": 0.37500002086162565, + "tpp_threshold_500_intended_diff_only": 0.40680001974105834, + "tpp_threshold_500_unintended_diff_only": 0.031799998879432675 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.012250006198883057, "tpp_threshold_10_total_metric": 0.1407499760389328, "tpp_threshold_10_intended_diff_only": 0.15700000524520874, - "tpp_threshold_10_unintended_diff_only": 0.01625002920627594 + "tpp_threshold_10_unintended_diff_only": 0.01625002920627594, + "tpp_threshold_20_total_metric": 0.22074997425079346, + "tpp_threshold_20_intended_diff_only": 0.3610000014305115, + "tpp_threshold_20_unintended_diff_only": 0.14025002717971802, + "tpp_threshold_50_total_metric": 0.187749981880188, + "tpp_threshold_50_intended_diff_only": 0.41600000858306885, + "tpp_threshold_50_unintended_diff_only": 0.22825002670288086, + "tpp_threshold_100_total_metric": 0.13749994337558746, + "tpp_threshold_100_intended_diff_only": 0.43699997663497925, + "tpp_threshold_100_unintended_diff_only": 0.2995000332593918, + "tpp_threshold_500_total_metric": 0.08750000596046448, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.3685000240802765 }, "1": { - "tpp_threshold_2_total_metric": 0.046500012278556824, + "tpp_threshold_2_total_metric": 0.049250006675720215, "tpp_threshold_2_intended_diff_only": 0.058000028133392334, - "tpp_threshold_2_unintended_diff_only": 0.01150001585483551, - "tpp_threshold_5_total_metric": 0.1862500160932541, + "tpp_threshold_2_unintended_diff_only": 0.00875002145767212, + "tpp_threshold_5_total_metric": 0.1860000193119049, "tpp_threshold_5_intended_diff_only": 0.27000004053115845, - "tpp_threshold_5_unintended_diff_only": 0.08375002443790436, - "tpp_threshold_10_total_metric": 0.1875000149011612, + "tpp_threshold_5_unintended_diff_only": 0.08400002121925354, + "tpp_threshold_10_total_metric": 0.17925001680850983, "tpp_threshold_10_intended_diff_only": 0.31700003147125244, - "tpp_threshold_10_unintended_diff_only": 0.12950001657009125 + "tpp_threshold_10_unintended_diff_only": 0.13775001466274261, + "tpp_threshold_20_total_metric": 0.19500000774860382, + "tpp_threshold_20_intended_diff_only": 0.3500000238418579, + "tpp_threshold_20_unintended_diff_only": 0.1550000160932541, + "tpp_threshold_50_total_metric": 0.18150001764297485, + "tpp_threshold_50_intended_diff_only": 0.3850000500679016, + "tpp_threshold_50_unintended_diff_only": 0.20350003242492676, + "tpp_threshold_100_total_metric": 0.15974999964237213, + "tpp_threshold_100_intended_diff_only": 0.4240000247955322, + "tpp_threshold_100_unintended_diff_only": 0.2642500251531601, + "tpp_threshold_500_total_metric": 0.08050002157688141, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.3785000294446945 }, "2": { - "tpp_threshold_2_total_metric": 0.1585000604391098, + "tpp_threshold_2_total_metric": 0.15175004303455353, "tpp_threshold_2_intended_diff_only": 0.18500006198883057, - "tpp_threshold_2_unintended_diff_only": 0.026500001549720764, - "tpp_threshold_5_total_metric": 0.19850002229213715, - "tpp_threshold_5_intended_diff_only": 0.2420000433921814, - "tpp_threshold_5_unintended_diff_only": 0.04350002110004425, - "tpp_threshold_10_total_metric": 0.1470000147819519, - "tpp_threshold_10_intended_diff_only": 0.34400004148483276, - "tpp_threshold_10_unintended_diff_only": 0.19700002670288086 + "tpp_threshold_2_unintended_diff_only": 0.03325001895427704, + "tpp_threshold_5_total_metric": 0.19575004279613495, + "tpp_threshold_5_intended_diff_only": 0.24100005626678467, + "tpp_threshold_5_unintended_diff_only": 0.04525001347064972, + "tpp_threshold_10_total_metric": 0.14900000393390656, + "tpp_threshold_10_intended_diff_only": 0.3450000286102295, + "tpp_threshold_10_unintended_diff_only": 0.19600002467632294, + "tpp_threshold_20_total_metric": 0.1262499988079071, + "tpp_threshold_20_intended_diff_only": 0.36000001430511475, + "tpp_threshold_20_unintended_diff_only": 0.23375001549720764, + "tpp_threshold_50_total_metric": 0.08650003373622894, + "tpp_threshold_50_intended_diff_only": 0.3890000581741333, + "tpp_threshold_50_unintended_diff_only": 0.30250002443790436, + "tpp_threshold_100_total_metric": 0.06325003504753113, + "tpp_threshold_100_intended_diff_only": 0.40800005197525024, + "tpp_threshold_100_unintended_diff_only": 0.3447500169277191, + "tpp_threshold_500_total_metric": 0.04874999821186066, + "tpp_threshold_500_intended_diff_only": 0.4520000219345093, + "tpp_threshold_500_unintended_diff_only": 0.4032500237226486 }, "6": { - "tpp_threshold_2_total_metric": 0.011749997735023499, + "tpp_threshold_2_total_metric": 0.009749993681907654, "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": -0.002749994397163391, - "tpp_threshold_5_total_metric": 0.15700004994869232, - "tpp_threshold_5_intended_diff_only": 0.18100005388259888, - "tpp_threshold_5_unintended_diff_only": 0.024000003933906555, - "tpp_threshold_10_total_metric": 0.2667500078678131, + "tpp_threshold_2_unintended_diff_only": -0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.16225002706050873, + "tpp_threshold_5_intended_diff_only": 0.1820000410079956, + "tpp_threshold_5_unintended_diff_only": 0.019750013947486877, + "tpp_threshold_10_total_metric": 0.27025000751018524, "tpp_threshold_10_intended_diff_only": 0.3680000305175781, - "tpp_threshold_10_unintended_diff_only": 0.10125002264976501 + "tpp_threshold_10_unintended_diff_only": 0.09775002300739288, + "tpp_threshold_20_total_metric": 0.29975004494190216, + "tpp_threshold_20_intended_diff_only": 0.4450000524520874, + "tpp_threshold_20_unintended_diff_only": 0.14525000751018524, + "tpp_threshold_50_total_metric": 0.25325003266334534, + "tpp_threshold_50_intended_diff_only": 0.47700005769729614, + "tpp_threshold_50_unintended_diff_only": 0.2237500250339508, + "tpp_threshold_100_total_metric": 0.18200001120567322, + "tpp_threshold_100_intended_diff_only": 0.4830000400543213, + "tpp_threshold_100_unintended_diff_only": 0.30100002884864807, + "tpp_threshold_500_total_metric": 0.12299998104572296, + "tpp_threshold_500_intended_diff_only": 0.49400001764297485, + "tpp_threshold_500_unintended_diff_only": 0.3710000365972519 }, "9": { - "tpp_threshold_2_total_metric": 0.006000027060508728, - "tpp_threshold_2_intended_diff_only": 0.00700002908706665, - "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_5_total_metric": 0.029250040650367737, - "tpp_threshold_5_intended_diff_only": 0.04700005054473877, - "tpp_threshold_5_unintended_diff_only": 0.017750009894371033, - "tpp_threshold_10_total_metric": 0.18299999833106995, - "tpp_threshold_10_intended_diff_only": 0.2630000114440918, - "tpp_threshold_10_unintended_diff_only": 0.08000001311302185 + "tpp_threshold_2_total_metric": 0.008250012993812561, + "tpp_threshold_2_intended_diff_only": 0.009000003337860107, + "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.025499999523162842, + "tpp_threshold_5_intended_diff_only": 0.046000003814697266, + "tpp_threshold_5_unintended_diff_only": 0.020500004291534424, + "tpp_threshold_10_total_metric": 0.18025000393390656, + "tpp_threshold_10_intended_diff_only": 0.26200002431869507, + "tpp_threshold_10_unintended_diff_only": 0.08175002038478851, + "tpp_threshold_20_total_metric": 0.2045000195503235, + "tpp_threshold_20_intended_diff_only": 0.32600003480911255, + "tpp_threshold_20_unintended_diff_only": 0.12150001525878906, + "tpp_threshold_50_total_metric": 0.1927499920129776, + "tpp_threshold_50_intended_diff_only": 0.406000018119812, + "tpp_threshold_50_unintended_diff_only": 0.2132500261068344, + "tpp_threshold_100_total_metric": 0.16950006783008575, + "tpp_threshold_100_intended_diff_only": 0.44800007343292236, + "tpp_threshold_100_unintended_diff_only": 0.2785000056028366, + "tpp_threshold_500_total_metric": 0.11150005459785461, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.3645000159740448 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.011499986052513123, - "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_2_unintended_diff_only": -0.005500003695487976, - "tpp_threshold_5_total_metric": 0.01075001060962677, - "tpp_threshold_5_intended_diff_only": 0.008000016212463379, - "tpp_threshold_5_unintended_diff_only": -0.002749994397163391, - "tpp_threshold_10_total_metric": 0.002249985933303833, - "tpp_threshold_10_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_10_unintended_diff_only": 0.0037499964237213135 + "tpp_threshold_2_total_metric": 0.004249975085258484, + "tpp_threshold_2_intended_diff_only": 0.006999969482421875, + "tpp_threshold_2_unintended_diff_only": 0.002749994397163391, + "tpp_threshold_5_total_metric": 0.0037499964237213135, + "tpp_threshold_5_intended_diff_only": 0.004999995231628418, + "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_10_total_metric": 0.0037499964237213135, + "tpp_threshold_10_intended_diff_only": 0.004999995231628418, + "tpp_threshold_10_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_20_total_metric": 0.018000036478042603, + "tpp_threshold_20_intended_diff_only": 0.021000027656555176, + "tpp_threshold_20_unintended_diff_only": 0.0029999911785125732, + "tpp_threshold_50_total_metric": 0.06699997186660767, + "tpp_threshold_50_intended_diff_only": 0.06699997186660767, + "tpp_threshold_50_unintended_diff_only": 0.0, + "tpp_threshold_100_total_metric": 0.12275004386901855, + "tpp_threshold_100_intended_diff_only": 0.13200002908706665, + "tpp_threshold_100_unintended_diff_only": 0.009249985218048096, + "tpp_threshold_500_total_metric": 0.4110000282526016, + "tpp_threshold_500_intended_diff_only": 0.4280000329017639, + "tpp_threshold_500_unintended_diff_only": 0.017000004649162292 }, "2": { - "tpp_threshold_2_total_metric": 0.003250017762184143, - "tpp_threshold_2_intended_diff_only": 0.008000016212463379, - "tpp_threshold_2_unintended_diff_only": 0.004749998450279236, - "tpp_threshold_5_total_metric": 0.0102500319480896, - "tpp_threshold_5_intended_diff_only": 0.017000019550323486, + "tpp_threshold_2_total_metric": 0.011500000953674316, + "tpp_threshold_2_intended_diff_only": 0.009000003337860107, + "tpp_threshold_2_unintended_diff_only": -0.002499997615814209, + "tpp_threshold_5_total_metric": 0.007250010967254639, + "tpp_threshold_5_intended_diff_only": 0.013999998569488525, "tpp_threshold_5_unintended_diff_only": 0.006749987602233887, - "tpp_threshold_10_total_metric": 0.02600005269050598, - "tpp_threshold_10_intended_diff_only": 0.029000043869018555, - "tpp_threshold_10_unintended_diff_only": 0.0029999911785125732 + "tpp_threshold_10_total_metric": 0.02625003457069397, + "tpp_threshold_10_intended_diff_only": 0.03200000524520874, + "tpp_threshold_10_unintended_diff_only": 0.0057499706745147705, + "tpp_threshold_20_total_metric": 0.07225003838539124, + "tpp_threshold_20_intended_diff_only": 0.07700002193450928, + "tpp_threshold_20_unintended_diff_only": 0.004749983549118042, + "tpp_threshold_50_total_metric": 0.12525007128715515, + "tpp_threshold_50_intended_diff_only": 0.1390000581741333, + "tpp_threshold_50_unintended_diff_only": 0.01374998688697815, + "tpp_threshold_100_total_metric": 0.20775000751018524, + "tpp_threshold_100_intended_diff_only": 0.22600001096725464, + "tpp_threshold_100_unintended_diff_only": 0.018250003457069397, + "tpp_threshold_500_total_metric": 0.4060000777244568, + "tpp_threshold_500_intended_diff_only": 0.43500006198883057, + "tpp_threshold_500_unintended_diff_only": 0.02899998426437378 }, "3": { - "tpp_threshold_2_total_metric": 0.011250004172325134, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": -0.006250008940696716, - "tpp_threshold_5_total_metric": 0.014500021934509277, - "tpp_threshold_5_intended_diff_only": 0.013000011444091797, - "tpp_threshold_5_unintended_diff_only": -0.0015000104904174805, - "tpp_threshold_10_total_metric": 0.017250001430511475, - "tpp_threshold_10_intended_diff_only": 0.018999993801116943, - "tpp_threshold_10_unintended_diff_only": 0.0017499923706054688 + "tpp_threshold_2_total_metric": 0.0027499794960021973, + "tpp_threshold_2_intended_diff_only": 0.001999974250793457, + "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402, + "tpp_threshold_5_total_metric": 0.011750012636184692, + "tpp_threshold_5_intended_diff_only": 0.00700002908706665, + "tpp_threshold_5_unintended_diff_only": -0.004749983549118042, + "tpp_threshold_10_total_metric": 0.016749992966651917, + "tpp_threshold_10_intended_diff_only": 0.014999985694885254, + "tpp_threshold_10_unintended_diff_only": -0.0017500072717666626, + "tpp_threshold_20_total_metric": 0.010500028729438782, + "tpp_threshold_20_intended_diff_only": 0.012000024318695068, + "tpp_threshold_20_unintended_diff_only": 0.0014999955892562866, + "tpp_threshold_50_total_metric": 0.047249987721443176, + "tpp_threshold_50_intended_diff_only": 0.046999990940093994, + "tpp_threshold_50_unintended_diff_only": -0.00024999678134918213, + "tpp_threshold_100_total_metric": 0.1042499989271164, + "tpp_threshold_100_intended_diff_only": 0.11599999666213989, + "tpp_threshold_100_unintended_diff_only": 0.011749997735023499, + "tpp_threshold_500_total_metric": 0.34849999845027924, + "tpp_threshold_500_intended_diff_only": 0.3799999952316284, + "tpp_threshold_500_unintended_diff_only": 0.03149999678134918 }, "5": { - "tpp_threshold_2_total_metric": -0.003500044345855713, - "tpp_threshold_2_intended_diff_only": 0.003999948501586914, - "tpp_threshold_2_unintended_diff_only": 0.007499992847442627, - "tpp_threshold_5_total_metric": 0.009999975562095642, - "tpp_threshold_5_intended_diff_only": 0.011999964714050293, - "tpp_threshold_5_unintended_diff_only": 0.001999989151954651, - "tpp_threshold_10_total_metric": 0.015499979257583618, - "tpp_threshold_10_intended_diff_only": 0.018999993801116943, - "tpp_threshold_10_unintended_diff_only": 0.003500014543533325 + "tpp_threshold_2_total_metric": 0.004999980330467224, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 1.4901161193847656e-08, + "tpp_threshold_5_total_metric": 0.018749937415122986, + "tpp_threshold_5_intended_diff_only": 0.01799994707107544, + "tpp_threshold_5_unintended_diff_only": -0.0007499903440475464, + "tpp_threshold_10_total_metric": 0.011249959468841553, + "tpp_threshold_10_intended_diff_only": 0.01699995994567871, + "tpp_threshold_10_unintended_diff_only": 0.005750000476837158, + "tpp_threshold_20_total_metric": 0.03449995815753937, + "tpp_threshold_20_intended_diff_only": 0.034999966621398926, + "tpp_threshold_20_unintended_diff_only": 0.0005000084638595581, + "tpp_threshold_50_total_metric": 0.1274999976158142, + "tpp_threshold_50_intended_diff_only": 0.13899999856948853, + "tpp_threshold_50_unintended_diff_only": 0.011500000953674316, + "tpp_threshold_100_total_metric": 0.23149998486042023, + "tpp_threshold_100_intended_diff_only": 0.2459999918937683, + "tpp_threshold_100_unintended_diff_only": 0.014500007033348083, + "tpp_threshold_500_total_metric": 0.3727499693632126, + "tpp_threshold_500_intended_diff_only": 0.4179999828338623, + "tpp_threshold_500_unintended_diff_only": 0.04525001347064972 }, "6": { - "tpp_threshold_2_total_metric": 0.015999987721443176, - "tpp_threshold_2_intended_diff_only": 0.015999972820281982, - "tpp_threshold_2_unintended_diff_only": -1.4901161193847656e-08, - "tpp_threshold_5_total_metric": 0.02025000751018524, - "tpp_threshold_5_intended_diff_only": 0.027000010013580322, - "tpp_threshold_5_unintended_diff_only": 0.006750002503395081, - "tpp_threshold_10_total_metric": 0.04375000298023224, - "tpp_threshold_10_intended_diff_only": 0.046000003814697266, - "tpp_threshold_10_unintended_diff_only": 0.002250000834465027 + "tpp_threshold_2_total_metric": 0.00599999725818634, + "tpp_threshold_2_intended_diff_only": 0.009999990463256836, + "tpp_threshold_2_unintended_diff_only": 0.003999993205070496, + "tpp_threshold_5_total_metric": 0.025999978184700012, + "tpp_threshold_5_intended_diff_only": 0.02599996328353882, + "tpp_threshold_5_unintended_diff_only": -1.4901161193847656e-08, + "tpp_threshold_10_total_metric": 0.05749998986721039, + "tpp_threshold_10_intended_diff_only": 0.06000000238418579, + "tpp_threshold_10_unintended_diff_only": 0.002500012516975403, + "tpp_threshold_20_total_metric": 0.09675000607967377, + "tpp_threshold_20_intended_diff_only": 0.10600000619888306, + "tpp_threshold_20_unintended_diff_only": 0.00925000011920929, + "tpp_threshold_50_total_metric": 0.19725000858306885, + "tpp_threshold_50_intended_diff_only": 0.203000009059906, + "tpp_threshold_50_unintended_diff_only": 0.005750000476837158, + "tpp_threshold_100_total_metric": 0.2617499977350235, + "tpp_threshold_100_intended_diff_only": 0.27300000190734863, + "tpp_threshold_100_unintended_diff_only": 0.011250004172325134, + "tpp_threshold_500_total_metric": 0.3367500305175781, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.03624999523162842 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-317382-tokens-1300000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-317382-tokens-1300000000/scr/saebench_scr_custom_sae_eval_results.json index 204c6202aa0706121f778a5006d68b51b658f6c4..013581341062b50b781ee8b861e996313dbcf6e9 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-317382-tokens-1300000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-317382-tokens-1300000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "f10c176b-9b1c-4898-b453-e83918863c29", - "datetime_epoch_millis": 1776885910506, + "eval_id": "7e261c04-6bb0-4026-8d96-74269cddf950", + "datetime_epoch_millis": 1777317397195, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.2885073924088952, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.2709502606944876, "scr_metric_threshold_10": 0.20186184242570115, "scr_dir2_threshold_10": 0.19500875267788764, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.27035680954058827, + "scr_metric_threshold_20": 0.2166289325602405, + "scr_dir2_threshold_20": 0.20953907403371513, + "scr_dir1_threshold_50": 0.11729968087135588, + "scr_metric_threshold_50": 0.26626300502796274, + "scr_dir2_threshold_50": 0.25901529970812726, + "scr_dir1_threshold_100": 0.0291723500274349, + "scr_metric_threshold_100": 0.2648691843249761, + "scr_dir2_threshold_100": 0.25028435093025925, + "scr_dir1_threshold_500": -1.1906623470141946, + "scr_metric_threshold_500": 0.1666632837766538, + "scr_dir2_threshold_500": 0.15979176734762407 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.2307692895509249, "scr_metric_threshold_10": 0.005181279943224265, "scr_dir2_threshold_10": 0.005181279943224265, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.20512859700616604, + "scr_metric_threshold_20": 0.09326427430734313, + "scr_dir2_threshold_20": 0.09326427430734313, + "scr_dir1_threshold_50": 0.5128207283533913, + "scr_metric_threshold_50": 0.25129531998580606, + "scr_dir2_threshold_50": 0.25129531998580606, + "scr_dir1_threshold_100": 0.5897435701496917, + "scr_metric_threshold_100": 0.2694299542032543, + "scr_dir2_threshold_100": 0.2694299542032543, + "scr_dir1_threshold_500": -2.602563916422071, + "scr_metric_threshold_500": 0.10103619422217953, + "scr_dir2_threshold_500": 0.10103619422217953 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.3560604589646946, "scr_metric_threshold_10": 0.2721711947358348, "scr_dir2_threshold_10": 0.2721711947358348, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.5454546275545891, + "scr_metric_threshold_20": -0.006116128797061692, + "scr_dir2_threshold_20": -0.006116128797061692, + "scr_dir1_threshold_50": 0.6060607976273744, + "scr_metric_threshold_50": 0.15290522497562042, + "scr_dir2_threshold_50": 0.15290522497562042, + "scr_dir1_threshold_100": 0.34090914222161817, + "scr_metric_threshold_100": 0.35779827383502094, + "scr_dir2_threshold_100": 0.35779827383502094, + "scr_dir1_threshold_500": -1.939393378376975, + "scr_metric_threshold_500": 0.03363925521540607, + "scr_dir2_threshold_500": 0.03363925521540607 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.5156247962732202, "scr_metric_threshold_10": 0.10804029093651613, "scr_dir2_threshold_10": 0.10804029093651613, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.32812537834973393, + "scr_metric_threshold_20": 0.16834176159853043, + "scr_dir2_threshold_20": 0.16834176159853043, + "scr_dir1_threshold_50": -1.4218750873114772, + "scr_metric_threshold_50": 0.15577896145062706, + "scr_dir2_threshold_50": 0.15577896145062706, + "scr_dir1_threshold_100": -0.921875087311477, + "scr_metric_threshold_100": -0.005025060155005, + "scr_dir2_threshold_100": -0.005025060155005, + "scr_dir1_threshold_500": -4.3437492433005325, + "scr_metric_threshold_500": -0.1005024011832269, + "scr_dir2_threshold_500": -0.1005024011832269 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": 0.0067113234832316924, "scr_metric_threshold_10": 0.044585981215805634, "scr_dir2_threshold_10": 0.044585981215805634, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.2214768751962653, + "scr_metric_threshold_20": -0.025477757787232685, + "scr_dir2_threshold_20": -0.025477757787232685, + "scr_dir1_threshold_50": 0.13422806978944357, + "scr_metric_threshold_50": -0.08280242807295153, + "scr_dir2_threshold_50": -0.08280242807295153, + "scr_dir1_threshold_100": -1.100671452373285, + "scr_metric_threshold_100": -0.2611465427598772, + "scr_dir2_threshold_100": -0.2611465427598772, + "scr_dir1_threshold_500": -1.7852352483493712, + "scr_metric_threshold_500": -0.5445859812158056, + "scr_dir2_threshold_500": -0.5445859812158056 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.07894729412163301, "scr_metric_threshold_10": 0.4999998817368318, "scr_dir2_threshold_10": 0.4999998817368318, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.11578955293692479, + "scr_metric_threshold_20": 0.595238157185469, + "scr_dir2_threshold_20": 0.595238157185469, + "scr_dir1_threshold_50": -0.17894726275076694, + "scr_metric_threshold_50": 0.6904761961077699, + "scr_dir2_threshold_50": 0.6904761961077699, + "scr_dir1_threshold_100": -0.1421053176441358, + "scr_metric_threshold_100": 0.7380950973057522, + "scr_dir2_threshold_100": 0.7380950973057522, + "scr_dir1_threshold_500": -0.17894726275076694, + "scr_metric_threshold_500": 0.801587280938177, + "scr_dir2_threshold_500": 0.801587280938177 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.3934425108216648, "scr_metric_threshold_10": 0.12977090206161526, "scr_dir2_threshold_10": 0.12977090206161526, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.4316940424658739, + "scr_metric_threshold_20": 0.17557251734733875, + "scr_dir2_threshold_20": 0.17557251734733875, + "scr_dir1_threshold_50": 0.4535520140184937, + "scr_metric_threshold_50": 0.28244272551540384, + "scr_dir2_threshold_50": 0.28244272551540384, + "scr_dir1_threshold_100": 0.5464479859815062, + "scr_metric_threshold_100": 0.3015266750097995, + "scr_dir2_threshold_100": 0.3015266750097995, + "scr_dir1_threshold_500": 0.5245903401373838, + "scr_metric_threshold_500": 0.3320610852002818, + "scr_dir2_threshold_500": 0.3320610852002818 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.40000005544617256, "scr_metric_threshold_10": 0.36909885188142205, "scr_dir2_threshold_10": 0.36909885188142205, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.5023254124779394, + "scr_metric_threshold_20": 0.4763948958977931, + "scr_dir2_threshold_20": 0.4763948958977931, + "scr_dir1_threshold_50": 0.5302325800601148, + "scr_metric_threshold_50": 0.3776824330771527, + "scr_dir2_threshold_50": 0.3776824330771527, + "scr_dir1_threshold_100": 0.6232557327183983, + "scr_metric_threshold_100": 0.4206008506837011, + "scr_dir2_threshold_100": 0.4206008506837011, + "scr_dir1_threshold_500": 0.44186052958857275, + "scr_metric_threshold_500": 0.35193143367601315, + "scr_dir2_threshold_500": 0.35193143367601315 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.18604635689435922, "scr_metric_threshold_10": 0.18604635689435922, "scr_dir2_threshold_10": 0.1312216389118512, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2558137407297439, + "scr_metric_threshold_20": 0.2558137407297439, + "scr_dir2_threshold_20": 0.19909487251754088, + "scr_dir1_threshold_50": 0.3023256071842735, + "scr_metric_threshold_50": 0.3023256071842735, + "scr_dir2_threshold_50": 0.2443439646255898, + "scr_dir1_threshold_100": 0.2976742264771627, + "scr_metric_threshold_100": 0.2976742264771627, + "scr_dir2_threshold_100": 0.1809955593194283, + "scr_dir1_threshold_500": 0.35813940336020533, + "scr_metric_threshold_500": 0.35813940336020533, + "scr_dir2_threshold_500": 0.3031672719279674 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-317382-tokens-1300000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-317382-tokens-1300000000/tpp/saebench_tpp_custom_sae_eval_results.json index 6fd715ad71fb29dcd969bb5408c56094f7495b01..93add208d777370837ee179062626613fd34a50e 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-317382-tokens-1300000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-317382-tokens-1300000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "741babd0-f44e-43bc-9a53-6d3219b8d652", - "datetime_epoch_millis": 1776886063440, + "eval_id": "47881ec7-cf88-4cf8-8d53-ae708a7cbe4b", + "datetime_epoch_millis": 1777317677846, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.02855000048875809, - "tpp_threshold_2_intended_diff_only": 0.032999998331069945, - "tpp_threshold_2_unintended_diff_only": 0.004449997842311859, - "tpp_threshold_5_total_metric": 0.07450000792741775, - "tpp_threshold_5_intended_diff_only": 0.0993000090122223, - "tpp_threshold_5_unintended_diff_only": 0.024800001084804537, - "tpp_threshold_10_total_metric": 0.09805000424385071, - "tpp_threshold_10_intended_diff_only": 0.15370001196861266, - "tpp_threshold_10_unintended_diff_only": 0.05565000772476196, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.028174997866153715, + "tpp_threshold_2_intended_diff_only": 0.03320000171661377, + "tpp_threshold_2_unintended_diff_only": 0.005025003850460053, + "tpp_threshold_5_total_metric": 0.07472501248121262, + "tpp_threshold_5_intended_diff_only": 0.09870001673698425, + "tpp_threshold_5_unintended_diff_only": 0.023975004255771638, + "tpp_threshold_10_total_metric": 0.09890000522136688, + "tpp_threshold_10_intended_diff_only": 0.15510001182556152, + "tpp_threshold_10_unintended_diff_only": 0.05620000660419464, + "tpp_threshold_20_total_metric": 0.1227250024676323, + "tpp_threshold_20_intended_diff_only": 0.21230001449584962, + "tpp_threshold_20_unintended_diff_only": 0.08957501202821731, + "tpp_threshold_50_total_metric": 0.1471000075340271, + "tpp_threshold_50_intended_diff_only": 0.2677000164985657, + "tpp_threshold_50_unintended_diff_only": 0.12060000896453858, + "tpp_threshold_100_total_metric": 0.17207499444484708, + "tpp_threshold_100_intended_diff_only": 0.32460000514984133, + "tpp_threshold_100_unintended_diff_only": 0.1525250107049942, + "tpp_threshold_500_total_metric": 0.23282500952482224, + "tpp_threshold_500_intended_diff_only": 0.43890002369880676, + "tpp_threshold_500_unintended_diff_only": 0.20607501417398452 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.04950000643730164, - "tpp_threshold_2_intended_diff_only": 0.05840001106262207, - "tpp_threshold_2_unintended_diff_only": 0.008900004625320434, - "tpp_threshold_5_total_metric": 0.1350500226020813, - "tpp_threshold_5_intended_diff_only": 0.18260003328323365, - "tpp_threshold_5_unintended_diff_only": 0.047550010681152347, - "tpp_threshold_10_total_metric": 0.1737999975681305, - "tpp_threshold_10_intended_diff_only": 0.28180001974105834, - "tpp_threshold_10_unintended_diff_only": 0.10800002217292785, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.0494499921798706, + "tpp_threshold_2_intended_diff_only": 0.05880000591278076, + "tpp_threshold_2_unintended_diff_only": 0.009350013732910157, + "tpp_threshold_5_total_metric": 0.13520001471042634, + "tpp_threshold_5_intended_diff_only": 0.182800030708313, + "tpp_threshold_5_unintended_diff_only": 0.04760001599788666, + "tpp_threshold_10_total_metric": 0.17215000987052917, + "tpp_threshold_10_intended_diff_only": 0.2818000316619873, + "tpp_threshold_10_unintended_diff_only": 0.10965002179145814, + "tpp_threshold_20_total_metric": 0.1992499977350235, + "tpp_threshold_20_intended_diff_only": 0.3744000196456909, + "tpp_threshold_20_unintended_diff_only": 0.17515002191066742, + "tpp_threshold_50_total_metric": 0.1827000081539154, + "tpp_threshold_50_intended_diff_only": 0.41860003471374513, + "tpp_threshold_50_unintended_diff_only": 0.23590002655982972, + "tpp_threshold_100_total_metric": 0.14964999556541442, + "tpp_threshold_100_intended_diff_only": 0.4428000211715698, + "tpp_threshold_100_unintended_diff_only": 0.2931500256061554, + "tpp_threshold_500_total_metric": 0.08850001096725464, + "tpp_threshold_500_intended_diff_only": 0.4674000382423401, + "tpp_threshold_500_unintended_diff_only": 0.3789000272750854 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.007599994540214539, - "tpp_threshold_2_intended_diff_only": 0.007599985599517823, - "tpp_threshold_2_unintended_diff_only": -8.940696716308593e-09, - "tpp_threshold_5_total_metric": 0.01394999325275421, - "tpp_threshold_5_intended_diff_only": 0.015999984741210938, - "tpp_threshold_5_unintended_diff_only": 0.002049991488456726, - "tpp_threshold_10_total_metric": 0.02230001091957092, - "tpp_threshold_10_intended_diff_only": 0.02560000419616699, - "tpp_threshold_10_unintended_diff_only": 0.003299993276596069, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.006900003552436829, + "tpp_threshold_2_intended_diff_only": 0.0075999975204467775, + "tpp_threshold_2_unintended_diff_only": 0.0006999939680099488, + "tpp_threshold_5_total_metric": 0.014250010251998901, + "tpp_threshold_5_intended_diff_only": 0.014600002765655517, + "tpp_threshold_5_unintended_diff_only": 0.0003499925136566162, + "tpp_threshold_10_total_metric": 0.02565000057220459, + "tpp_threshold_10_intended_diff_only": 0.02839999198913574, + "tpp_threshold_10_unintended_diff_only": 0.002749991416931152, + "tpp_threshold_20_total_metric": 0.04620000720024109, + "tpp_threshold_20_intended_diff_only": 0.0502000093460083, + "tpp_threshold_20_unintended_diff_only": 0.004000002145767212, + "tpp_threshold_50_total_metric": 0.11150000691413879, + "tpp_threshold_50_intended_diff_only": 0.11679999828338623, + "tpp_threshold_50_unintended_diff_only": 0.005299991369247437, + "tpp_threshold_100_total_metric": 0.19449999332427978, + "tpp_threshold_100_intended_diff_only": 0.2063999891281128, + "tpp_threshold_100_unintended_diff_only": 0.011899995803833007, + "tpp_threshold_500_total_metric": 0.37715000808238985, + "tpp_threshold_500_intended_diff_only": 0.4104000091552734, + "tpp_threshold_500_unintended_diff_only": 0.0332500010728836 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.0017500221729278564, "tpp_threshold_10_total_metric": 0.05774995684623718, "tpp_threshold_10_intended_diff_only": 0.06599998474121094, - "tpp_threshold_10_unintended_diff_only": 0.008250027894973755 + "tpp_threshold_10_unintended_diff_only": 0.008250027894973755, + "tpp_threshold_20_total_metric": 0.21074995398521423, + "tpp_threshold_20_intended_diff_only": 0.3579999804496765, + "tpp_threshold_20_unintended_diff_only": 0.14725002646446228, + "tpp_threshold_50_total_metric": 0.18774999678134918, + "tpp_threshold_50_intended_diff_only": 0.4150000214576721, + "tpp_threshold_50_unintended_diff_only": 0.22725002467632294, + "tpp_threshold_100_total_metric": 0.16274996101856232, + "tpp_threshold_100_intended_diff_only": 0.4359999895095825, + "tpp_threshold_100_unintended_diff_only": 0.2732500284910202, + "tpp_threshold_500_total_metric": 0.08550000190734863, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.37050002813339233 }, "1": { - "tpp_threshold_2_total_metric": 0.06900003552436829, - "tpp_threshold_2_intended_diff_only": 0.08500003814697266, - "tpp_threshold_2_unintended_diff_only": 0.01600000262260437, - "tpp_threshold_5_total_metric": 0.18700000643730164, - "tpp_threshold_5_intended_diff_only": 0.281000018119812, - "tpp_threshold_5_unintended_diff_only": 0.09400001168251038, - "tpp_threshold_10_total_metric": 0.1927499920129776, + "tpp_threshold_2_total_metric": 0.07274998724460602, + "tpp_threshold_2_intended_diff_only": 0.08300000429153442, + "tpp_threshold_2_unintended_diff_only": 0.010250017046928406, + "tpp_threshold_5_total_metric": 0.18324998021125793, + "tpp_threshold_5_intended_diff_only": 0.28200000524520874, + "tpp_threshold_5_unintended_diff_only": 0.0987500250339508, + "tpp_threshold_10_total_metric": 0.1834999918937683, "tpp_threshold_10_intended_diff_only": 0.3370000123977661, - "tpp_threshold_10_unintended_diff_only": 0.1442500203847885 + "tpp_threshold_10_unintended_diff_only": 0.1535000205039978, + "tpp_threshold_20_total_metric": 0.18474997580051422, + "tpp_threshold_20_intended_diff_only": 0.35600000619888306, + "tpp_threshold_20_unintended_diff_only": 0.17125003039836884, + "tpp_threshold_50_total_metric": 0.1900000125169754, + "tpp_threshold_50_intended_diff_only": 0.40400004386901855, + "tpp_threshold_50_unintended_diff_only": 0.21400003135204315, + "tpp_threshold_100_total_metric": 0.16724997758865356, + "tpp_threshold_100_intended_diff_only": 0.4350000023841858, + "tpp_threshold_100_unintended_diff_only": 0.2677500247955322, + "tpp_threshold_500_total_metric": 0.07975003123283386, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.37925001978874207 }, "2": { - "tpp_threshold_2_total_metric": 0.153500035405159, - "tpp_threshold_2_intended_diff_only": 0.1820000410079956, - "tpp_threshold_2_unintended_diff_only": 0.02850000560283661, - "tpp_threshold_5_total_metric": 0.21050004661083221, + "tpp_threshold_2_total_metric": 0.1485000103712082, + "tpp_threshold_2_intended_diff_only": 0.18300002813339233, + "tpp_threshold_2_unintended_diff_only": 0.03450001776218414, + "tpp_threshold_5_total_metric": 0.21525004506111145, "tpp_threshold_5_intended_diff_only": 0.28700006008148193, - "tpp_threshold_5_unintended_diff_only": 0.07650001347064972, - "tpp_threshold_10_total_metric": 0.15950000286102295, - "tpp_threshold_10_intended_diff_only": 0.3410000205039978, - "tpp_threshold_10_unintended_diff_only": 0.18150001764297485 + "tpp_threshold_5_unintended_diff_only": 0.07175001502037048, + "tpp_threshold_10_total_metric": 0.1587500423192978, + "tpp_threshold_10_intended_diff_only": 0.3420000672340393, + "tpp_threshold_10_unintended_diff_only": 0.18325002491474152, + "tpp_threshold_20_total_metric": 0.109250009059906, + "tpp_threshold_20_intended_diff_only": 0.359000027179718, + "tpp_threshold_20_unintended_diff_only": 0.249750018119812, + "tpp_threshold_50_total_metric": 0.09399999678134918, + "tpp_threshold_50_intended_diff_only": 0.38200002908706665, + "tpp_threshold_50_unintended_diff_only": 0.28800003230571747, + "tpp_threshold_100_total_metric": 0.0585000216960907, + "tpp_threshold_100_intended_diff_only": 0.40800005197525024, + "tpp_threshold_100_unintended_diff_only": 0.34950003027915955, + "tpp_threshold_500_total_metric": 0.04674999415874481, + "tpp_threshold_500_intended_diff_only": 0.4520000219345093, + "tpp_threshold_500_unintended_diff_only": 0.40525002777576447 }, "6": { - "tpp_threshold_2_total_metric": 0.011750012636184692, + "tpp_threshold_2_total_metric": 0.009499981999397278, "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": -0.002750009298324585, - "tpp_threshold_5_total_metric": 0.21775005757808685, - "tpp_threshold_5_intended_diff_only": 0.2640000581741333, - "tpp_threshold_5_unintended_diff_only": 0.04625000059604645, - "tpp_threshold_10_total_metric": 0.2625000476837158, + "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704, + "tpp_threshold_5_total_metric": 0.22300003468990326, + "tpp_threshold_5_intended_diff_only": 0.26500004529953003, + "tpp_threshold_5_unintended_diff_only": 0.04200001060962677, + "tpp_threshold_10_total_metric": 0.2657500356435776, "tpp_threshold_10_intended_diff_only": 0.3840000629425049, - "tpp_threshold_10_unintended_diff_only": 0.12150001525878906 + "tpp_threshold_10_unintended_diff_only": 0.11825002729892731, + "tpp_threshold_20_total_metric": 0.27650001645088196, + "tpp_threshold_20_intended_diff_only": 0.45100003480911255, + "tpp_threshold_20_unintended_diff_only": 0.1745000183582306, + "tpp_threshold_50_total_metric": 0.2227499932050705, + "tpp_threshold_50_intended_diff_only": 0.48000001907348633, + "tpp_threshold_50_unintended_diff_only": 0.25725002586841583, + "tpp_threshold_100_total_metric": 0.1785000115633011, + "tpp_threshold_100_intended_diff_only": 0.4830000400543213, + "tpp_threshold_100_unintended_diff_only": 0.3045000284910202, + "tpp_threshold_500_total_metric": 0.12024997174739838, + "tpp_threshold_500_intended_diff_only": 0.49400001764297485, + "tpp_threshold_500_unintended_diff_only": 0.3737500458955765 }, "9": { - "tpp_threshold_2_total_metric": 0.003999993205070496, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_5_total_metric": 0.04375001788139343, - "tpp_threshold_5_intended_diff_only": 0.06300002336502075, - "tpp_threshold_5_unintended_diff_only": 0.01925000548362732, - "tpp_threshold_10_total_metric": 0.1964999884366989, - "tpp_threshold_10_intended_diff_only": 0.281000018119812, - "tpp_threshold_10_unintended_diff_only": 0.0845000296831131 + "tpp_threshold_2_total_metric": 0.0072500258684158325, + "tpp_threshold_2_intended_diff_only": 0.008000016212463379, + "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.03825002908706665, + "tpp_threshold_5_intended_diff_only": 0.06200003623962402, + "tpp_threshold_5_unintended_diff_only": 0.023750007152557373, + "tpp_threshold_10_total_metric": 0.19500002264976501, + "tpp_threshold_10_intended_diff_only": 0.2800000309944153, + "tpp_threshold_10_unintended_diff_only": 0.08500000834465027, + "tpp_threshold_20_total_metric": 0.21500003337860107, + "tpp_threshold_20_intended_diff_only": 0.34800004959106445, + "tpp_threshold_20_unintended_diff_only": 0.13300001621246338, + "tpp_threshold_50_total_metric": 0.21900004148483276, + "tpp_threshold_50_intended_diff_only": 0.41200006008148193, + "tpp_threshold_50_unintended_diff_only": 0.19300001859664917, + "tpp_threshold_100_total_metric": 0.18125000596046448, + "tpp_threshold_100_intended_diff_only": 0.4520000219345093, + "tpp_threshold_100_unintended_diff_only": 0.2707500159740448, + "tpp_threshold_500_total_metric": 0.11025005578994751, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.3657500147819519 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.010749980807304382, - "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_2_unintended_diff_only": -0.004749998450279236, - "tpp_threshold_5_total_metric": 0.011000022292137146, - "tpp_threshold_5_intended_diff_only": 0.008000016212463379, - "tpp_threshold_5_unintended_diff_only": -0.003000006079673767, - "tpp_threshold_10_total_metric": 0.0014999955892562866, - "tpp_threshold_10_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_10_unintended_diff_only": 0.00449998676776886 + "tpp_threshold_2_total_metric": 0.005000025033950806, + "tpp_threshold_2_intended_diff_only": 0.008000016212463379, + "tpp_threshold_2_unintended_diff_only": 0.0029999911785125732, + "tpp_threshold_5_total_metric": 0.005499988794326782, + "tpp_threshold_5_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_5_unintended_diff_only": 0.0004999935626983643, + "tpp_threshold_10_total_metric": 0.0032500028610229492, + "tpp_threshold_10_intended_diff_only": 0.004999995231628418, + "tpp_threshold_10_unintended_diff_only": 0.0017499923706054688, + "tpp_threshold_20_total_metric": 0.01825001835823059, + "tpp_threshold_20_intended_diff_only": 0.022000014781951904, + "tpp_threshold_20_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_50_total_metric": 0.06300000846385956, + "tpp_threshold_50_intended_diff_only": 0.06099998950958252, + "tpp_threshold_50_unintended_diff_only": -0.0020000189542770386, + "tpp_threshold_100_total_metric": 0.14749999344348907, + "tpp_threshold_100_intended_diff_only": 0.15700000524520874, + "tpp_threshold_100_unintended_diff_only": 0.009500011801719666, + "tpp_threshold_500_total_metric": 0.41724999248981476, + "tpp_threshold_500_intended_diff_only": 0.4359999895095825, + "tpp_threshold_500_unintended_diff_only": 0.01874999701976776 }, "2": { - "tpp_threshold_2_total_metric": 0.0045000165700912476, - "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": 0.00449998676776886, - "tpp_threshold_5_total_metric": 0.011500030755996704, - "tpp_threshold_5_intended_diff_only": 0.018000006675720215, - "tpp_threshold_5_unintended_diff_only": 0.006499975919723511, - "tpp_threshold_10_total_metric": 0.028500065207481384, - "tpp_threshold_10_intended_diff_only": 0.033000051975250244, - "tpp_threshold_10_unintended_diff_only": 0.00449998676776886 + "tpp_threshold_2_total_metric": 0.013750046491622925, + "tpp_threshold_2_intended_diff_only": 0.01100003719329834, + "tpp_threshold_2_unintended_diff_only": -0.002750009298324585, + "tpp_threshold_5_total_metric": 0.007750064134597778, + "tpp_threshold_5_intended_diff_only": 0.01500004529953003, + "tpp_threshold_5_unintended_diff_only": 0.007249981164932251, + "tpp_threshold_10_total_metric": 0.030500024557113647, + "tpp_threshold_10_intended_diff_only": 0.03600001335144043, + "tpp_threshold_10_unintended_diff_only": 0.005499988794326782, + "tpp_threshold_20_total_metric": 0.07225003838539124, + "tpp_threshold_20_intended_diff_only": 0.07600003480911255, + "tpp_threshold_20_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_50_total_metric": 0.15050005912780762, + "tpp_threshold_50_intended_diff_only": 0.16300004720687866, + "tpp_threshold_50_unintended_diff_only": 0.012499988079071045, + "tpp_threshold_100_total_metric": 0.24500001966953278, + "tpp_threshold_100_intended_diff_only": 0.2630000114440918, + "tpp_threshold_100_unintended_diff_only": 0.01799999177455902, + "tpp_threshold_500_total_metric": 0.40800003707408905, + "tpp_threshold_500_intended_diff_only": 0.43800002336502075, + "tpp_threshold_500_unintended_diff_only": 0.0299999862909317 }, "3": { - "tpp_threshold_2_total_metric": 0.009499996900558472, + "tpp_threshold_2_total_metric": 0.0034999847412109375, "tpp_threshold_2_intended_diff_only": 0.001999974250793457, - "tpp_threshold_2_unintended_diff_only": -0.007500022649765015, - "tpp_threshold_5_total_metric": 0.01600000262260437, - "tpp_threshold_5_intended_diff_only": 0.013999998569488525, - "tpp_threshold_5_unintended_diff_only": -0.0020000040531158447, - "tpp_threshold_10_total_metric": 0.01799999177455902, - "tpp_threshold_10_intended_diff_only": 0.019999980926513672, - "tpp_threshold_10_unintended_diff_only": 0.001999989151954651 + "tpp_threshold_2_unintended_diff_only": -0.0015000104904174805, + "tpp_threshold_5_total_metric": 0.012750029563903809, + "tpp_threshold_5_intended_diff_only": 0.00700002908706665, + "tpp_threshold_5_unintended_diff_only": -0.005750000476837158, + "tpp_threshold_10_total_metric": 0.020500019192695618, + "tpp_threshold_10_intended_diff_only": 0.018000006675720215, + "tpp_threshold_10_unintended_diff_only": -0.002500012516975403, + "tpp_threshold_20_total_metric": 0.007250010967254639, + "tpp_threshold_20_intended_diff_only": 0.009000003337860107, + "tpp_threshold_20_unintended_diff_only": 0.0017499923706054688, + "tpp_threshold_50_total_metric": 0.04975003004074097, + "tpp_threshold_50_intended_diff_only": 0.050000011920928955, + "tpp_threshold_50_unintended_diff_only": 0.0002499818801879883, + "tpp_threshold_100_total_metric": 0.09274999797344208, + "tpp_threshold_100_intended_diff_only": 0.09799998998641968, + "tpp_threshold_100_unintended_diff_only": 0.0052499920129776, + "tpp_threshold_500_total_metric": 0.35349999368190765, + "tpp_threshold_500_intended_diff_only": 0.3889999985694885, + "tpp_threshold_500_unintended_diff_only": 0.03550000488758087 }, "5": { - "tpp_threshold_2_total_metric": -0.00850003957748413, - "tpp_threshold_2_intended_diff_only": -0.001000046730041504, - "tpp_threshold_2_unintended_diff_only": 0.007499992847442627, - "tpp_threshold_5_total_metric": 0.007249951362609863, - "tpp_threshold_5_intended_diff_only": 0.008999943733215332, - "tpp_threshold_5_unintended_diff_only": 0.0017499923706054688, - "tpp_threshold_10_total_metric": 0.011000007390975952, - "tpp_threshold_10_intended_diff_only": 0.013999998569488525, - "tpp_threshold_10_unintended_diff_only": 0.0029999911785125732 + "tpp_threshold_2_total_metric": 0.0, + "tpp_threshold_2_intended_diff_only": 0.0, + "tpp_threshold_2_unintended_diff_only": 0.0, + "tpp_threshold_5_total_metric": 0.01649998128414154, + "tpp_threshold_5_intended_diff_only": 0.015999972820281982, + "tpp_threshold_5_unintended_diff_only": -0.0005000084638595581, + "tpp_threshold_10_total_metric": 0.007249966263771057, + "tpp_threshold_10_intended_diff_only": 0.012999951839447021, + "tpp_threshold_10_unintended_diff_only": 0.005749985575675964, + "tpp_threshold_20_total_metric": 0.0274999737739563, + "tpp_threshold_20_intended_diff_only": 0.02899998426437378, + "tpp_threshold_20_unintended_diff_only": 0.0015000104904174805, + "tpp_threshold_50_total_metric": 0.1149999350309372, + "tpp_threshold_50_intended_diff_only": 0.1239999532699585, + "tpp_threshold_50_unintended_diff_only": 0.009000018239021301, + "tpp_threshold_100_total_metric": 0.2109999656677246, + "tpp_threshold_100_intended_diff_only": 0.22499996423721313, + "tpp_threshold_100_unintended_diff_only": 0.013999998569488525, + "tpp_threshold_500_total_metric": 0.3739999979734421, + "tpp_threshold_500_intended_diff_only": 0.41600000858306885, + "tpp_threshold_500_unintended_diff_only": 0.04200001060962677 }, "6": { - "tpp_threshold_2_total_metric": 0.021750018000602722, - "tpp_threshold_2_intended_diff_only": 0.022000014781951904, - "tpp_threshold_2_unintended_diff_only": 0.00024999678134918213, - "tpp_threshold_5_total_metric": 0.023999959230422974, - "tpp_threshold_5_intended_diff_only": 0.030999958515167236, - "tpp_threshold_5_unintended_diff_only": 0.006999999284744263, - "tpp_threshold_10_total_metric": 0.05249999463558197, - "tpp_threshold_10_intended_diff_only": 0.05500000715255737, - "tpp_threshold_10_unintended_diff_only": 0.002500012516975403 + "tpp_threshold_2_total_metric": 0.012249961495399475, + "tpp_threshold_2_intended_diff_only": 0.01699995994567871, + "tpp_threshold_2_unintended_diff_only": 0.004749998450279236, + "tpp_threshold_5_total_metric": 0.028749987483024597, + "tpp_threshold_5_intended_diff_only": 0.02899998426437378, + "tpp_threshold_5_unintended_diff_only": 0.00024999678134918213, + "tpp_threshold_10_total_metric": 0.06674998998641968, + "tpp_threshold_10_intended_diff_only": 0.06999999284744263, + "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492, + "tpp_threshold_20_total_metric": 0.10574999451637268, + "tpp_threshold_20_intended_diff_only": 0.11500000953674316, + "tpp_threshold_20_unintended_diff_only": 0.009250015020370483, + "tpp_threshold_50_total_metric": 0.17925000190734863, + "tpp_threshold_50_intended_diff_only": 0.18599998950958252, + "tpp_threshold_50_unintended_diff_only": 0.006749987602233887, + "tpp_threshold_100_total_metric": 0.2762499898672104, + "tpp_threshold_100_intended_diff_only": 0.2889999747276306, + "tpp_threshold_100_unintended_diff_only": 0.012749984860420227, + "tpp_threshold_500_total_metric": 0.3330000191926956, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.040000006556510925 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-341796-tokens-1400000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-341796-tokens-1400000000/scr/saebench_scr_custom_sae_eval_results.json index 28c9604308d44c348e242f608724ff8ce97e8161..331ba3de630071f1bea860d3814fc498701d74d0 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-341796-tokens-1400000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-341796-tokens-1400000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "ee0d2024-0f8a-447a-8cf2-2d691b0736f7", - "datetime_epoch_millis": 1776886027710, + "eval_id": "4ae4fbf3-e7ee-418e-b465-45cd831027dc", + "datetime_epoch_millis": 1777317523062, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.29573145348122126, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.318147144192202, "scr_metric_threshold_10": 0.2178056251175732, "scr_dir2_threshold_10": 0.2061909045784877, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2715260894565377, + "scr_metric_threshold_20": 0.2595251976969898, + "scr_dir2_threshold_20": 0.24778416638655848, + "scr_dir1_threshold_50": 0.3220415073303419, + "scr_metric_threshold_50": 0.3325713457900344, + "scr_dir2_threshold_50": 0.3218352782092032, + "scr_dir1_threshold_100": 0.07808705309726434, + "scr_metric_threshold_100": 0.3342134066696645, + "scr_dir2_threshold_100": 0.3154009425730885, + "scr_dir1_threshold_500": -1.1534098777555588, + "scr_metric_threshold_500": 0.15969367766919482, + "scr_dir2_threshold_500": 0.13723497080528102 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.5641028776049328, "scr_metric_threshold_10": 0.036269422851059746, "scr_dir2_threshold_10": 0.036269422851059746, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2820514388024664, + "scr_metric_threshold_20": 0.14248705143262683, + "scr_dir2_threshold_20": 0.14248705143262683, + "scr_dir1_threshold_50": 0.5128207283533913, + "scr_metric_threshold_50": 0.2616580342884179, + "scr_dir2_threshold_50": 0.2616580342884179, + "scr_dir1_threshold_100": 0.5256410746257707, + "scr_metric_threshold_100": 0.3082901714420895, + "scr_dir2_threshold_100": 0.3082901714420895, + "scr_dir1_threshold_500": -2.564102113442909, + "scr_metric_threshold_500": 0.025906708548447916, + "scr_dir2_threshold_500": 0.025906708548447916 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.3636365688864726, "scr_metric_threshold_10": 0.34250758728798886, "scr_dir2_threshold_10": 0.34250758728798886, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.5454546275545891, + "scr_metric_threshold_20": 0.22629968192630534, + "scr_dir2_threshold_20": 0.22629968192630534, + "scr_dir1_threshold_50": 0.37878788562954907, + "scr_metric_threshold_50": 0.44954129945407983, + "scr_dir2_threshold_50": 0.44954129945407983, + "scr_dir1_threshold_100": 0.128787998517109, + "scr_metric_threshold_100": 0.553516764944451, + "scr_dir2_threshold_100": 0.553516764944451, + "scr_dir1_threshold_500": -2.0454539502292293, + "scr_metric_threshold_500": -0.24770631499321016, + "scr_dir2_threshold_500": -0.24770631499321016 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.4687504074535596, "scr_metric_threshold_10": 0.10301508102112025, "scr_dir2_threshold_10": 0.10301508102112025, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2656252619344312, + "scr_metric_threshold_20": 0.1809045617464338, + "scr_dir2_threshold_20": 0.1809045617464338, + "scr_dir1_threshold_50": 0.32812537834973393, + "scr_metric_threshold_50": 0.19346736189433716, + "scr_dir2_threshold_50": 0.19346736189433716, + "scr_dir1_threshold_100": -0.15624982537704588, + "scr_metric_threshold_100": 0.037688550204100946, + "scr_dir2_threshold_100": 0.037688550204100946, + "scr_dir1_threshold_500": -4.249999534338789, + "scr_metric_threshold_500": -0.052763730669115945, + "scr_dir2_threshold_500": -0.052763730669115945 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": -0.03355741747856333, "scr_metric_threshold_10": 0.08917196243161127, "scr_dir2_threshold_10": 0.08917196243161127, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.2617452161268579, + "scr_metric_threshold_20": 0.00955420662613804, + "scr_dir2_threshold_20": 0.00955420662613804, + "scr_dir1_threshold_50": 0.14765071675590696, + "scr_metric_threshold_50": -0.07324841127051662, + "scr_dir2_threshold_50": -0.07324841127051662, + "scr_dir1_threshold_100": -1.167785487268007, + "scr_metric_threshold_100": -0.2611465427598772, + "scr_dir2_threshold_100": -0.2611465427598772, + "scr_dir1_threshold_500": -1.6241614845957986, + "scr_metric_threshold_500": -0.5414013089483273, + "scr_dir2_threshold_500": -0.5414013089483273 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.09473687842942387, "scr_metric_threshold_10": 0.4325397322853009, "scr_dir2_threshold_10": 0.4325397322853009, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.11052646273721473, + "scr_metric_threshold_20": 0.5357141758984867, + "scr_dir2_threshold_20": 0.5357141758984867, + "scr_dir1_threshold_50": -0.13157882353605502, + "scr_metric_threshold_50": 0.6785713525451061, + "scr_dir2_threshold_50": 0.6785713525451061, + "scr_dir1_threshold_100": -0.11052614902855408, + "scr_metric_threshold_100": 0.7380950973057522, + "scr_dir2_threshold_100": 0.7380950973057522, + "scr_dir1_threshold_500": -0.16315799215163673, + "scr_metric_threshold_500": 0.773809391467407, + "scr_dir2_threshold_500": 0.773809391467407 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.4316940424658739, "scr_metric_threshold_10": 0.1183206688641517, "scr_dir2_threshold_10": 0.1183206688641517, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.4535520140184937, + "scr_metric_threshold_20": 0.19465646684173443, + "scr_dir2_threshold_20": 0.19465646684173443, + "scr_dir1_threshold_50": 0.4754099855711136, + "scr_metric_threshold_50": 0.2709922648193172, + "scr_dir2_threshold_50": 0.2709922648193172, + "scr_dir1_threshold_100": 0.4699452484015855, + "scr_metric_threshold_100": 0.3282443408011273, + "scr_dir2_threshold_100": 0.3282443408011273, + "scr_dir1_threshold_500": 0.4754099855711136, + "scr_metric_threshold_500": 0.3320610852002818, + "scr_dir2_threshold_500": 0.3320610852002818 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.40000005544617256, "scr_metric_threshold_10": 0.3648068054696092, "scr_dir2_threshold_10": 0.3648068054696092, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.48372100373097293, + "scr_metric_threshold_20": 0.493562314103202, + "scr_dir2_threshold_20": 0.493562314103202, + "scr_dir1_threshold_50": 0.5348836822468563, + "scr_metric_threshold_50": 0.549356359317294, + "scr_dir2_threshold_50": 0.549356359317294, + "scr_dir1_threshold_100": 0.5581394704114272, + "scr_metric_threshold_100": 0.5922747769238424, + "scr_dir2_threshold_100": 0.5922747769238424, + "scr_dir1_threshold_500": 0.49302320810445616, + "scr_metric_threshold_500": 0.5364807317097504, + "scr_dir2_threshold_500": 0.5364807317097504 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.2558137407297439, "scr_metric_threshold_10": 0.2558137407297439, "scr_dir2_threshold_10": 0.16289597641705988, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.29302312300099165, + "scr_metric_threshold_20": 0.29302312300099165, + "scr_dir2_threshold_20": 0.19909487251754088, + "scr_dir1_threshold_50": 0.3302325052722394, + "scr_metric_threshold_50": 0.3302325052722394, + "scr_dir2_threshold_50": 0.2443439646255898, + "scr_dir1_threshold_100": 0.3767440944958292, + "scr_metric_threshold_100": 0.3767440944958292, + "scr_dir2_threshold_100": 0.2262443817232214, + "scr_dir1_threshold_500": 0.4511628590383247, + "scr_metric_threshold_500": 0.4511628590383247, + "scr_dir2_threshold_500": 0.2714932041270145 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-341796-tokens-1400000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-341796-tokens-1400000000/tpp/saebench_tpp_custom_sae_eval_results.json index 35994fb36f56da888ff2e1c67c1021f161a419dc..5b492397f051c3ef3a46bb2484f67afddb6b778c 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-341796-tokens-1400000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-341796-tokens-1400000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "1046eb3f-8c1b-4ba5-ba3c-594ea58905af", - "datetime_epoch_millis": 1776886177672, + "eval_id": "9eacd63d-b991-4aa3-80f2-c5cd6cb7f767", + "datetime_epoch_millis": 1777317798761, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.02099999934434891, - "tpp_threshold_2_intended_diff_only": 0.024000000953674317, - "tpp_threshold_2_unintended_diff_only": 0.003000001609325409, - "tpp_threshold_5_total_metric": 0.07117501050233842, - "tpp_threshold_5_intended_diff_only": 0.09110001325607299, - "tpp_threshold_5_unintended_diff_only": 0.019925002753734586, - "tpp_threshold_10_total_metric": 0.09664999395608902, - "tpp_threshold_10_intended_diff_only": 0.14990000128746034, - "tpp_threshold_10_unintended_diff_only": 0.05325000733137131, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.020099997520446777, + "tpp_threshold_2_intended_diff_only": 0.024300003051757814, + "tpp_threshold_2_unintended_diff_only": 0.004200005531311035, + "tpp_threshold_5_total_metric": 0.0722500130534172, + "tpp_threshold_5_intended_diff_only": 0.09150001406669617, + "tpp_threshold_5_unintended_diff_only": 0.019250001013278964, + "tpp_threshold_10_total_metric": 0.09729999452829362, + "tpp_threshold_10_intended_diff_only": 0.15040000081062319, + "tpp_threshold_10_unintended_diff_only": 0.05310000628232956, + "tpp_threshold_20_total_metric": 0.1315750077366829, + "tpp_threshold_20_intended_diff_only": 0.20780001282691954, + "tpp_threshold_20_unintended_diff_only": 0.07622500509023666, + "tpp_threshold_50_total_metric": 0.15980000793933868, + "tpp_threshold_50_intended_diff_only": 0.26720001697540285, + "tpp_threshold_50_unintended_diff_only": 0.10740000903606414, + "tpp_threshold_100_total_metric": 0.1829250082373619, + "tpp_threshold_100_intended_diff_only": 0.3314000189304352, + "tpp_threshold_100_unintended_diff_only": 0.14847501069307328, + "tpp_threshold_500_total_metric": 0.2333500251173973, + "tpp_threshold_500_intended_diff_only": 0.4380000352859497, + "tpp_threshold_500_unintended_diff_only": 0.2046500101685524 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.03469999432563782, - "tpp_threshold_2_intended_diff_only": 0.040400004386901854, - "tpp_threshold_2_unintended_diff_only": 0.005700010061264038, - "tpp_threshold_5_total_metric": 0.12625001072883607, - "tpp_threshold_5_intended_diff_only": 0.16380002498626708, - "tpp_threshold_5_unintended_diff_only": 0.03755001425743103, - "tpp_threshold_10_total_metric": 0.16529999673366547, + "tpp_threshold_2_total_metric": 0.034550005197525026, + "tpp_threshold_2_intended_diff_only": 0.04200001955032349, + "tpp_threshold_2_unintended_diff_only": 0.007450014352798462, + "tpp_threshold_5_total_metric": 0.12605001032352448, + "tpp_threshold_5_intended_diff_only": 0.16400002241134642, + "tpp_threshold_5_unintended_diff_only": 0.03795001208782196, + "tpp_threshold_10_total_metric": 0.16494999527931214, "tpp_threshold_10_intended_diff_only": 0.26860001087188723, - "tpp_threshold_10_unintended_diff_only": 0.10330001413822174, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_10_unintended_diff_only": 0.10365001559257507, + "tpp_threshold_20_total_metric": 0.21355001330375672, + "tpp_threshold_20_intended_diff_only": 0.3624000310897827, + "tpp_threshold_20_unintended_diff_only": 0.148850017786026, + "tpp_threshold_50_total_metric": 0.20535000562667846, + "tpp_threshold_50_intended_diff_only": 0.41320003271102906, + "tpp_threshold_50_unintended_diff_only": 0.20785002708435057, + "tpp_threshold_100_total_metric": 0.15640001595020295, + "tpp_threshold_100_intended_diff_only": 0.43960003852844237, + "tpp_threshold_100_unintended_diff_only": 0.28320002257823945, + "tpp_threshold_500_total_metric": 0.09075000882148743, + "tpp_threshold_500_intended_diff_only": 0.46720004081726074, + "tpp_threshold_500_unintended_diff_only": 0.3764500319957733 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.0073000043630599976, - "tpp_threshold_2_intended_diff_only": 0.0075999975204467775, - "tpp_threshold_2_unintended_diff_only": 0.0002999931573867798, - "tpp_threshold_5_total_metric": 0.016100010275840758, - "tpp_threshold_5_intended_diff_only": 0.018400001525878906, - "tpp_threshold_5_unintended_diff_only": 0.002299991250038147, - "tpp_threshold_10_total_metric": 0.027999991178512575, - "tpp_threshold_10_intended_diff_only": 0.031199991703033447, - "tpp_threshold_10_unintended_diff_only": 0.003200000524520874, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.005649989843368531, + "tpp_threshold_2_intended_diff_only": 0.006599986553192138, + "tpp_threshold_2_unintended_diff_only": 0.0009499967098236084, + "tpp_threshold_5_total_metric": 0.018450015783309938, + "tpp_threshold_5_intended_diff_only": 0.0190000057220459, + "tpp_threshold_5_unintended_diff_only": 0.0005499899387359619, + "tpp_threshold_10_total_metric": 0.029649993777275084, + "tpp_threshold_10_intended_diff_only": 0.03219999074935913, + "tpp_threshold_10_unintended_diff_only": 0.0025499969720840453, + "tpp_threshold_20_total_metric": 0.04960000216960907, + "tpp_threshold_20_intended_diff_only": 0.0531999945640564, + "tpp_threshold_20_unintended_diff_only": 0.0035999923944473266, + "tpp_threshold_50_total_metric": 0.1142500102519989, + "tpp_threshold_50_intended_diff_only": 0.12120000123977662, + "tpp_threshold_50_unintended_diff_only": 0.00694999098777771, + "tpp_threshold_100_total_metric": 0.20945000052452087, + "tpp_threshold_100_intended_diff_only": 0.22319999933242798, + "tpp_threshold_100_unintended_diff_only": 0.013749998807907105, + "tpp_threshold_500_total_metric": 0.3759500414133072, + "tpp_threshold_500_intended_diff_only": 0.4088000297546387, + "tpp_threshold_500_unintended_diff_only": 0.03284998834133148 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.0020000189542770386, "tpp_threshold_10_total_metric": 0.011749953031539917, "tpp_threshold_10_intended_diff_only": 0.014999985694885254, - "tpp_threshold_10_unintended_diff_only": 0.003250032663345337 + "tpp_threshold_10_unintended_diff_only": 0.003250032663345337, + "tpp_threshold_20_total_metric": 0.23874998092651367, + "tpp_threshold_20_intended_diff_only": 0.3190000057220459, + "tpp_threshold_20_unintended_diff_only": 0.08025002479553223, + "tpp_threshold_50_total_metric": 0.2254999428987503, + "tpp_threshold_50_intended_diff_only": 0.4089999794960022, + "tpp_threshold_50_unintended_diff_only": 0.1835000365972519, + "tpp_threshold_100_total_metric": 0.18924999237060547, + "tpp_threshold_100_intended_diff_only": 0.44200003147125244, + "tpp_threshold_100_unintended_diff_only": 0.252750039100647, + "tpp_threshold_500_total_metric": 0.09324999153614044, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.3627500385046005 }, "1": { - "tpp_threshold_2_total_metric": -2.9802322387695312e-08, - "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162, - "tpp_threshold_5_total_metric": 0.20000001788139343, + "tpp_threshold_2_total_metric": 0.004499971866607666, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 0.000500023365020752, + "tpp_threshold_5_total_metric": 0.20025001466274261, "tpp_threshold_5_intended_diff_only": 0.25700002908706665, - "tpp_threshold_5_unintended_diff_only": 0.05700001120567322, - "tpp_threshold_10_total_metric": 0.20800000429153442, + "tpp_threshold_5_unintended_diff_only": 0.056750014424324036, + "tpp_threshold_10_total_metric": 0.20299997925758362, "tpp_threshold_10_intended_diff_only": 0.3140000104904175, - "tpp_threshold_10_unintended_diff_only": 0.10600000619888306 + "tpp_threshold_10_unintended_diff_only": 0.11100003123283386, + "tpp_threshold_20_total_metric": 0.2227499932050705, + "tpp_threshold_20_intended_diff_only": 0.34200000762939453, + "tpp_threshold_20_unintended_diff_only": 0.11925001442432404, + "tpp_threshold_50_total_metric": 0.22325001657009125, + "tpp_threshold_50_intended_diff_only": 0.39000004529953003, + "tpp_threshold_50_unintended_diff_only": 0.16675002872943878, + "tpp_threshold_100_total_metric": 0.17649997770786285, + "tpp_threshold_100_intended_diff_only": 0.4259999990463257, + "tpp_threshold_100_unintended_diff_only": 0.24950002133846283, + "tpp_threshold_500_total_metric": 0.08125001192092896, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.377750039100647 }, "2": { - "tpp_threshold_2_total_metric": 0.14650000631809235, - "tpp_threshold_2_intended_diff_only": 0.17400002479553223, - "tpp_threshold_2_unintended_diff_only": 0.02750001847743988, - "tpp_threshold_5_total_metric": 0.20575004816055298, + "tpp_threshold_2_total_metric": 0.14125004410743713, + "tpp_threshold_2_intended_diff_only": 0.17600005865097046, + "tpp_threshold_2_unintended_diff_only": 0.034750014543533325, + "tpp_threshold_5_total_metric": 0.20950005948543549, "tpp_threshold_5_intended_diff_only": 0.2820000648498535, - "tpp_threshold_5_unintended_diff_only": 0.07625001668930054, - "tpp_threshold_10_total_metric": 0.1392500102519989, - "tpp_threshold_10_intended_diff_only": 0.3400000333786011, - "tpp_threshold_10_unintended_diff_only": 0.20075002312660217 + "tpp_threshold_5_unintended_diff_only": 0.07250000536441803, + "tpp_threshold_10_total_metric": 0.14225001633167267, + "tpp_threshold_10_intended_diff_only": 0.3410000205039978, + "tpp_threshold_10_unintended_diff_only": 0.19875000417232513, + "tpp_threshold_20_total_metric": 0.1052500307559967, + "tpp_threshold_20_intended_diff_only": 0.35700005292892456, + "tpp_threshold_20_unintended_diff_only": 0.25175002217292786, + "tpp_threshold_50_total_metric": 0.08875003457069397, + "tpp_threshold_50_intended_diff_only": 0.3760000467300415, + "tpp_threshold_50_unintended_diff_only": 0.28725001215934753, + "tpp_threshold_100_total_metric": 0.05075004696846008, + "tpp_threshold_100_intended_diff_only": 0.393000066280365, + "tpp_threshold_100_unintended_diff_only": 0.3422500193119049, + "tpp_threshold_500_total_metric": 0.0482499897480011, + "tpp_threshold_500_intended_diff_only": 0.4520000219345093, + "tpp_threshold_500_unintended_diff_only": 0.4037500321865082 }, "6": { - "tpp_threshold_2_total_metric": 0.011750012636184692, + "tpp_threshold_2_total_metric": 0.009499981999397278, "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": -0.002750009298324585, - "tpp_threshold_5_total_metric": 0.1757500171661377, - "tpp_threshold_5_intended_diff_only": 0.2070000171661377, - "tpp_threshold_5_unintended_diff_only": 0.03125, - "tpp_threshold_10_total_metric": 0.2652500122785568, + "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704, + "tpp_threshold_5_total_metric": 0.1757500022649765, + "tpp_threshold_5_intended_diff_only": 0.20800000429153442, + "tpp_threshold_5_unintended_diff_only": 0.03225000202655792, + "tpp_threshold_10_total_metric": 0.26874999701976776, "tpp_threshold_10_intended_diff_only": 0.3830000162124634, - "tpp_threshold_10_unintended_diff_only": 0.11775000393390656 + "tpp_threshold_10_unintended_diff_only": 0.11425001919269562, + "tpp_threshold_20_total_metric": 0.280750036239624, + "tpp_threshold_20_intended_diff_only": 0.4540000557899475, + "tpp_threshold_20_unintended_diff_only": 0.1732500195503235, + "tpp_threshold_50_total_metric": 0.24549999833106995, + "tpp_threshold_50_intended_diff_only": 0.4790000319480896, + "tpp_threshold_50_unintended_diff_only": 0.23350003361701965, + "tpp_threshold_100_total_metric": 0.1745000183582306, + "tpp_threshold_100_intended_diff_only": 0.4830000400543213, + "tpp_threshold_100_unintended_diff_only": 0.3085000216960907, + "tpp_threshold_500_total_metric": 0.11750000715255737, + "tpp_threshold_500_intended_diff_only": 0.4930000305175781, + "tpp_threshold_500_unintended_diff_only": 0.37550002336502075 }, "9": { - "tpp_threshold_2_total_metric": 0.003999993205070496, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_5_total_metric": 0.03475002944469452, + "tpp_threshold_2_total_metric": 0.006250038743019104, + "tpp_threshold_2_intended_diff_only": 0.00700002908706665, + "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.0297500342130661, "tpp_threshold_5_intended_diff_only": 0.05600005388259888, - "tpp_threshold_5_unintended_diff_only": 0.021250024437904358, - "tpp_threshold_10_total_metric": 0.20225000381469727, - "tpp_threshold_10_intended_diff_only": 0.29100000858306885, - "tpp_threshold_10_unintended_diff_only": 0.08875000476837158 + "tpp_threshold_5_unintended_diff_only": 0.026250019669532776, + "tpp_threshold_10_total_metric": 0.1990000307559967, + "tpp_threshold_10_intended_diff_only": 0.2900000214576721, + "tpp_threshold_10_unintended_diff_only": 0.09099999070167542, + "tpp_threshold_20_total_metric": 0.22025002539157867, + "tpp_threshold_20_intended_diff_only": 0.3400000333786011, + "tpp_threshold_20_unintended_diff_only": 0.1197500079870224, + "tpp_threshold_50_total_metric": 0.24375003576278687, + "tpp_threshold_50_intended_diff_only": 0.41200006008148193, + "tpp_threshold_50_unintended_diff_only": 0.16825002431869507, + "tpp_threshold_100_total_metric": 0.1910000443458557, + "tpp_threshold_100_intended_diff_only": 0.4540000557899475, + "tpp_threshold_100_unintended_diff_only": 0.2630000114440918, + "tpp_threshold_500_total_metric": 0.11350004374980927, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.36250002682209015 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.008999988436698914, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": -0.003999993205070496, - "tpp_threshold_5_total_metric": 0.01150001585483551, - "tpp_threshold_5_intended_diff_only": 0.009000003337860107, - "tpp_threshold_5_unintended_diff_only": -0.002500012516975403, - "tpp_threshold_10_total_metric": 0.0009999722242355347, - "tpp_threshold_10_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_10_unintended_diff_only": 0.005000010132789612 + "tpp_threshold_2_total_metric": 0.005250006914138794, + "tpp_threshold_2_intended_diff_only": 0.009000003337860107, + "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_5_total_metric": 0.00700002908706665, + "tpp_threshold_5_intended_diff_only": 0.008000016212463379, + "tpp_threshold_5_unintended_diff_only": 0.0009999871253967285, + "tpp_threshold_10_total_metric": 0.003000006079673767, + "tpp_threshold_10_intended_diff_only": 0.004999995231628418, + "tpp_threshold_10_unintended_diff_only": 0.001999989151954651, + "tpp_threshold_20_total_metric": 0.016249999403953552, + "tpp_threshold_20_intended_diff_only": 0.019999980926513672, + "tpp_threshold_20_unintended_diff_only": 0.0037499815225601196, + "tpp_threshold_50_total_metric": 0.06299999356269836, + "tpp_threshold_50_intended_diff_only": 0.06199997663497925, + "tpp_threshold_50_unintended_diff_only": -0.0010000169277191162, + "tpp_threshold_100_total_metric": 0.143249973654747, + "tpp_threshold_100_intended_diff_only": 0.1499999761581421, + "tpp_threshold_100_unintended_diff_only": 0.006750002503395081, + "tpp_threshold_500_total_metric": 0.41075004637241364, + "tpp_threshold_500_intended_diff_only": 0.4240000247955322, + "tpp_threshold_500_unintended_diff_only": 0.013249978423118591 }, "2": { - "tpp_threshold_2_total_metric": 0.0075000375509262085, - "tpp_threshold_2_intended_diff_only": 0.012000024318695068, - "tpp_threshold_2_unintended_diff_only": 0.00449998676776886, - "tpp_threshold_5_total_metric": 0.023750051856040955, - "tpp_threshold_5_intended_diff_only": 0.030000030994415283, - "tpp_threshold_5_unintended_diff_only": 0.006249979138374329, - "tpp_threshold_10_total_metric": 0.05175001919269562, + "tpp_threshold_2_total_metric": 0.016500011086463928, + "tpp_threshold_2_intended_diff_only": 0.013999998569488525, + "tpp_threshold_2_unintended_diff_only": -0.002500012516975403, + "tpp_threshold_5_total_metric": 0.019000038504600525, + "tpp_threshold_5_intended_diff_only": 0.026000022888183594, + "tpp_threshold_5_unintended_diff_only": 0.006999984383583069, + "tpp_threshold_10_total_metric": 0.04975001513957977, "tpp_threshold_10_intended_diff_only": 0.054000020027160645, - "tpp_threshold_10_unintended_diff_only": 0.002250000834465027 + "tpp_threshold_10_unintended_diff_only": 0.004250004887580872, + "tpp_threshold_20_total_metric": 0.07975003123283386, + "tpp_threshold_20_intended_diff_only": 0.08100003004074097, + "tpp_threshold_20_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_50_total_metric": 0.15800003707408905, + "tpp_threshold_50_intended_diff_only": 0.1690000295639038, + "tpp_threshold_50_unintended_diff_only": 0.010999992489814758, + "tpp_threshold_100_total_metric": 0.2757500857114792, + "tpp_threshold_100_intended_diff_only": 0.29600006341934204, + "tpp_threshold_100_unintended_diff_only": 0.020249977707862854, + "tpp_threshold_500_total_metric": 0.40750008821487427, + "tpp_threshold_500_intended_diff_only": 0.43900007009506226, + "tpp_threshold_500_unintended_diff_only": 0.03149998188018799 }, "3": { - "tpp_threshold_2_total_metric": 0.012000009417533875, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": -0.0070000141859054565, - "tpp_threshold_5_total_metric": 0.016499996185302734, - "tpp_threshold_5_intended_diff_only": 0.014999985694885254, - "tpp_threshold_5_unintended_diff_only": -0.0015000104904174805, - "tpp_threshold_10_total_metric": 0.01675000786781311, - "tpp_threshold_10_intended_diff_only": 0.018000006675720215, - "tpp_threshold_10_unintended_diff_only": 0.0012499988079071045 + "tpp_threshold_2_total_metric": 0.0007500052452087402, + "tpp_threshold_2_intended_diff_only": 0.0, + "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402, + "tpp_threshold_5_total_metric": 0.009749993681907654, + "tpp_threshold_5_intended_diff_only": 0.004999995231628418, + "tpp_threshold_5_unintended_diff_only": -0.004749998450279236, + "tpp_threshold_10_total_metric": 0.01600000262260437, + "tpp_threshold_10_intended_diff_only": 0.013999998569488525, + "tpp_threshold_10_unintended_diff_only": -0.0020000040531158447, + "tpp_threshold_20_total_metric": 0.007999986410140991, + "tpp_threshold_20_intended_diff_only": 0.010999977588653564, + "tpp_threshold_20_unintended_diff_only": 0.0029999911785125732, + "tpp_threshold_50_total_metric": 0.04050002992153168, + "tpp_threshold_50_intended_diff_only": 0.04000002145767212, + "tpp_threshold_50_unintended_diff_only": -0.0005000084638595581, + "tpp_threshold_100_total_metric": 0.08500000834465027, + "tpp_threshold_100_intended_diff_only": 0.09200000762939453, + "tpp_threshold_100_unintended_diff_only": 0.006999999284744263, + "tpp_threshold_500_total_metric": 0.35725001990795135, + "tpp_threshold_500_intended_diff_only": 0.3880000114440918, + "tpp_threshold_500_unintended_diff_only": 0.030749991536140442 }, "5": { - "tpp_threshold_2_total_metric": -0.005500033497810364, + "tpp_threshold_2_total_metric": 0.0029999613761901855, "tpp_threshold_2_intended_diff_only": 0.0029999613761901855, - "tpp_threshold_2_unintended_diff_only": 0.00849999487400055, - "tpp_threshold_5_total_metric": 0.01299998164176941, - "tpp_threshold_5_intended_diff_only": 0.014999985694885254, - "tpp_threshold_5_unintended_diff_only": 0.0020000040531158447, - "tpp_threshold_10_total_metric": 0.0172499418258667, - "tpp_threshold_10_intended_diff_only": 0.02199995517730713, - "tpp_threshold_10_unintended_diff_only": 0.00475001335144043 + "tpp_threshold_2_unintended_diff_only": 0.0, + "tpp_threshold_5_total_metric": 0.02049998939037323, + "tpp_threshold_5_intended_diff_only": 0.019999980926513672, + "tpp_threshold_5_unintended_diff_only": -0.0005000084638595581, + "tpp_threshold_10_total_metric": 0.012249961495399475, + "tpp_threshold_10_intended_diff_only": 0.01799994707107544, + "tpp_threshold_10_unintended_diff_only": 0.005749985575675964, + "tpp_threshold_20_total_metric": 0.04599998891353607, + "tpp_threshold_20_intended_diff_only": 0.046999990940093994, + "tpp_threshold_20_unintended_diff_only": 0.0010000020265579224, + "tpp_threshold_50_total_metric": 0.13274995982646942, + "tpp_threshold_50_intended_diff_only": 0.15099996328353882, + "tpp_threshold_50_unintended_diff_only": 0.018250003457069397, + "tpp_threshold_100_total_metric": 0.26899996399879456, + "tpp_threshold_100_intended_diff_only": 0.28999996185302734, + "tpp_threshold_100_unintended_diff_only": 0.020999997854232788, + "tpp_threshold_500_total_metric": 0.37400001287460327, + "tpp_threshold_500_intended_diff_only": 0.42000001668930054, + "tpp_threshold_500_unintended_diff_only": 0.046000003814697266 }, "6": { - "tpp_threshold_2_total_metric": 0.013500019907951355, - "tpp_threshold_2_intended_diff_only": 0.013000011444091797, - "tpp_threshold_2_unintended_diff_only": -0.0005000084638595581, - "tpp_threshold_5_total_metric": 0.015750005841255188, - "tpp_threshold_5_intended_diff_only": 0.023000001907348633, - "tpp_threshold_5_unintended_diff_only": 0.007249996066093445, - "tpp_threshold_10_total_metric": 0.053250014781951904, - "tpp_threshold_10_intended_diff_only": 0.0559999942779541, - "tpp_threshold_10_unintended_diff_only": 0.0027499794960021973 + "tpp_threshold_2_total_metric": 0.0027499645948410034, + "tpp_threshold_2_intended_diff_only": 0.006999969482421875, + "tpp_threshold_2_unintended_diff_only": 0.004250004887580872, + "tpp_threshold_5_total_metric": 0.036000028252601624, + "tpp_threshold_5_intended_diff_only": 0.03600001335144043, + "tpp_threshold_5_unintended_diff_only": -1.4901161193847656e-08, + "tpp_threshold_10_total_metric": 0.06724998354911804, + "tpp_threshold_10_intended_diff_only": 0.06999999284744263, + "tpp_threshold_10_unintended_diff_only": 0.002750009298324585, + "tpp_threshold_20_total_metric": 0.09800000488758087, + "tpp_threshold_20_intended_diff_only": 0.10699999332427979, + "tpp_threshold_20_unintended_diff_only": 0.008999988436698914, + "tpp_threshold_50_total_metric": 0.177000030875206, + "tpp_threshold_50_intended_diff_only": 0.18400001525878906, + "tpp_threshold_50_unintended_diff_only": 0.006999984383583069, + "tpp_threshold_100_total_metric": 0.27424997091293335, + "tpp_threshold_100_intended_diff_only": 0.2879999876022339, + "tpp_threshold_100_unintended_diff_only": 0.013750016689300537, + "tpp_threshold_500_total_metric": 0.3302500396966934, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.04274998605251312 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-36621-tokens-150000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-36621-tokens-150000000/scr/saebench_scr_custom_sae_eval_results.json index a7332020bfd676c5c4d7dd0e8b16a25e7ef1efaf..02ff5973abdd888cc5174b874596173e26eb6969 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-36621-tokens-150000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-36621-tokens-150000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "cca7884e-2c23-40d8-9267-172dcce0d980", - "datetime_epoch_millis": 1776886349225, + "eval_id": "24bb8c69-0395-42bf-9bbf-393aaebfb200", + "datetime_epoch_millis": 1777317528036, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.21228453030290387, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.31016100751738523, "scr_metric_threshold_10": 0.23549713630750585, "scr_dir2_threshold_10": 0.24712242598313489, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.31543387835359465, + "scr_metric_threshold_20": 0.3069632724374114, + "scr_dir2_threshold_20": 0.3149739227937308, + "scr_dir1_threshold_50": 0.1988876140030742, + "scr_metric_threshold_50": 0.37701390886542524, + "scr_dir2_threshold_50": 0.3821018004981899, + "scr_dir1_threshold_100": 0.25416206607371694, + "scr_metric_threshold_100": 0.43762714271230724, + "scr_dir2_threshold_100": 0.44252561875760116, + "scr_dir1_threshold_500": -0.2234237339772708, + "scr_metric_threshold_500": 0.22606711540544355, + "scr_dir2_threshold_500": 0.22959758188411156 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.5256410746257707, "scr_metric_threshold_10": 0.090673634335731, "scr_dir2_threshold_10": 0.090673634335731, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.5641028776049328, + "scr_metric_threshold_20": 0.23316053135219456, + "scr_dir2_threshold_20": 0.23316053135219456, + "scr_dir1_threshold_50": 0.3589742805987668, + "scr_metric_threshold_50": 0.2927460227800901, + "scr_dir2_threshold_50": 0.2927460227800901, + "scr_dir1_threshold_100": 0.4487182328294704, + "scr_metric_threshold_100": 0.31606209135692587, + "scr_dir2_threshold_100": 0.31606209135692587, + "scr_dir1_threshold_500": -1.4743589253742293, + "scr_metric_threshold_500": 0.12953369715840288, + "scr_dir2_threshold_500": 0.12953369715840288 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.4393940557023345, "scr_metric_threshold_10": 0.3516819627607703, "scr_dir2_threshold_10": 0.3516819627607703, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.4469697140738727, + "scr_metric_threshold_20": 0.40061172224601954, + "scr_dir2_threshold_20": 0.40061172224601954, + "scr_dir1_threshold_50": -0.05303006015100743, + "scr_metric_threshold_50": 0.5168196276077031, + "scr_dir2_threshold_50": 0.5168196276077031, + "scr_dir1_threshold_100": 0.24242422874090183, + "scr_metric_threshold_100": 0.5810397090856065, + "scr_dir2_threshold_100": 0.5810397090856065, + "scr_dir1_threshold_500": -0.5151510909679566, + "scr_metric_threshold_500": 0.28440363460714707, + "scr_dir2_threshold_500": 0.28440363460714707 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.5312495925464404, "scr_metric_threshold_10": 0.13316589123232284, "scr_dir2_threshold_10": 0.13316589123232284, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.43749988358469727, + "scr_metric_threshold_20": 0.16080402160563206, + "scr_dir2_threshold_20": 0.16080402160563206, + "scr_dir1_threshold_50": 0.3593749708961743, + "scr_metric_threshold_50": 0.2386935023309456, + "scr_dir2_threshold_50": 0.2386935023309456, + "scr_dir1_threshold_100": 0.2656252619344312, + "scr_metric_threshold_100": 0.2688441627817573, + "scr_dir2_threshold_100": 0.2688441627817573, + "scr_dir1_threshold_500": -0.4843752037267798, + "scr_metric_threshold_500": -0.08793960103532353, + "scr_dir2_threshold_500": -0.08793960103532353 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": 0.523489632191311, "scr_metric_threshold_10": 0.2388534572401228, "scr_dir2_threshold_10": 0.2388534572401228, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.42952990333245994, + "scr_metric_threshold_20": 0.3089171962431611, + "scr_dir2_threshold_20": 0.3089171962431611, + "scr_dir1_threshold_50": 0.41610725636599655, + "scr_metric_threshold_50": 0.3407642985653504, + "scr_dir2_threshold_50": 0.3407642985653504, + "scr_dir1_threshold_100": 0.4630873208110233, + "scr_metric_threshold_100": 0.5063695343586597, + "scr_dir2_threshold_100": 0.5063695343586597, + "scr_dir1_threshold_500": 0.013422646966463385, + "scr_metric_threshold_500": -0.3439489708328287, + "scr_dir2_threshold_500": -0.3439489708328287 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.08421069803000372, "scr_metric_threshold_10": 0.515872927644938, "scr_dir2_threshold_10": 0.515872927644938, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.06842111372221286, + "scr_metric_threshold_20": 0.6150794054390177, + "scr_dir2_threshold_20": 0.6150794054390177, + "scr_dir1_threshold_50": -0.18947375685884774, + "scr_metric_threshold_50": 0.6865079937623274, + "scr_dir2_threshold_50": 0.6865079937623274, + "scr_dir1_threshold_100": -0.1421053176441358, + "scr_metric_threshold_100": 0.6944443984532124, + "scr_dir2_threshold_100": 0.6944443984532124, + "scr_dir1_threshold_500": -0.26315796078077064, + "scr_metric_threshold_500": 0.595238157185469, + "scr_dir2_threshold_500": 0.595238157185469 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.1912566810955531, "scr_metric_threshold_10": 0.11450392446499717, "scr_dir2_threshold_10": 0.11450392446499717, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.251366184292382, + "scr_metric_threshold_20": 0.16412205665125212, + "scr_dir2_threshold_20": 0.16412205665125212, + "scr_dir1_threshold_50": 0.3224041847027747, + "scr_metric_threshold_50": 0.2709922648193172, + "scr_dir2_threshold_50": 0.2709922648193172, + "scr_dir1_threshold_100": 0.2950818016891243, + "scr_metric_threshold_100": 0.3702289841890732, + "scr_dir2_threshold_100": 0.3702289841890732, + "scr_dir1_threshold_500": 0.4153004823742847, + "scr_metric_threshold_500": 0.4618319872618971, + "scr_dir2_threshold_500": 0.4618319872618971 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.11162783863611286, "scr_metric_threshold_10": 0.3648068054696092, "scr_dir2_threshold_10": 0.3648068054696092, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.18604658254743003, + "scr_metric_threshold_20": 0.4334764782912447, + "scr_dir2_threshold_20": 0.4334764782912447, + "scr_dir1_threshold_50": 0.20930237071200095, + "scr_metric_threshold_50": 0.5021458952989327, + "scr_dir2_threshold_50": 0.5021458952989327, + "scr_dir1_threshold_100": 0.23720926106331344, + "scr_metric_threshold_100": 0.5407725223076157, + "scr_dir2_threshold_100": 0.5407725223076157, + "scr_dir1_threshold_500": 0.22790705668983022, + "scr_metric_threshold_500": 0.4763948958977931, + "scr_dir2_threshold_500": 0.4763948958977931 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.07441848731155573, "scr_metric_threshold_10": 0.07441848731155573, "scr_dir2_threshold_10": 0.16742080471658802, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.13953476767076942, + "scr_metric_threshold_20": 0.13953476767076942, + "scr_dir2_threshold_20": 0.20361997052132483, + "scr_dir1_threshold_50": 0.16744166575873534, + "scr_metric_threshold_50": 0.16744166575873534, + "scr_dir2_threshold_50": 0.208144798820853, + "scr_dir1_threshold_100": 0.22325573916560695, + "scr_metric_threshold_100": 0.22325573916560695, + "scr_dir2_threshold_100": 0.26244354752795823, + "scr_dir1_threshold_500": 0.29302312300099165, + "scr_metric_threshold_500": 0.29302312300099165, + "scr_dir2_threshold_500": 0.3212668548303358 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-36621-tokens-150000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-36621-tokens-150000000/tpp/saebench_tpp_custom_sae_eval_results.json index 1f79d1b75200e8580b7cfb4fa29f8d8ad1641e25..a52263365d05ef9874f02a2f5052bca75d1e10d3 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-36621-tokens-150000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-36621-tokens-150000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "5bc96b3d-e5d5-4509-a36b-af52fc6011a3", - "datetime_epoch_millis": 1776886504387, + "eval_id": "3fb463c9-ba39-4caa-9e4c-89b22f6f4a41", + "datetime_epoch_millis": 1777317803319, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.008874990046024323, - "tpp_threshold_2_intended_diff_only": 0.008799993991851806, - "tpp_threshold_2_unintended_diff_only": -7.499605417251589e-05, - "tpp_threshold_5_total_metric": 0.0228750005364418, - "tpp_threshold_5_intended_diff_only": 0.026200002431869505, - "tpp_threshold_5_unintended_diff_only": 0.003325001895427704, - "tpp_threshold_10_total_metric": 0.06649999469518661, - "tpp_threshold_10_intended_diff_only": 0.07329999804496765, - "tpp_threshold_10_unintended_diff_only": 0.006800003349781036, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.007924996316432953, + "tpp_threshold_2_intended_diff_only": 0.009100002050399781, + "tpp_threshold_2_unintended_diff_only": 0.0011750057339668274, + "tpp_threshold_5_total_metric": 0.024074992537498473, + "tpp_threshold_5_intended_diff_only": 0.025599998235702512, + "tpp_threshold_5_unintended_diff_only": 0.0015250056982040404, + "tpp_threshold_10_total_metric": 0.06730000227689743, + "tpp_threshold_10_intended_diff_only": 0.0746000051498413, + "tpp_threshold_10_unintended_diff_only": 0.0073000028729438785, + "tpp_threshold_20_total_metric": 0.11055000871419907, + "tpp_threshold_20_intended_diff_only": 0.12020001411437987, + "tpp_threshold_20_unintended_diff_only": 0.009650005400180817, + "tpp_threshold_50_total_metric": 0.19522501081228255, + "tpp_threshold_50_intended_diff_only": 0.21430001854896544, + "tpp_threshold_50_unintended_diff_only": 0.019075007736682893, + "tpp_threshold_100_total_metric": 0.2717500224709511, + "tpp_threshold_100_intended_diff_only": 0.30330002307891846, + "tpp_threshold_100_unintended_diff_only": 0.03155000060796738, + "tpp_threshold_500_total_metric": 0.32325002253055574, + "tpp_threshold_500_intended_diff_only": 0.4257000327110291, + "tpp_threshold_500_unintended_diff_only": 0.10245001018047332 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.009499993920326234, - "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": -0.0004999905824661255, - "tpp_threshold_5_total_metric": 0.031799998879432675, - "tpp_threshold_5_intended_diff_only": 0.03560000658035278, - "tpp_threshold_5_unintended_diff_only": 0.003800007700920105, - "tpp_threshold_10_total_metric": 0.11019999086856842, - "tpp_threshold_10_intended_diff_only": 0.12100000381469726, - "tpp_threshold_10_unintended_diff_only": 0.010800012946128845, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.008499985933303833, + "tpp_threshold_2_intended_diff_only": 0.009399998188018798, + "tpp_threshold_2_unintended_diff_only": 0.0009000122547149659, + "tpp_threshold_5_total_metric": 0.03449999392032623, + "tpp_threshold_5_intended_diff_only": 0.03680000305175781, + "tpp_threshold_5_unintended_diff_only": 0.0023000091314315796, + "tpp_threshold_10_total_metric": 0.10905000567436218, + "tpp_threshold_10_intended_diff_only": 0.12080001831054688, + "tpp_threshold_10_unintended_diff_only": 0.011750012636184692, + "tpp_threshold_20_total_metric": 0.1711000084877014, + "tpp_threshold_20_intended_diff_only": 0.18520002365112304, + "tpp_threshold_20_unintended_diff_only": 0.01410001516342163, + "tpp_threshold_50_total_metric": 0.29200000166893003, + "tpp_threshold_50_intended_diff_only": 0.32460001707077024, + "tpp_threshold_50_unintended_diff_only": 0.03260001540184021, + "tpp_threshold_100_total_metric": 0.3614000231027603, + "tpp_threshold_100_intended_diff_only": 0.4128000378608704, + "tpp_threshold_100_unintended_diff_only": 0.05140001475811005, + "tpp_threshold_500_total_metric": 0.2880500257015228, + "tpp_threshold_500_intended_diff_only": 0.4648000478744507, + "tpp_threshold_500_unintended_diff_only": 0.17675002217292785 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.008249986171722411, - "tpp_threshold_2_intended_diff_only": 0.008599984645843505, - "tpp_threshold_2_unintended_diff_only": 0.00034999847412109375, - "tpp_threshold_5_total_metric": 0.013950002193450928, - "tpp_threshold_5_intended_diff_only": 0.01679999828338623, - "tpp_threshold_5_unintended_diff_only": 0.0028499960899353027, - "tpp_threshold_10_total_metric": 0.02279999852180481, - "tpp_threshold_10_intended_diff_only": 0.025599992275238036, - "tpp_threshold_10_unintended_diff_only": 0.0027999937534332275, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.007350006699562072, + "tpp_threshold_2_intended_diff_only": 0.008800005912780762, + "tpp_threshold_2_unintended_diff_only": 0.001449999213218689, + "tpp_threshold_5_total_metric": 0.013649991154670716, + "tpp_threshold_5_intended_diff_only": 0.014399993419647216, + "tpp_threshold_5_unintended_diff_only": 0.0007500022649765015, + "tpp_threshold_10_total_metric": 0.025549998879432677, + "tpp_threshold_10_intended_diff_only": 0.02839999198913574, + "tpp_threshold_10_unintended_diff_only": 0.002849993109703064, + "tpp_threshold_20_total_metric": 0.050000008940696714, + "tpp_threshold_20_intended_diff_only": 0.05520000457763672, + "tpp_threshold_20_unintended_diff_only": 0.005199995636940002, + "tpp_threshold_50_total_metric": 0.09845001995563507, + "tpp_threshold_50_intended_diff_only": 0.10400002002716065, + "tpp_threshold_50_unintended_diff_only": 0.005550000071525574, + "tpp_threshold_100_total_metric": 0.18210002183914184, + "tpp_threshold_100_intended_diff_only": 0.19380000829696656, + "tpp_threshold_100_unintended_diff_only": 0.011699986457824708, + "tpp_threshold_500_total_metric": 0.35845001935958865, + "tpp_threshold_500_intended_diff_only": 0.3866000175476074, + "tpp_threshold_500_unintended_diff_only": 0.028149998188018797 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.0015000104904174805, "tpp_threshold_10_total_metric": 0.0342499315738678, "tpp_threshold_10_intended_diff_only": 0.039999961853027344, - "tpp_threshold_10_unintended_diff_only": 0.005750030279159546 + "tpp_threshold_10_unintended_diff_only": 0.005750030279159546, + "tpp_threshold_20_total_metric": 0.13749995827674866, + "tpp_threshold_20_intended_diff_only": 0.1589999794960022, + "tpp_threshold_20_unintended_diff_only": 0.02150002121925354, + "tpp_threshold_50_total_metric": 0.24274995923042297, + "tpp_threshold_50_intended_diff_only": 0.265999972820282, + "tpp_threshold_50_unintended_diff_only": 0.02325001358985901, + "tpp_threshold_100_total_metric": 0.3435000032186508, + "tpp_threshold_100_intended_diff_only": 0.37300002574920654, + "tpp_threshold_100_unintended_diff_only": 0.029500022530555725, + "tpp_threshold_500_total_metric": 0.2985000014305115, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.1575000286102295 }, "1": { - "tpp_threshold_2_total_metric": -0.0002500265836715698, - "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983, - "tpp_threshold_5_total_metric": 0.04749999940395355, + "tpp_threshold_2_total_metric": 0.004499971866607666, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 0.000500023365020752, + "tpp_threshold_5_total_metric": 0.050999999046325684, "tpp_threshold_5_intended_diff_only": 0.05500000715255737, - "tpp_threshold_5_unintended_diff_only": 0.007500007748603821, - "tpp_threshold_10_total_metric": 0.13649998605251312, - "tpp_threshold_10_intended_diff_only": 0.14800000190734863, - "tpp_threshold_10_unintended_diff_only": 0.01150001585483551 + "tpp_threshold_5_unintended_diff_only": 0.0040000081062316895, + "tpp_threshold_10_total_metric": 0.13799996674060822, + "tpp_threshold_10_intended_diff_only": 0.14899998903274536, + "tpp_threshold_10_unintended_diff_only": 0.011000022292137146, + "tpp_threshold_20_total_metric": 0.15974996984004974, + "tpp_threshold_20_intended_diff_only": 0.171999990940094, + "tpp_threshold_20_unintended_diff_only": 0.01225002110004425, + "tpp_threshold_50_total_metric": 0.27275002002716064, + "tpp_threshold_50_intended_diff_only": 0.3030000329017639, + "tpp_threshold_50_unintended_diff_only": 0.03025001287460327, + "tpp_threshold_100_total_metric": 0.33875003457069397, + "tpp_threshold_100_intended_diff_only": 0.39900004863739014, + "tpp_threshold_100_unintended_diff_only": 0.06025001406669617, + "tpp_threshold_500_total_metric": 0.2370000183582306, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.22200003266334534 }, "2": { - "tpp_threshold_2_total_metric": 0.021499991416931152, - "tpp_threshold_2_intended_diff_only": 0.018000006675720215, - "tpp_threshold_2_unintended_diff_only": -0.0034999847412109375, - "tpp_threshold_5_total_metric": 0.07300002872943878, - "tpp_threshold_5_intended_diff_only": 0.07600003480911255, - "tpp_threshold_5_unintended_diff_only": 0.003000006079673767, - "tpp_threshold_10_total_metric": 0.2199999988079071, - "tpp_threshold_10_intended_diff_only": 0.23500001430511475, - "tpp_threshold_10_unintended_diff_only": 0.015000015497207642 + "tpp_threshold_2_total_metric": 0.01075001060962677, + "tpp_threshold_2_intended_diff_only": 0.013000011444091797, + "tpp_threshold_2_unintended_diff_only": 0.002250000834465027, + "tpp_threshold_5_total_metric": 0.07375001907348633, + "tpp_threshold_5_intended_diff_only": 0.07700002193450928, + "tpp_threshold_5_unintended_diff_only": 0.0032500028610229492, + "tpp_threshold_10_total_metric": 0.2170000523328781, + "tpp_threshold_10_intended_diff_only": 0.23600006103515625, + "tpp_threshold_10_unintended_diff_only": 0.019000008702278137, + "tpp_threshold_20_total_metric": 0.2615000456571579, + "tpp_threshold_20_intended_diff_only": 0.2780000567436218, + "tpp_threshold_20_unintended_diff_only": 0.016500011086463928, + "tpp_threshold_50_total_metric": 0.30249999463558197, + "tpp_threshold_50_intended_diff_only": 0.3410000205039978, + "tpp_threshold_50_unintended_diff_only": 0.03850002586841583, + "tpp_threshold_100_total_metric": 0.3310000151395798, + "tpp_threshold_100_intended_diff_only": 0.38700002431869507, + "tpp_threshold_100_unintended_diff_only": 0.056000009179115295, + "tpp_threshold_500_total_metric": 0.22000005841255188, + "tpp_threshold_500_intended_diff_only": 0.4440000653266907, + "tpp_threshold_500_unintended_diff_only": 0.2240000069141388 }, "6": { - "tpp_threshold_2_total_metric": 0.011750012636184692, + "tpp_threshold_2_total_metric": 0.009499981999397278, "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": -0.002750009298324585, - "tpp_threshold_5_total_metric": 0.013000011444091797, - "tpp_threshold_5_intended_diff_only": 0.018000006675720215, - "tpp_threshold_5_unintended_diff_only": 0.004999995231628418, - "tpp_threshold_10_total_metric": 0.07600003480911255, - "tpp_threshold_10_intended_diff_only": 0.08500003814697266, - "tpp_threshold_10_unintended_diff_only": 0.009000003337860107 + "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704, + "tpp_threshold_5_total_metric": 0.01924997568130493, + "tpp_threshold_5_intended_diff_only": 0.018999993801116943, + "tpp_threshold_5_unintended_diff_only": -0.0002499818801879883, + "tpp_threshold_10_total_metric": 0.07725001871585846, + "tpp_threshold_10_intended_diff_only": 0.08600002527236938, + "tpp_threshold_10_unintended_diff_only": 0.008750006556510925, + "tpp_threshold_20_total_metric": 0.1287500411272049, + "tpp_threshold_20_intended_diff_only": 0.1350000500679016, + "tpp_threshold_20_unintended_diff_only": 0.006250008940696716, + "tpp_threshold_50_total_metric": 0.3552500158548355, + "tpp_threshold_50_intended_diff_only": 0.4100000262260437, + "tpp_threshold_50_unintended_diff_only": 0.05475001037120819, + "tpp_threshold_100_total_metric": 0.38600002229213715, + "tpp_threshold_100_intended_diff_only": 0.46400004625320435, + "tpp_threshold_100_unintended_diff_only": 0.0780000239610672, + "tpp_threshold_500_total_metric": 0.3410000056028366, + "tpp_threshold_500_intended_diff_only": 0.48900002241134644, + "tpp_threshold_500_unintended_diff_only": 0.14800001680850983 }, "9": { - "tpp_threshold_2_total_metric": 0.0050000399351119995, - "tpp_threshold_2_intended_diff_only": 0.006000041961669922, - "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_5_total_metric": 0.011999979615211487, - "tpp_threshold_5_intended_diff_only": 0.013999998569488525, - "tpp_threshold_5_unintended_diff_only": 0.0020000189542770386, - "tpp_threshold_10_total_metric": 0.08425000309944153, - "tpp_threshold_10_intended_diff_only": 0.09700000286102295, - "tpp_threshold_10_unintended_diff_only": 0.012749999761581421 + "tpp_threshold_2_total_metric": 0.008250012993812561, + "tpp_threshold_2_intended_diff_only": 0.009000003337860107, + "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.015000000596046448, + "tpp_threshold_5_intended_diff_only": 0.018000006675720215, + "tpp_threshold_5_unintended_diff_only": 0.003000006079673767, + "tpp_threshold_10_total_metric": 0.07875005900859833, + "tpp_threshold_10_intended_diff_only": 0.09300005435943604, + "tpp_threshold_10_unintended_diff_only": 0.014249995350837708, + "tpp_threshold_20_total_metric": 0.1680000275373459, + "tpp_threshold_20_intended_diff_only": 0.1820000410079956, + "tpp_threshold_20_unintended_diff_only": 0.01400001347064972, + "tpp_threshold_50_total_metric": 0.28675001859664917, + "tpp_threshold_50_intended_diff_only": 0.3030000329017639, + "tpp_threshold_50_unintended_diff_only": 0.016250014305114746, + "tpp_threshold_100_total_metric": 0.40775004029273987, + "tpp_threshold_100_intended_diff_only": 0.4410000443458557, + "tpp_threshold_100_unintended_diff_only": 0.033250004053115845, + "tpp_threshold_500_total_metric": 0.3437500447034836, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.13225002586841583 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.010749965906143188, - "tpp_threshold_2_intended_diff_only": 0.006999969482421875, - "tpp_threshold_2_unintended_diff_only": -0.0037499964237213135, - "tpp_threshold_5_total_metric": 0.011749982833862305, - "tpp_threshold_5_intended_diff_only": 0.009999990463256836, - "tpp_threshold_5_unintended_diff_only": -0.0017499923706054688, - "tpp_threshold_10_total_metric": 0.001749977469444275, - "tpp_threshold_10_intended_diff_only": 0.006999969482421875, - "tpp_threshold_10_unintended_diff_only": 0.0052499920129776 + "tpp_threshold_2_total_metric": 0.008000031113624573, + "tpp_threshold_2_intended_diff_only": 0.012000024318695068, + "tpp_threshold_2_unintended_diff_only": 0.003999993205070496, + "tpp_threshold_5_total_metric": 0.006750002503395081, + "tpp_threshold_5_intended_diff_only": 0.008000016212463379, + "tpp_threshold_5_unintended_diff_only": 0.0012500137090682983, + "tpp_threshold_10_total_metric": 0.003999993205070496, + "tpp_threshold_10_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_10_unintended_diff_only": 0.001999989151954651, + "tpp_threshold_20_total_metric": 0.025000005960464478, + "tpp_threshold_20_intended_diff_only": 0.027000010013580322, + "tpp_threshold_20_unintended_diff_only": 0.0020000040531158447, + "tpp_threshold_50_total_metric": 0.083250030875206, + "tpp_threshold_50_intended_diff_only": 0.08100003004074097, + "tpp_threshold_50_unintended_diff_only": -0.002250000834465027, + "tpp_threshold_100_total_metric": 0.1327500194311142, + "tpp_threshold_100_intended_diff_only": 0.1380000114440918, + "tpp_threshold_100_unintended_diff_only": 0.0052499920129776, + "tpp_threshold_500_total_metric": 0.38975001871585846, + "tpp_threshold_500_intended_diff_only": 0.3920000195503235, + "tpp_threshold_500_unintended_diff_only": 0.002250000834465027 }, "2": { - "tpp_threshold_2_total_metric": 0.0035000592470169067, - "tpp_threshold_2_intended_diff_only": 0.010000050067901611, - "tpp_threshold_2_unintended_diff_only": 0.006499990820884705, - "tpp_threshold_5_total_metric": 0.01700003445148468, - "tpp_threshold_5_intended_diff_only": 0.025000035762786865, - "tpp_threshold_5_unintended_diff_only": 0.008000001311302185, - "tpp_threshold_10_total_metric": 0.04400007426738739, - "tpp_threshold_10_intended_diff_only": 0.04700005054473877, - "tpp_threshold_10_unintended_diff_only": 0.0029999762773513794 + "tpp_threshold_2_total_metric": 0.010750040411949158, + "tpp_threshold_2_intended_diff_only": 0.01100003719329834, + "tpp_threshold_2_unintended_diff_only": 0.00024999678134918213, + "tpp_threshold_5_total_metric": 0.014250025153160095, + "tpp_threshold_5_intended_diff_only": 0.022000014781951904, + "tpp_threshold_5_unintended_diff_only": 0.007749989628791809, + "tpp_threshold_10_total_metric": 0.04425002634525299, + "tpp_threshold_10_intended_diff_only": 0.04900002479553223, + "tpp_threshold_10_unintended_diff_only": 0.004749998450279236, + "tpp_threshold_20_total_metric": 0.0777500569820404, + "tpp_threshold_20_intended_diff_only": 0.08400005102157593, + "tpp_threshold_20_unintended_diff_only": 0.0062499940395355225, + "tpp_threshold_50_total_metric": 0.12025007605552673, + "tpp_threshold_50_intended_diff_only": 0.1300000548362732, + "tpp_threshold_50_unintended_diff_only": 0.00974997878074646, + "tpp_threshold_100_total_metric": 0.27500008046627045, + "tpp_threshold_100_intended_diff_only": 0.30100005865097046, + "tpp_threshold_100_unintended_diff_only": 0.025999978184700012, + "tpp_threshold_500_total_metric": 0.4050000458955765, + "tpp_threshold_500_intended_diff_only": 0.437000036239624, + "tpp_threshold_500_unintended_diff_only": 0.031999990344047546 }, "3": { - "tpp_threshold_2_total_metric": 0.012749999761581421, - "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_2_unintended_diff_only": -0.006750017404556274, - "tpp_threshold_5_total_metric": 0.016499996185302734, - "tpp_threshold_5_intended_diff_only": 0.014999985694885254, - "tpp_threshold_5_unintended_diff_only": -0.0015000104904174805, - "tpp_threshold_10_total_metric": 0.01950003206729889, - "tpp_threshold_10_intended_diff_only": 0.021000027656555176, - "tpp_threshold_10_unintended_diff_only": 0.0014999955892562866 + "tpp_threshold_2_total_metric": 0.0029999762773513794, + "tpp_threshold_2_intended_diff_only": 0.001999974250793457, + "tpp_threshold_2_unintended_diff_only": -0.0010000020265579224, + "tpp_threshold_5_total_metric": 0.011499971151351929, + "tpp_threshold_5_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_5_unintended_diff_only": -0.005499988794326782, + "tpp_threshold_10_total_metric": 0.02000001072883606, + "tpp_threshold_10_intended_diff_only": 0.018000006675720215, + "tpp_threshold_10_unintended_diff_only": -0.0020000040531158447, + "tpp_threshold_20_total_metric": 0.008750036358833313, + "tpp_threshold_20_intended_diff_only": 0.012000024318695068, + "tpp_threshold_20_unintended_diff_only": 0.0032499879598617554, + "tpp_threshold_50_total_metric": 0.04725003242492676, + "tpp_threshold_50_intended_diff_only": 0.04400002956390381, + "tpp_threshold_50_unintended_diff_only": -0.0032500028610229492, + "tpp_threshold_100_total_metric": 0.10300001502037048, + "tpp_threshold_100_intended_diff_only": 0.10699999332427979, + "tpp_threshold_100_unintended_diff_only": 0.003999978303909302, + "tpp_threshold_500_total_metric": 0.33225004374980927, + "tpp_threshold_500_intended_diff_only": 0.3400000333786011, + "tpp_threshold_500_unintended_diff_only": 0.007749989628791809 }, "5": { - "tpp_threshold_2_total_metric": -0.008250057697296143, - "tpp_threshold_2_intended_diff_only": -0.001000046730041504, - "tpp_threshold_2_unintended_diff_only": 0.007250010967254639, - "tpp_threshold_5_total_metric": -0.001999989151954651, - "tpp_threshold_5_intended_diff_only": 0.0, - "tpp_threshold_5_unintended_diff_only": 0.001999989151954651, - "tpp_threshold_10_total_metric": 0.005999952554702759, - "tpp_threshold_10_intended_diff_only": 0.007999956607818604, - "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447 + "tpp_threshold_2_total_metric": 0.0007499754428863525, + "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, + "tpp_threshold_2_unintended_diff_only": 0.000250011682510376, + "tpp_threshold_5_total_metric": 0.006499975919723511, + "tpp_threshold_5_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_5_unintended_diff_only": -0.0004999935626983643, + "tpp_threshold_10_total_metric": 0.001999974250793457, + "tpp_threshold_10_intended_diff_only": 0.006999969482421875, + "tpp_threshold_10_unintended_diff_only": 0.004999995231628418, + "tpp_threshold_20_total_metric": 0.034249961376190186, + "tpp_threshold_20_intended_diff_only": 0.034999966621398926, + "tpp_threshold_20_unintended_diff_only": 0.0007500052452087402, + "tpp_threshold_50_total_metric": 0.0627499669790268, + "tpp_threshold_50_intended_diff_only": 0.07099997997283936, + "tpp_threshold_50_unintended_diff_only": 0.008250012993812561, + "tpp_threshold_100_total_metric": 0.12849996984004974, + "tpp_threshold_100_intended_diff_only": 0.1369999647140503, + "tpp_threshold_100_unintended_diff_only": 0.00849999487400055, + "tpp_threshold_500_total_metric": 0.3200000077486038, + "tpp_threshold_500_intended_diff_only": 0.3920000195503235, + "tpp_threshold_500_unintended_diff_only": 0.07200001180171967 }, "6": { - "tpp_threshold_2_total_metric": 0.022499963641166687, - "tpp_threshold_2_intended_diff_only": 0.0209999680519104, - "tpp_threshold_2_unintended_diff_only": -0.0014999955892562866, - "tpp_threshold_5_total_metric": 0.02649998664855957, - "tpp_threshold_5_intended_diff_only": 0.0339999794960022, - "tpp_threshold_5_unintended_diff_only": 0.007499992847442627, - "tpp_threshold_10_total_metric": 0.042749956250190735, - "tpp_threshold_10_intended_diff_only": 0.04499995708465576, - "tpp_threshold_10_unintended_diff_only": 0.002250000834465027 + "tpp_threshold_2_total_metric": 0.014250010251998901, + "tpp_threshold_2_intended_diff_only": 0.018000006675720215, + "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_5_total_metric": 0.02924998104572296, + "tpp_threshold_5_intended_diff_only": 0.029999971389770508, + "tpp_threshold_5_unintended_diff_only": 0.0007499903440475464, + "tpp_threshold_10_total_metric": 0.05749998986721039, + "tpp_threshold_10_intended_diff_only": 0.06199997663497925, + "tpp_threshold_10_unintended_diff_only": 0.00449998676776886, + "tpp_threshold_20_total_metric": 0.1042499840259552, + "tpp_threshold_20_intended_diff_only": 0.11799997091293335, + "tpp_threshold_20_unintended_diff_only": 0.01374998688697815, + "tpp_threshold_50_total_metric": 0.17874999344348907, + "tpp_threshold_50_intended_diff_only": 0.1940000057220459, + "tpp_threshold_50_unintended_diff_only": 0.015250012278556824, + "tpp_threshold_100_total_metric": 0.27125002443790436, + "tpp_threshold_100_intended_diff_only": 0.28600001335144043, + "tpp_threshold_100_unintended_diff_only": 0.014749988913536072, + "tpp_threshold_500_total_metric": 0.3452499806880951, + "tpp_threshold_500_intended_diff_only": 0.37199997901916504, + "tpp_threshold_500_unintended_diff_only": 0.026749998331069946 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-366210-tokens-1500000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-366210-tokens-1500000000/scr/saebench_scr_custom_sae_eval_results.json index 1bf0a57163ee211e97b0227fb637b83b2b99dddf..5a1be86310c96b9527f7c881b9020756802b75d0 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-366210-tokens-1500000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-366210-tokens-1500000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "e81d4b4c-86dd-4b68-af3d-4dea2d44039b", - "datetime_epoch_millis": 1776886866141, + "eval_id": "73319541-c05b-4ccc-8b94-f13b29a5f786", + "datetime_epoch_millis": 1777317574054, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.3064341517762689, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.2193571108810824, "scr_metric_threshold_10": 0.23299617539347295, "scr_dir2_threshold_10": 0.22370700659247295, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2380992428185572, + "scr_metric_threshold_20": 0.2664283945461302, + "scr_dir2_threshold_20": 0.257562767827177, + "scr_dir1_threshold_50": 0.20460997694614932, + "scr_metric_threshold_50": 0.3379127570315681, + "scr_dir2_threshold_50": 0.32995735300586543, + "scr_dir1_threshold_100": 0.050597810757260865, + "scr_metric_threshold_100": 0.320738200291545, + "scr_dir2_threshold_100": 0.2985004778092627, + "scr_dir1_threshold_500": -1.1350601095370043, + "scr_metric_threshold_500": 0.1776629517746716, + "scr_dir2_threshold_500": 0.163122795574028 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.4487182328294704, "scr_metric_threshold_10": 0.046632137153671566, "scr_dir2_threshold_10": 0.046632137153671566, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.14102610148224506, + "scr_metric_threshold_20": 0.13471497710162714, + "scr_dir2_threshold_20": 0.13471497710162714, + "scr_dir1_threshold_50": 0.4102564298503083, + "scr_metric_threshold_50": 0.25906739431680575, + "scr_dir2_threshold_50": 0.25906739431680575, + "scr_dir1_threshold_100": 0.5384614208981502, + "scr_metric_threshold_100": 0.3082901714420895, + "scr_dir2_threshold_100": 0.3082901714420895, + "scr_dir1_threshold_500": -2.5769224597152887, + "scr_metric_threshold_500": 0.08031092003311918, + "scr_dir2_threshold_500": 0.08031092003311918 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.5075758841466581, "scr_metric_threshold_10": 0.3608563382335518, "scr_dir2_threshold_10": 0.3608563382335518, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.5227272008897346, + "scr_metric_threshold_20": 0.2660550659387731, + "scr_dir2_threshold_20": 0.2660550659387731, + "scr_dir1_threshold_50": 0.38636354400108736, + "scr_metric_threshold_50": 0.49235474786507843, + "scr_dir2_threshold_50": 0.49235474786507843, + "scr_dir1_threshold_100": 0.5000002257751198, + "scr_metric_threshold_100": 0.5290520674790153, + "scr_dir2_threshold_100": 0.5290520674790153, + "scr_dir1_threshold_500": -1.8560602331895748, + "scr_metric_threshold_500": -0.321100954221084, + "scr_dir2_threshold_500": -0.321100954221084 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.43749988358469727, "scr_metric_threshold_10": 0.13065336115482035, "scr_dir2_threshold_10": 0.13065336115482035, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.09374970896174313, + "scr_metric_threshold_20": 0.17085429167603294, + "scr_dir2_threshold_20": 0.17085429167603294, + "scr_dir1_threshold_50": -0.7187499417923486, + "scr_metric_threshold_50": 0.16080402160563206, + "scr_dir2_threshold_50": 0.16080402160563206, + "scr_dir1_threshold_100": -0.921875087311477, + "scr_metric_threshold_100": 0.05025135035200431, + "scr_dir2_threshold_100": 0.05025135035200431, + "scr_dir1_threshold_500": -4.296874854480872, + "scr_metric_threshold_500": 0.0904522808732169, + "scr_dir2_threshold_500": 0.0904522808732169 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": -0.8120809423135005, "scr_metric_threshold_10": 0.11146504795136565, "scr_dir2_threshold_10": 0.11146504795136565, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.1744968107512386, + "scr_metric_threshold_20": 0.022293085519754383, + "scr_dir2_threshold_20": 0.022293085519754383, + "scr_dir1_threshold_50": 0.32214752750714554, + "scr_metric_threshold_50": -0.12101906475380056, + "scr_dir2_threshold_50": -0.12101906475380056, + "scr_dir1_threshold_100": -1.0469800644450267, + "scr_metric_threshold_100": -0.2547770084012175, + "scr_dir2_threshold_100": -0.2547770084012175, + "scr_dir1_threshold_500": -1.6912755194905202, + "scr_metric_threshold_500": -0.5382164468571459, + "scr_dir2_threshold_500": -0.5382164468571459 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.08947378822971379, "scr_metric_threshold_10": 0.45634918288429205, "scr_dir2_threshold_10": 0.45634918288429205, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.11578955293692479, + "scr_metric_threshold_20": 0.5277777712076017, + "scr_dir2_threshold_20": 0.5277777712076017, + "scr_dir1_threshold_50": -0.11052614902855408, + "scr_metric_threshold_50": 0.6865079937623274, + "scr_dir2_threshold_50": 0.6865079937623274, + "scr_dir1_threshold_100": -0.09999996862913393, + "scr_metric_threshold_100": 0.7380950973057522, + "scr_dir2_threshold_100": 0.7380950973057522, + "scr_dir1_threshold_500": -0.13684191373576507, + "scr_metric_threshold_500": 0.801587280938177, + "scr_dir2_threshold_500": 0.801587280938177 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.4371584539269045, "scr_metric_threshold_10": 0.12595415766246074, "scr_dir2_threshold_10": 0.12595415766246074, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.4153004823742847, + "scr_metric_threshold_20": 0.19847321124088896, + "scr_dir2_threshold_20": 0.19847321124088896, + "scr_dir1_threshold_50": 0.4590164254795243, + "scr_metric_threshold_50": 0.297709930610645, + "scr_dir2_threshold_50": 0.297709930610645, + "scr_dir1_threshold_100": 0.4863388084931748, + "scr_metric_threshold_100": 0.36259549539076413, + "scr_dir2_threshold_100": 0.36259549539076413, + "scr_dir1_threshold_500": 0.5519127231510342, + "scr_metric_threshold_500": 0.35496177909383203, + "scr_dir2_threshold_500": 0.35496177909383203 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.4093022598196558, "scr_metric_threshold_10": 0.3948498512825616, "scr_dir2_threshold_10": 0.3948498512825616, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.5116278940822855, + "scr_metric_threshold_20": 0.5321889411118851, + "scr_dir2_threshold_20": 0.5321889411118851, + "scr_dir1_threshold_50": 0.5441858866203395, + "scr_metric_threshold_50": 0.5836909399141642, + "scr_dir2_threshold_50": 0.5836909399141642, + "scr_dir1_threshold_100": 0.5627905725981688, + "scr_metric_threshold_100": 0.4463518500848407, + "scr_dir2_threshold_100": 0.4463518500848407, + "scr_dir1_threshold_500": 0.4744185221266269, + "scr_metric_threshold_500": 0.5021458952989327, + "scr_dir2_threshold_500": 0.5021458952989327 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.2372093268250598, "scr_metric_threshold_10": 0.2372093268250598, "scr_dir2_threshold_10": 0.16289597641705988, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.27906981257247854, + "scr_metric_threshold_20": 0.27906981257247854, + "scr_dir2_threshold_20": 0.208144798820853, + "scr_dir1_threshold_50": 0.3441860929316922, + "scr_metric_threshold_50": 0.3441860929316922, + "scr_dir2_threshold_50": 0.2805428607260708, + "scr_dir1_threshold_100": 0.386046578679111, + "scr_metric_threshold_100": 0.386046578679111, + "scr_dir2_threshold_100": 0.208144798820853, + "scr_dir1_threshold_500": 0.4511628590383247, + "scr_metric_threshold_500": 0.4511628590383247, + "scr_dir2_threshold_500": 0.33484160943317604 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-366210-tokens-1500000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-366210-tokens-1500000000/tpp/saebench_tpp_custom_sae_eval_results.json index a07cc956363069fbb69a5cb09965f7b379cd8374..f7494d474ec062d362a34c0609b32cff73fa4d23 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-366210-tokens-1500000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-366210-tokens-1500000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "3e660730-edb8-473e-9d5f-6f939a52b841", - "datetime_epoch_millis": 1776887026749, + "eval_id": "62838e27-2c52-4c09-b695-fa8bebfe94e5", + "datetime_epoch_millis": 1777317847872, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.02302500158548355, - "tpp_threshold_2_intended_diff_only": 0.026000005006790162, - "tpp_threshold_2_unintended_diff_only": 0.00297500342130661, - "tpp_threshold_5_total_metric": 0.08602501600980758, - "tpp_threshold_5_intended_diff_only": 0.11180001497268677, - "tpp_threshold_5_unintended_diff_only": 0.02577499896287918, - "tpp_threshold_10_total_metric": 0.10325000882148742, - "tpp_threshold_10_intended_diff_only": 0.15720001459121705, - "tpp_threshold_10_unintended_diff_only": 0.053950005769729616, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.022224995493888854, + "tpp_threshold_2_intended_diff_only": 0.026500004529953006, + "tpp_threshold_2_unintended_diff_only": 0.004275009036064148, + "tpp_threshold_5_total_metric": 0.08510000854730607, + "tpp_threshold_5_intended_diff_only": 0.11040001511573792, + "tpp_threshold_5_unintended_diff_only": 0.025300006568431854, + "tpp_threshold_10_total_metric": 0.10447500497102738, + "tpp_threshold_10_intended_diff_only": 0.1586000144481659, + "tpp_threshold_10_unintended_diff_only": 0.05412500947713852, + "tpp_threshold_20_total_metric": 0.13792500346899034, + "tpp_threshold_20_intended_diff_only": 0.21370001435279845, + "tpp_threshold_20_unintended_diff_only": 0.07577501088380813, + "tpp_threshold_50_total_metric": 0.16550001502037048, + "tpp_threshold_50_intended_diff_only": 0.27290002107620237, + "tpp_threshold_50_unintended_diff_only": 0.1074000060558319, + "tpp_threshold_100_total_metric": 0.18742501735687256, + "tpp_threshold_100_intended_diff_only": 0.33280002474784853, + "tpp_threshold_100_unintended_diff_only": 0.14537500739097595, + "tpp_threshold_500_total_metric": 0.2368500128388405, + "tpp_threshold_500_intended_diff_only": 0.43860002756118777, + "tpp_threshold_500_unintended_diff_only": 0.20175001472234727 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.037299996614456175, - "tpp_threshold_2_intended_diff_only": 0.043000006675720216, - "tpp_threshold_2_unintended_diff_only": 0.005700010061264038, - "tpp_threshold_5_total_metric": 0.15745001435279846, - "tpp_threshold_5_intended_diff_only": 0.20680001974105836, - "tpp_threshold_5_unintended_diff_only": 0.049350005388259885, - "tpp_threshold_10_total_metric": 0.1800000101327896, + "tpp_threshold_2_total_metric": 0.03739998936653137, + "tpp_threshold_2_intended_diff_only": 0.04480000734329224, + "tpp_threshold_2_unintended_diff_only": 0.007400017976760864, + "tpp_threshold_5_total_metric": 0.15745001137256623, + "tpp_threshold_5_intended_diff_only": 0.207200026512146, + "tpp_threshold_5_unintended_diff_only": 0.04975001513957977, + "tpp_threshold_10_total_metric": 0.17965000569820405, "tpp_threshold_10_intended_diff_only": 0.2850000262260437, - "tpp_threshold_10_unintended_diff_only": 0.10500001609325409, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_10_unintended_diff_only": 0.10535002052783966, + "tpp_threshold_20_total_metric": 0.22324999868869783, + "tpp_threshold_20_intended_diff_only": 0.3706000208854675, + "tpp_threshold_20_unintended_diff_only": 0.1473500221967697, + "tpp_threshold_50_total_metric": 0.20945002734661103, + "tpp_threshold_50_intended_diff_only": 0.4186000466346741, + "tpp_threshold_50_unintended_diff_only": 0.20915001928806304, + "tpp_threshold_100_total_metric": 0.16580002307891845, + "tpp_threshold_100_intended_diff_only": 0.4430000424385071, + "tpp_threshold_100_unintended_diff_only": 0.2772000193595886, + "tpp_threshold_500_total_metric": 0.09770000874996185, + "tpp_threshold_500_intended_diff_only": 0.4674000382423401, + "tpp_threshold_500_unintended_diff_only": 0.36970002949237823 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.008750006556510925, - "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": 0.00024999678134918213, - "tpp_threshold_5_total_metric": 0.014600017666816711, - "tpp_threshold_5_intended_diff_only": 0.016800010204315187, - "tpp_threshold_5_unintended_diff_only": 0.002199992537498474, - "tpp_threshold_10_total_metric": 0.02650000751018524, - "tpp_threshold_10_intended_diff_only": 0.02940000295639038, - "tpp_threshold_10_unintended_diff_only": 0.002899995446205139, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.0070500016212463375, + "tpp_threshold_2_intended_diff_only": 0.00820000171661377, + "tpp_threshold_2_unintended_diff_only": 0.0011500000953674316, + "tpp_threshold_5_total_metric": 0.012750005722045899, + "tpp_threshold_5_intended_diff_only": 0.013600003719329835, + "tpp_threshold_5_unintended_diff_only": 0.0008499979972839356, + "tpp_threshold_10_total_metric": 0.029300004243850708, + "tpp_threshold_10_intended_diff_only": 0.03220000267028809, + "tpp_threshold_10_unintended_diff_only": 0.002899998426437378, + "tpp_threshold_20_total_metric": 0.052600008249282834, + "tpp_threshold_20_intended_diff_only": 0.056800007820129395, + "tpp_threshold_20_unintended_diff_only": 0.004199999570846558, + "tpp_threshold_50_total_metric": 0.12155000269412994, + "tpp_threshold_50_intended_diff_only": 0.1271999955177307, + "tpp_threshold_50_unintended_diff_only": 0.005649992823600769, + "tpp_threshold_100_total_metric": 0.20905001163482667, + "tpp_threshold_100_intended_diff_only": 0.22260000705718994, + "tpp_threshold_100_unintended_diff_only": 0.013549995422363282, + "tpp_threshold_500_total_metric": 0.3760000169277191, + "tpp_threshold_500_intended_diff_only": 0.4098000168800354, + "tpp_threshold_500_unintended_diff_only": 0.033799999952316286 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.0015000104904174805, "tpp_threshold_10_total_metric": 0.04349999129772186, "tpp_threshold_10_intended_diff_only": 0.050000011920928955, - "tpp_threshold_10_unintended_diff_only": 0.006500020623207092 + "tpp_threshold_10_unintended_diff_only": 0.006500020623207092, + "tpp_threshold_20_total_metric": 0.2409999668598175, + "tpp_threshold_20_intended_diff_only": 0.35199999809265137, + "tpp_threshold_20_unintended_diff_only": 0.11100003123283386, + "tpp_threshold_50_total_metric": 0.226500004529953, + "tpp_threshold_50_intended_diff_only": 0.4100000262260437, + "tpp_threshold_50_unintended_diff_only": 0.1835000216960907, + "tpp_threshold_100_total_metric": 0.21800000965595245, + "tpp_threshold_100_intended_diff_only": 0.44200003147125244, + "tpp_threshold_100_unintended_diff_only": 0.2240000218153, + "tpp_threshold_500_total_metric": 0.11824999749660492, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.33775003254413605 }, "1": { - "tpp_threshold_2_total_metric": -2.9802322387695312e-08, - "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162, - "tpp_threshold_5_total_metric": 0.20250000059604645, + "tpp_threshold_2_total_metric": 0.004249975085258484, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 0.0007500201463699341, + "tpp_threshold_5_total_metric": 0.2032499611377716, "tpp_threshold_5_intended_diff_only": 0.2639999985694885, - "tpp_threshold_5_unintended_diff_only": 0.06149999797344208, - "tpp_threshold_10_total_metric": 0.2124999612569809, + "tpp_threshold_5_unintended_diff_only": 0.06075003743171692, + "tpp_threshold_10_total_metric": 0.2072499841451645, "tpp_threshold_10_intended_diff_only": 0.3199999928474426, - "tpp_threshold_10_unintended_diff_only": 0.10750003159046173 + "tpp_threshold_10_unintended_diff_only": 0.11275000870227814, + "tpp_threshold_20_total_metric": 0.22249998152256012, + "tpp_threshold_20_intended_diff_only": 0.34299999475479126, + "tpp_threshold_20_unintended_diff_only": 0.12050001323223114, + "tpp_threshold_50_total_metric": 0.22625002264976501, + "tpp_threshold_50_intended_diff_only": 0.39900004863739014, + "tpp_threshold_50_unintended_diff_only": 0.17275002598762512, + "tpp_threshold_100_total_metric": 0.18025001883506775, + "tpp_threshold_100_intended_diff_only": 0.4320000410079956, + "tpp_threshold_100_unintended_diff_only": 0.25175002217292786, + "tpp_threshold_500_total_metric": 0.0832500159740448, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.37575003504753113 }, "2": { - "tpp_threshold_2_total_metric": 0.1600000411272049, - "tpp_threshold_2_intended_diff_only": 0.18700003623962402, - "tpp_threshold_2_unintended_diff_only": 0.02699999511241913, - "tpp_threshold_5_total_metric": 0.20900003612041473, + "tpp_threshold_2_total_metric": 0.1549999862909317, + "tpp_threshold_2_intended_diff_only": 0.18900001049041748, + "tpp_threshold_2_unintended_diff_only": 0.03400002419948578, + "tpp_threshold_5_total_metric": 0.21400004625320435, "tpp_threshold_5_intended_diff_only": 0.28800004720687866, - "tpp_threshold_5_unintended_diff_only": 0.07900001108646393, - "tpp_threshold_10_total_metric": 0.14600004255771637, - "tpp_threshold_10_intended_diff_only": 0.34800004959106445, - "tpp_threshold_10_unintended_diff_only": 0.20200000703334808 + "tpp_threshold_5_unintended_diff_only": 0.07400000095367432, + "tpp_threshold_10_total_metric": 0.1497500091791153, + "tpp_threshold_10_intended_diff_only": 0.3490000367164612, + "tpp_threshold_10_unintended_diff_only": 0.1992500275373459, + "tpp_threshold_20_total_metric": 0.11675000190734863, + "tpp_threshold_20_intended_diff_only": 0.3630000352859497, + "tpp_threshold_20_unintended_diff_only": 0.24625003337860107, + "tpp_threshold_50_total_metric": 0.09075003862380981, + "tpp_threshold_50_intended_diff_only": 0.3880000710487366, + "tpp_threshold_50_unintended_diff_only": 0.29725003242492676, + "tpp_threshold_100_total_metric": 0.053750038146972656, + "tpp_threshold_100_intended_diff_only": 0.4020000696182251, + "tpp_threshold_100_unintended_diff_only": 0.34825003147125244, + "tpp_threshold_500_total_metric": 0.04649999737739563, + "tpp_threshold_500_intended_diff_only": 0.4520000219345093, + "tpp_threshold_500_unintended_diff_only": 0.40550002455711365 }, "6": { - "tpp_threshold_2_total_metric": 0.012499988079071045, + "tpp_threshold_2_total_metric": 0.010499969124794006, "tpp_threshold_2_intended_diff_only": 0.009999990463256836, - "tpp_threshold_2_unintended_diff_only": -0.002499997615814209, - "tpp_threshold_5_total_metric": 0.1927500218153, - "tpp_threshold_5_intended_diff_only": 0.2200000286102295, - "tpp_threshold_5_unintended_diff_only": 0.027250006794929504, - "tpp_threshold_10_total_metric": 0.2982500344514847, + "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704, + "tpp_threshold_5_total_metric": 0.19349999725818634, + "tpp_threshold_5_intended_diff_only": 0.22100001573562622, + "tpp_threshold_5_unintended_diff_only": 0.02750001847743988, + "tpp_threshold_10_total_metric": 0.30150002241134644, "tpp_threshold_10_intended_diff_only": 0.4180000424385071, - "tpp_threshold_10_unintended_diff_only": 0.1197500079870224 + "tpp_threshold_10_unintended_diff_only": 0.11650002002716064, + "tpp_threshold_20_total_metric": 0.30000001192092896, + "tpp_threshold_20_intended_diff_only": 0.4610000252723694, + "tpp_threshold_20_unintended_diff_only": 0.16100001335144043, + "tpp_threshold_50_total_metric": 0.24650003015995026, + "tpp_threshold_50_intended_diff_only": 0.4830000400543213, + "tpp_threshold_50_unintended_diff_only": 0.23650000989437103, + "tpp_threshold_100_total_metric": 0.17675000429153442, + "tpp_threshold_100_intended_diff_only": 0.484000027179718, + "tpp_threshold_100_unintended_diff_only": 0.3072500228881836, + "tpp_threshold_500_total_metric": 0.12324999272823334, + "tpp_threshold_500_intended_diff_only": 0.49400001764297485, + "tpp_threshold_500_unintended_diff_only": 0.3707500249147415 }, "9": { - "tpp_threshold_2_total_metric": 0.002749994397163391, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983, - "tpp_threshold_5_total_metric": 0.1665000170469284, - "tpp_threshold_5_intended_diff_only": 0.24400001764297485, - "tpp_threshold_5_unintended_diff_only": 0.07750000059604645, - "tpp_threshold_10_total_metric": 0.19975002110004425, - "tpp_threshold_10_intended_diff_only": 0.2890000343322754, - "tpp_threshold_10_unintended_diff_only": 0.08925001323223114 + "tpp_threshold_2_total_metric": 0.006000027060508728, + "tpp_threshold_2_intended_diff_only": 0.00700002908706665, + "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, + "tpp_threshold_5_total_metric": 0.1600000560283661, + "tpp_threshold_5_intended_diff_only": 0.24500006437301636, + "tpp_threshold_5_unintended_diff_only": 0.08500000834465027, + "tpp_threshold_10_total_metric": 0.19625002145767212, + "tpp_threshold_10_intended_diff_only": 0.28800004720687866, + "tpp_threshold_10_unintended_diff_only": 0.09175002574920654, + "tpp_threshold_20_total_metric": 0.23600003123283386, + "tpp_threshold_20_intended_diff_only": 0.3340000510215759, + "tpp_threshold_20_unintended_diff_only": 0.09800001978874207, + "tpp_threshold_50_total_metric": 0.257250040769577, + "tpp_threshold_50_intended_diff_only": 0.41300004720687866, + "tpp_threshold_50_unintended_diff_only": 0.15575000643730164, + "tpp_threshold_100_total_metric": 0.200250044465065, + "tpp_threshold_100_intended_diff_only": 0.45500004291534424, + "tpp_threshold_100_unintended_diff_only": 0.25474999845027924, + "tpp_threshold_500_total_metric": 0.11725004017353058, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.35875003039836884 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.009999975562095642, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": -0.004999980330467224, - "tpp_threshold_5_total_metric": 0.012500017881393433, - "tpp_threshold_5_intended_diff_only": 0.009000003337860107, - "tpp_threshold_5_unintended_diff_only": -0.003500014543533325, - "tpp_threshold_10_total_metric": 0.0007499903440475464, - "tpp_threshold_10_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_10_unintended_diff_only": 0.0052499920129776 + "tpp_threshold_2_total_metric": 0.00599999725818634, + "tpp_threshold_2_intended_diff_only": 0.009000003337860107, + "tpp_threshold_2_unintended_diff_only": 0.003000006079673767, + "tpp_threshold_5_total_metric": 0.005499988794326782, + "tpp_threshold_5_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_5_unintended_diff_only": 0.0004999935626983643, + "tpp_threshold_10_total_metric": 0.004499971866607666, + "tpp_threshold_10_intended_diff_only": 0.006999969482421875, + "tpp_threshold_10_unintended_diff_only": 0.002499997615814209, + "tpp_threshold_20_total_metric": 0.025499999523162842, + "tpp_threshold_20_intended_diff_only": 0.02799999713897705, + "tpp_threshold_20_unintended_diff_only": 0.002499997615814209, + "tpp_threshold_50_total_metric": 0.07224999368190765, + "tpp_threshold_50_intended_diff_only": 0.07099997997283936, + "tpp_threshold_50_unintended_diff_only": -0.0012500137090682983, + "tpp_threshold_100_total_metric": 0.14424999058246613, + "tpp_threshold_100_intended_diff_only": 0.15399998426437378, + "tpp_threshold_100_unintended_diff_only": 0.009749993681907654, + "tpp_threshold_500_total_metric": 0.41724999248981476, + "tpp_threshold_500_intended_diff_only": 0.4350000023841858, + "tpp_threshold_500_unintended_diff_only": 0.017750009894371033 }, "2": { - "tpp_threshold_2_total_metric": 0.006750032305717468, - "tpp_threshold_2_intended_diff_only": 0.012000024318695068, - "tpp_threshold_2_unintended_diff_only": 0.0052499920129776, - "tpp_threshold_5_total_metric": 0.013500064611434937, - "tpp_threshold_5_intended_diff_only": 0.020000040531158447, - "tpp_threshold_5_unintended_diff_only": 0.006499975919723511, - "tpp_threshold_10_total_metric": 0.03900006413459778, + "tpp_threshold_2_total_metric": 0.015750020742416382, + "tpp_threshold_2_intended_diff_only": 0.013999998569488525, + "tpp_threshold_2_unintended_diff_only": -0.0017500221729278564, + "tpp_threshold_5_total_metric": 0.008250042796134949, + "tpp_threshold_5_intended_diff_only": 0.016000032424926758, + "tpp_threshold_5_unintended_diff_only": 0.007749989628791809, + "tpp_threshold_10_total_metric": 0.037250056862831116, "tpp_threshold_10_intended_diff_only": 0.04200005531311035, - "tpp_threshold_10_unintended_diff_only": 0.0029999911785125732 + "tpp_threshold_10_unintended_diff_only": 0.004749998450279236, + "tpp_threshold_20_total_metric": 0.08600001037120819, + "tpp_threshold_20_intended_diff_only": 0.09200000762939453, + "tpp_threshold_20_unintended_diff_only": 0.00599999725818634, + "tpp_threshold_50_total_metric": 0.1597500443458557, + "tpp_threshold_50_intended_diff_only": 0.1730000376701355, + "tpp_threshold_50_unintended_diff_only": 0.013249993324279785, + "tpp_threshold_100_total_metric": 0.2812500447034836, + "tpp_threshold_100_intended_diff_only": 0.3020000457763672, + "tpp_threshold_100_unintended_diff_only": 0.020750001072883606, + "tpp_threshold_500_total_metric": 0.4060000777244568, + "tpp_threshold_500_intended_diff_only": 0.43900007009506226, + "tpp_threshold_500_unintended_diff_only": 0.03299999237060547 }, "3": { - "tpp_threshold_2_total_metric": 0.010750025510787964, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": -0.006750017404556274, - "tpp_threshold_5_total_metric": 0.01875002682209015, - "tpp_threshold_5_intended_diff_only": 0.017000019550323486, - "tpp_threshold_5_unintended_diff_only": -0.0017500072717666626, - "tpp_threshold_10_total_metric": 0.019250035285949707, - "tpp_threshold_10_intended_diff_only": 0.021000027656555176, - "tpp_threshold_10_unintended_diff_only": 0.0017499923706054688 + "tpp_threshold_2_total_metric": 0.0005000084638595581, + "tpp_threshold_2_intended_diff_only": 0.0, + "tpp_threshold_2_unintended_diff_only": -0.0005000084638595581, + "tpp_threshold_5_total_metric": 0.011250019073486328, + "tpp_threshold_5_intended_diff_only": 0.00700002908706665, + "tpp_threshold_5_unintended_diff_only": -0.004249989986419678, + "tpp_threshold_10_total_metric": 0.02000001072883606, + "tpp_threshold_10_intended_diff_only": 0.018000006675720215, + "tpp_threshold_10_unintended_diff_only": -0.0020000040531158447, + "tpp_threshold_20_total_metric": 0.010000035166740417, + "tpp_threshold_20_intended_diff_only": 0.012000024318695068, + "tpp_threshold_20_unintended_diff_only": 0.001999989151954651, + "tpp_threshold_50_total_metric": 0.03924998641014099, + "tpp_threshold_50_intended_diff_only": 0.038999974727630615, + "tpp_threshold_50_unintended_diff_only": -0.000250011682510376, + "tpp_threshold_100_total_metric": 0.09149999916553497, + "tpp_threshold_100_intended_diff_only": 0.09799998998641968, + "tpp_threshold_100_unintended_diff_only": 0.006499990820884705, + "tpp_threshold_500_total_metric": 0.3537500202655792, + "tpp_threshold_500_intended_diff_only": 0.3840000033378601, + "tpp_threshold_500_unintended_diff_only": 0.030249983072280884 }, "5": { - "tpp_threshold_2_total_metric": -0.0034999996423721313, + "tpp_threshold_2_total_metric": 0.004749983549118042, "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": 0.00849999487400055, - "tpp_threshold_5_total_metric": 0.01124998927116394, - "tpp_threshold_5_intended_diff_only": 0.013999998569488525, - "tpp_threshold_5_unintended_diff_only": 0.002750009298324585, - "tpp_threshold_10_total_metric": 0.017999976873397827, - "tpp_threshold_10_intended_diff_only": 0.019999980926513672, - "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447 + "tpp_threshold_2_unintended_diff_only": 0.000250011682510376, + "tpp_threshold_5_total_metric": 0.020749971270561218, + "tpp_threshold_5_intended_diff_only": 0.0209999680519104, + "tpp_threshold_5_unintended_diff_only": 0.00024999678134918213, + "tpp_threshold_10_total_metric": 0.01675000786781311, + "tpp_threshold_10_intended_diff_only": 0.023000001907348633, + "tpp_threshold_10_unintended_diff_only": 0.0062499940395355225, + "tpp_threshold_20_total_metric": 0.040499985218048096, + "tpp_threshold_20_intended_diff_only": 0.041999995708465576, + "tpp_threshold_20_unintended_diff_only": 0.0015000104904174805, + "tpp_threshold_50_total_metric": 0.1472499966621399, + "tpp_threshold_50_intended_diff_only": 0.15700000524520874, + "tpp_threshold_50_unintended_diff_only": 0.009750008583068848, + "tpp_threshold_100_total_metric": 0.25699999928474426, + "tpp_threshold_100_intended_diff_only": 0.27300000190734863, + "tpp_threshold_100_unintended_diff_only": 0.01600000262260437, + "tpp_threshold_500_total_metric": 0.3722499758005142, + "tpp_threshold_500_intended_diff_only": 0.4179999828338623, + "tpp_threshold_500_unintended_diff_only": 0.045750007033348083 }, "6": { - "tpp_threshold_2_total_metric": 0.019749999046325684, - "tpp_threshold_2_intended_diff_only": 0.018999993801116943, - "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402, - "tpp_threshold_5_total_metric": 0.0169999897480011, - "tpp_threshold_5_intended_diff_only": 0.02399998903274536, - "tpp_threshold_5_unintended_diff_only": 0.006999999284744263, - "tpp_threshold_10_total_metric": 0.05549997091293335, - "tpp_threshold_10_intended_diff_only": 0.05799996852874756, - "tpp_threshold_10_unintended_diff_only": 0.002499997615814209 + "tpp_threshold_2_total_metric": 0.008249998092651367, + "tpp_threshold_2_intended_diff_only": 0.013000011444091797, + "tpp_threshold_2_unintended_diff_only": 0.00475001335144043, + "tpp_threshold_5_total_metric": 0.018000006675720215, + "tpp_threshold_5_intended_diff_only": 0.018000006675720215, + "tpp_threshold_5_unintended_diff_only": 0.0, + "tpp_threshold_10_total_metric": 0.06799997389316559, + "tpp_threshold_10_intended_diff_only": 0.07099997997283936, + "tpp_threshold_10_unintended_diff_only": 0.003000006079673767, + "tpp_threshold_20_total_metric": 0.10100001096725464, + "tpp_threshold_20_intended_diff_only": 0.11000001430511475, + "tpp_threshold_20_unintended_diff_only": 0.009000003337860107, + "tpp_threshold_50_total_metric": 0.18924999237060547, + "tpp_threshold_50_intended_diff_only": 0.19599997997283936, + "tpp_threshold_50_unintended_diff_only": 0.006749987602233887, + "tpp_threshold_100_total_metric": 0.27125002443790436, + "tpp_threshold_100_intended_diff_only": 0.28600001335144043, + "tpp_threshold_100_unintended_diff_only": 0.014749988913536072, + "tpp_threshold_500_total_metric": 0.3307500183582306, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.04225000739097595 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-48828-tokens-200000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-48828-tokens-200000000/scr/saebench_scr_custom_sae_eval_results.json index 7ffcc624924e022163ecbdad336a0493966e26d5..82b06927e582d06eb4d02ab0052aee7b158e564a 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-48828-tokens-200000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-48828-tokens-200000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "5cc6d973-70a7-4cd6-b5f7-bf900f7b29c1", - "datetime_epoch_millis": 1776886607378, + "eval_id": "a37c4bb3-d5b0-4272-91e6-4c8751e20e9e", + "datetime_epoch_millis": 1777317623684, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.21586031401333375, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.1613699480060317, "scr_metric_threshold_10": 0.25517605668515764, "scr_dir2_threshold_10": 0.2593694763087202, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.1910093029330642, + "scr_metric_threshold_20": 0.3165838101358981, + "scr_dir2_threshold_20": 0.312574556185402, + "scr_dir1_threshold_50": 0.16166733397527638, + "scr_metric_threshold_50": 0.39608407394084566, + "scr_dir2_threshold_50": 0.3930008270806805, + "scr_dir1_threshold_100": 0.18280301469479515, + "scr_metric_threshold_100": 0.4187884297226195, + "scr_dir2_threshold_100": 0.4200722327355718, + "scr_dir1_threshold_500": -0.2977460896134362, + "scr_metric_threshold_500": 0.2694611342827654, + "scr_dir2_threshold_500": 0.27021090255243746 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.5384614208981502, "scr_metric_threshold_10": 0.12435226279901532, "scr_dir2_threshold_10": 0.12435226279901532, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.5512825313325533, + "scr_metric_threshold_20": 0.20207254286052237, + "scr_dir2_threshold_20": 0.20207254286052237, + "scr_dir1_threshold_50": 0.5384614208981502, + "scr_metric_threshold_50": 0.29792745713947766, + "scr_dir2_threshold_50": 0.29792745713947766, + "scr_dir1_threshold_100": 0.5000003820810118, + "scr_metric_threshold_100": 0.3212435257163134, + "scr_dir2_threshold_100": 0.3212435257163134, + "scr_dir1_threshold_500": -1.0512813850895177, + "scr_metric_threshold_500": 0.3704663028415971, + "scr_dir2_threshold_500": 0.3704663028415971 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.10606057185225456, "scr_metric_threshold_10": 0.34862389836223945, "scr_dir2_threshold_10": 0.34862389836223945, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.04545485332970889, + "scr_metric_threshold_20": 0.35474002715930114, + "scr_dir2_threshold_20": 0.35474002715930114, + "scr_dir1_threshold_50": 0.03787874340793094, + "scr_metric_threshold_50": 0.5412845073503276, + "scr_dir2_threshold_50": 0.5412845073503276, + "scr_dir1_threshold_100": 0.2878790820706107, + "scr_metric_threshold_100": 0.5779816446870757, + "scr_dir2_threshold_100": 0.5779816446870757, + "scr_dir1_threshold_500": -0.8636358915611131, + "scr_metric_threshold_500": 0.29969413887699026, + "scr_dir2_threshold_500": 0.29969413887699026 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.17187462165026607, "scr_metric_threshold_10": 0.128140681316927, "scr_dir2_threshold_10": 0.128140681316927, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.28125005820765137, + "scr_metric_threshold_20": 0.16834176159853043, + "scr_dir2_threshold_20": 0.16834176159853043, + "scr_dir1_threshold_50": 0.3593749708961743, + "scr_metric_threshold_50": 0.2386935023309456, + "scr_dir2_threshold_50": 0.2386935023309456, + "scr_dir1_threshold_100": -0.09374970896174313, + "scr_metric_threshold_100": 0.042713610359105945, + "scr_dir2_threshold_100": 0.042713610359105945, + "scr_dir1_threshold_500": 0.0, + "scr_metric_threshold_500": -0.04522614043660845, + "scr_dir2_threshold_500": -0.04522614043660845 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": -0.1744968107512386, "scr_metric_threshold_10": 0.2611465427598772, "scr_dir2_threshold_10": 0.2611465427598772, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.1409397933038777, + "scr_metric_threshold_20": 0.3598727118176265, + "scr_dir2_threshold_20": 0.3598727118176265, + "scr_dir1_threshold_50": -0.40268460939953316, + "scr_metric_threshold_50": 0.4522293465167161, + "scr_dir2_threshold_50": 0.4522293465167161, + "scr_dir1_threshold_100": -0.12751714633741432, + "scr_metric_threshold_100": 0.5732484112705166, + "scr_dir2_threshold_100": 0.5732484112705166, + "scr_dir1_threshold_500": -1.2684565396100895, + "scr_metric_threshold_500": -0.38853495204863436, + "scr_dir2_threshold_500": -0.38853495204863436 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.08421069803000372, "scr_metric_threshold_10": 0.5396826147702655, "scr_dir2_threshold_10": 0.5396826147702655, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.0631580235225028, + "scr_metric_threshold_20": 0.6190476077844602, + "scr_dir2_threshold_20": 0.6190476077844602, + "scr_dir1_threshold_50": -0.17894726275076694, + "scr_metric_threshold_50": 0.6865079937623274, + "scr_dir2_threshold_50": 0.6865079937623274, + "scr_dir1_threshold_100": -0.15263149804355594, + "scr_metric_threshold_100": 0.7063492420158761, + "scr_dir2_threshold_100": 0.7063492420158761, + "scr_dir1_threshold_500": -0.2473683764729798, + "scr_metric_threshold_500": 0.6428570583834513, + "scr_dir2_threshold_500": 0.6428570583834513 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.2950818016891243, "scr_metric_threshold_10": 0.12977090206161526, "scr_dir2_threshold_10": 0.12977090206161526, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3278689218723027, + "scr_metric_threshold_20": 0.17938926174649328, + "scr_dir2_threshold_20": 0.17938926174649328, + "scr_dir1_threshold_50": 0.4043716594522235, + "scr_metric_threshold_50": 0.22519087703221677, + "scr_dir2_threshold_50": 0.22519087703221677, + "scr_dir1_threshold_100": 0.3879780993606342, + "scr_metric_threshold_100": 0.31679388010504067, + "scr_dir2_threshold_100": 0.31679388010504067, + "scr_dir1_threshold_500": 0.5464479859815062, + "scr_metric_threshold_500": 0.4694657035588292, + "scr_dir2_threshold_500": 0.4694657035588292 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.17209299875634235, "scr_metric_threshold_10": 0.41201726948797046, "scr_dir2_threshold_10": 0.41201726948797046, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.21860457508548414, + "scr_metric_threshold_20": 0.46781131470206244, + "scr_dir2_threshold_20": 0.46781131470206244, + "scr_dir1_threshold_50": 0.2930233189968013, + "scr_metric_threshold_50": 0.48497847709352376, + "scr_dir2_threshold_50": 0.48497847709352376, + "scr_dir1_threshold_100": 0.3720928878639972, + "scr_metric_threshold_100": 0.5236051041022068, + "scr_dir2_threshold_100": 0.5236051041022068, + "scr_dir1_threshold_500": 0.22325567727222573, + "scr_metric_threshold_500": 0.5278971505140198, + "scr_dir2_threshold_500": 0.5278971505140198 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.09767428192335063, "scr_metric_threshold_10": 0.09767428192335063, "scr_dir2_threshold_10": 0.1312216389118512, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.18139525341818818, + "scr_metric_threshold_20": 0.18139525341818818, + "scr_dir2_threshold_20": 0.1493212218142196, + "scr_dir1_threshold_50": 0.24186043030123083, + "scr_metric_threshold_50": 0.24186043030123083, + "scr_dir2_threshold_50": 0.2171944554199093, + "scr_dir1_threshold_100": 0.2883720195248206, + "scr_metric_threshold_100": 0.2883720195248206, + "scr_dir2_threshold_100": 0.2986424436284392, + "scr_dir1_threshold_500": 0.27906981257247854, + "scr_metric_threshold_500": 0.27906981257247854, + "scr_dir2_threshold_500": 0.2850679587298548 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-48828-tokens-200000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-48828-tokens-200000000/tpp/saebench_tpp_custom_sae_eval_results.json index 341fe80cdd64f84753239297351ac57f0c01a51d..9067d067aedc2b0e073ab2b2c64a2a00a98cebfd 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-48828-tokens-200000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-48828-tokens-200000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "78264222-05e3-4676-a40e-b5ca2e6e59c4", - "datetime_epoch_millis": 1776886768606, + "eval_id": "ed5e8dd3-ebe7-4034-9a63-cce13150c43a", + "datetime_epoch_millis": 1777317899194, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.009649993479251861, - "tpp_threshold_2_intended_diff_only": 0.009399998188018798, - "tpp_threshold_2_unintended_diff_only": -0.0002499952912330627, - "tpp_threshold_5_total_metric": 0.03132500797510147, - "tpp_threshold_5_intended_diff_only": 0.034900009632110596, - "tpp_threshold_5_unintended_diff_only": 0.0035750016570091246, - "tpp_threshold_10_total_metric": 0.07525000125169753, - "tpp_threshold_10_intended_diff_only": 0.08310000896453858, - "tpp_threshold_10_unintended_diff_only": 0.007850007712841034, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.007799991965293884, + "tpp_threshold_2_intended_diff_only": 0.008899998664855958, + "tpp_threshold_2_unintended_diff_only": 0.0011000066995620727, + "tpp_threshold_5_total_metric": 0.031874993443489076, + "tpp_threshold_5_intended_diff_only": 0.033799999952316286, + "tpp_threshold_5_unintended_diff_only": 0.0019250065088272095, + "tpp_threshold_10_total_metric": 0.07579998970031739, + "tpp_threshold_10_intended_diff_only": 0.08429999351501465, + "tpp_threshold_10_unintended_diff_only": 0.008500003814697265, + "tpp_threshold_20_total_metric": 0.12705000191926957, + "tpp_threshold_20_intended_diff_only": 0.13920000791549683, + "tpp_threshold_20_unintended_diff_only": 0.012150005996227264, + "tpp_threshold_50_total_metric": 0.19905001521110532, + "tpp_threshold_50_intended_diff_only": 0.22660001516342163, + "tpp_threshold_50_unintended_diff_only": 0.027549999952316283, + "tpp_threshold_100_total_metric": 0.255075016617775, + "tpp_threshold_100_intended_diff_only": 0.3065000236034393, + "tpp_threshold_100_unintended_diff_only": 0.051425006985664365, + "tpp_threshold_500_total_metric": 0.2917000249028206, + "tpp_threshold_500_intended_diff_only": 0.4251000344753265, + "tpp_threshold_500_unintended_diff_only": 0.13340000957250595 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.01219998300075531, - "tpp_threshold_2_intended_diff_only": 0.011599993705749512, - "tpp_threshold_2_unintended_diff_only": -0.0005999892950057983, - "tpp_threshold_5_total_metric": 0.049300011992454526, - "tpp_threshold_5_intended_diff_only": 0.0538000226020813, - "tpp_threshold_5_unintended_diff_only": 0.00450001060962677, - "tpp_threshold_10_total_metric": 0.12895000278949736, - "tpp_threshold_10_intended_diff_only": 0.1414000153541565, - "tpp_threshold_10_unintended_diff_only": 0.012450012564659118, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.010449990630149841, + "tpp_threshold_2_intended_diff_only": 0.01140000820159912, + "tpp_threshold_2_unintended_diff_only": 0.0009500175714492798, + "tpp_threshold_5_total_metric": 0.05114998817443848, + "tpp_threshold_5_intended_diff_only": 0.05460000038146973, + "tpp_threshold_5_unintended_diff_only": 0.00345001220703125, + "tpp_threshold_10_total_metric": 0.12804999053478242, + "tpp_threshold_10_intended_diff_only": 0.14140000343322753, + "tpp_threshold_10_unintended_diff_only": 0.013350012898445129, + "tpp_threshold_20_total_metric": 0.20720000565052032, + "tpp_threshold_20_intended_diff_only": 0.22640001773834229, + "tpp_threshold_20_unintended_diff_only": 0.01920001208782196, + "tpp_threshold_50_total_metric": 0.3053500235080719, + "tpp_threshold_50_intended_diff_only": 0.3548000335693359, + "tpp_threshold_50_unintended_diff_only": 0.04945001006126404, + "tpp_threshold_100_total_metric": 0.33660002052783966, + "tpp_threshold_100_intended_diff_only": 0.42760003805160524, + "tpp_threshold_100_unintended_diff_only": 0.09100001752376556, + "tpp_threshold_500_total_metric": 0.22840003073215484, + "tpp_threshold_500_intended_diff_only": 0.466200053691864, + "tpp_threshold_500_unintended_diff_only": 0.23780002295970917 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.007100003957748413, - "tpp_threshold_2_intended_diff_only": 0.007200002670288086, - "tpp_threshold_2_unintended_diff_only": 9.999871253967285e-05, - "tpp_threshold_5_total_metric": 0.013350003957748413, - "tpp_threshold_5_intended_diff_only": 0.015999996662139894, - "tpp_threshold_5_unintended_diff_only": 0.0026499927043914795, - "tpp_threshold_10_total_metric": 0.021549999713897705, - "tpp_threshold_10_intended_diff_only": 0.024800002574920654, - "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.005149993300437927, + "tpp_threshold_2_intended_diff_only": 0.006399989128112793, + "tpp_threshold_2_unintended_diff_only": 0.0012499958276748658, + "tpp_threshold_5_total_metric": 0.012599998712539673, + "tpp_threshold_5_intended_diff_only": 0.012999999523162841, + "tpp_threshold_5_unintended_diff_only": 0.0004000008106231689, + "tpp_threshold_10_total_metric": 0.023549988865852356, + "tpp_threshold_10_intended_diff_only": 0.027199983596801758, + "tpp_threshold_10_unintended_diff_only": 0.003649994730949402, + "tpp_threshold_20_total_metric": 0.0468999981880188, + "tpp_threshold_20_intended_diff_only": 0.051999998092651364, + "tpp_threshold_20_unintended_diff_only": 0.0050999999046325685, + "tpp_threshold_50_total_metric": 0.09275000691413879, + "tpp_threshold_50_intended_diff_only": 0.09839999675750732, + "tpp_threshold_50_unintended_diff_only": 0.005649989843368531, + "tpp_threshold_100_total_metric": 0.17355001270771026, + "tpp_threshold_100_intended_diff_only": 0.18540000915527344, + "tpp_threshold_100_unintended_diff_only": 0.011849996447563172, + "tpp_threshold_500_total_metric": 0.35500001907348633, + "tpp_threshold_500_intended_diff_only": 0.3840000152587891, + "tpp_threshold_500_unintended_diff_only": 0.028999996185302735 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.0012500137090682983, "tpp_threshold_10_total_metric": 0.009249940514564514, "tpp_threshold_10_intended_diff_only": 0.011999964714050293, - "tpp_threshold_10_unintended_diff_only": 0.002750024199485779 + "tpp_threshold_10_unintended_diff_only": 0.002750024199485779, + "tpp_threshold_20_total_metric": 0.1772499680519104, + "tpp_threshold_20_intended_diff_only": 0.20399999618530273, + "tpp_threshold_20_unintended_diff_only": 0.026750028133392334, + "tpp_threshold_50_total_metric": 0.2902500182390213, + "tpp_threshold_50_intended_diff_only": 0.33100003004074097, + "tpp_threshold_50_unintended_diff_only": 0.040750011801719666, + "tpp_threshold_100_total_metric": 0.3554999530315399, + "tpp_threshold_100_intended_diff_only": 0.4129999876022339, + "tpp_threshold_100_unintended_diff_only": 0.05750003457069397, + "tpp_threshold_500_total_metric": 0.24500000476837158, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.21100002527236938 }, "1": { - "tpp_threshold_2_total_metric": 0.0002499818801879883, - "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 0.0007500052452087402, - "tpp_threshold_5_total_metric": 0.06250002980232239, - "tpp_threshold_5_intended_diff_only": 0.07200002670288086, - "tpp_threshold_5_unintended_diff_only": 0.009499996900558472, - "tpp_threshold_10_total_metric": 0.14600001275539398, - "tpp_threshold_10_intended_diff_only": 0.16500002145767212, - "tpp_threshold_10_unintended_diff_only": 0.019000008702278137 + "tpp_threshold_2_total_metric": 0.004499971866607666, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 0.000500023365020752, + "tpp_threshold_5_total_metric": 0.06474997103214264, + "tpp_threshold_5_intended_diff_only": 0.07099997997283936, + "tpp_threshold_5_unintended_diff_only": 0.006250008940696716, + "tpp_threshold_10_total_metric": 0.14949999749660492, + "tpp_threshold_10_intended_diff_only": 0.16600000858306885, + "tpp_threshold_10_unintended_diff_only": 0.016500011086463928, + "tpp_threshold_20_total_metric": 0.17675000429153442, + "tpp_threshold_20_intended_diff_only": 0.1940000057220459, + "tpp_threshold_20_unintended_diff_only": 0.017250001430511475, + "tpp_threshold_50_total_metric": 0.2604999840259552, + "tpp_threshold_50_intended_diff_only": 0.3059999942779541, + "tpp_threshold_50_unintended_diff_only": 0.0455000102519989, + "tpp_threshold_100_total_metric": 0.3087500333786011, + "tpp_threshold_100_intended_diff_only": 0.40400004386901855, + "tpp_threshold_100_unintended_diff_only": 0.09525001049041748, + "tpp_threshold_500_total_metric": 0.1952500343322754, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.26375001668930054 }, "2": { - "tpp_threshold_2_total_metric": 0.034000009298324585, - "tpp_threshold_2_intended_diff_only": 0.03100001811981201, - "tpp_threshold_2_unintended_diff_only": -0.0029999911785125732, - "tpp_threshold_5_total_metric": 0.13975003361701965, - "tpp_threshold_5_intended_diff_only": 0.14400005340576172, - "tpp_threshold_5_unintended_diff_only": 0.004250019788742065, - "tpp_threshold_10_total_metric": 0.2150000035762787, - "tpp_threshold_10_intended_diff_only": 0.2250000238418579, - "tpp_threshold_10_unintended_diff_only": 0.010000020265579224 + "tpp_threshold_2_total_metric": 0.021750018000602722, + "tpp_threshold_2_intended_diff_only": 0.025000035762786865, + "tpp_threshold_2_unintended_diff_only": 0.003250017762184143, + "tpp_threshold_5_total_metric": 0.13774999976158142, + "tpp_threshold_5_intended_diff_only": 0.14300000667572021, + "tpp_threshold_5_unintended_diff_only": 0.005250006914138794, + "tpp_threshold_10_total_metric": 0.21074999868869781, + "tpp_threshold_10_intended_diff_only": 0.22600001096725464, + "tpp_threshold_10_unintended_diff_only": 0.015250012278556824, + "tpp_threshold_20_total_metric": 0.27675004303455353, + "tpp_threshold_20_intended_diff_only": 0.3020000457763672, + "tpp_threshold_20_unintended_diff_only": 0.02525000274181366, + "tpp_threshold_50_total_metric": 0.2890000492334366, + "tpp_threshold_50_intended_diff_only": 0.35200005769729614, + "tpp_threshold_50_unintended_diff_only": 0.06300000846385956, + "tpp_threshold_100_total_metric": 0.27775004506111145, + "tpp_threshold_100_intended_diff_only": 0.3890000581741333, + "tpp_threshold_100_unintended_diff_only": 0.11125001311302185, + "tpp_threshold_500_total_metric": 0.13625003397464752, + "tpp_threshold_500_intended_diff_only": 0.4490000605583191, + "tpp_threshold_500_unintended_diff_only": 0.31275002658367157 }, "6": { - "tpp_threshold_2_total_metric": 0.011500000953674316, + "tpp_threshold_2_total_metric": 0.009499981999397278, "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": -0.002499997615814209, - "tpp_threshold_5_total_metric": 0.020250022411346436, - "tpp_threshold_5_intended_diff_only": 0.026000022888183594, - "tpp_threshold_5_unintended_diff_only": 0.005750000476837158, - "tpp_threshold_10_total_metric": 0.16025002300739288, + "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704, + "tpp_threshold_5_total_metric": 0.025499999523162842, + "tpp_threshold_5_intended_diff_only": 0.027000010013580322, + "tpp_threshold_5_unintended_diff_only": 0.0015000104904174805, + "tpp_threshold_10_total_metric": 0.1600000113248825, "tpp_threshold_10_intended_diff_only": 0.17400002479553223, - "tpp_threshold_10_unintended_diff_only": 0.013750001788139343 + "tpp_threshold_10_unintended_diff_only": 0.01400001347064972, + "tpp_threshold_20_total_metric": 0.20149999856948853, + "tpp_threshold_20_intended_diff_only": 0.2120000123977661, + "tpp_threshold_20_unintended_diff_only": 0.010500013828277588, + "tpp_threshold_50_total_metric": 0.3760000169277191, + "tpp_threshold_50_intended_diff_only": 0.45600003004074097, + "tpp_threshold_50_unintended_diff_only": 0.08000001311302185, + "tpp_threshold_100_total_metric": 0.3530000150203705, + "tpp_threshold_100_intended_diff_only": 0.4790000319480896, + "tpp_threshold_100_unintended_diff_only": 0.12600001692771912, + "tpp_threshold_500_total_metric": 0.2722500413656235, + "tpp_threshold_500_intended_diff_only": 0.49100005626678467, + "tpp_threshold_500_unintended_diff_only": 0.2187500149011612 }, "9": { - "tpp_threshold_2_total_metric": 0.00449998676776886, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": 0.0005000084638595581, - "tpp_threshold_5_total_metric": 0.010250002145767212, - "tpp_threshold_5_intended_diff_only": 0.012000024318695068, - "tpp_threshold_5_unintended_diff_only": 0.0017500221729278564, - "tpp_threshold_10_total_metric": 0.11425003409385681, - "tpp_threshold_10_intended_diff_only": 0.13100004196166992, - "tpp_threshold_10_unintended_diff_only": 0.01675000786781311 + "tpp_threshold_2_total_metric": 0.00575004518032074, + "tpp_threshold_2_intended_diff_only": 0.006000041961669922, + "tpp_threshold_2_unintended_diff_only": 0.00024999678134918213, + "tpp_threshold_5_total_metric": 0.013999998569488525, + "tpp_threshold_5_intended_diff_only": 0.017000019550323486, + "tpp_threshold_5_unintended_diff_only": 0.003000020980834961, + "tpp_threshold_10_total_metric": 0.11075000464916229, + "tpp_threshold_10_intended_diff_only": 0.1290000081062317, + "tpp_threshold_10_unintended_diff_only": 0.018250003457069397, + "tpp_threshold_20_total_metric": 0.20375001430511475, + "tpp_threshold_20_intended_diff_only": 0.2200000286102295, + "tpp_threshold_20_unintended_diff_only": 0.016250014305114746, + "tpp_threshold_50_total_metric": 0.3110000491142273, + "tpp_threshold_50_intended_diff_only": 0.3290000557899475, + "tpp_threshold_50_unintended_diff_only": 0.018000006675720215, + "tpp_threshold_100_total_metric": 0.3880000561475754, + "tpp_threshold_100_intended_diff_only": 0.4530000686645508, + "tpp_threshold_100_unintended_diff_only": 0.0650000125169754, + "tpp_threshold_500_total_metric": 0.29325003921985626, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.18275003135204315 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.013250023126602173, - "tpp_threshold_2_intended_diff_only": 0.008000016212463379, - "tpp_threshold_2_unintended_diff_only": -0.005250006914138794, - "tpp_threshold_5_total_metric": 0.012250006198883057, - "tpp_threshold_5_intended_diff_only": 0.009999990463256836, - "tpp_threshold_5_unintended_diff_only": -0.0022500157356262207, - "tpp_threshold_10_total_metric": 0.0040000081062316895, - "tpp_threshold_10_intended_diff_only": 0.008000016212463379, - "tpp_threshold_10_unintended_diff_only": 0.0040000081062316895 + "tpp_threshold_2_total_metric": 0.0065000057220458984, + "tpp_threshold_2_intended_diff_only": 0.009999990463256836, + "tpp_threshold_2_unintended_diff_only": 0.0034999847412109375, + "tpp_threshold_5_total_metric": 0.0070000141859054565, + "tpp_threshold_5_intended_diff_only": 0.008000016212463379, + "tpp_threshold_5_unintended_diff_only": 0.0010000020265579224, + "tpp_threshold_10_total_metric": 0.004749983549118042, + "tpp_threshold_10_intended_diff_only": 0.006999969482421875, + "tpp_threshold_10_unintended_diff_only": 0.002249985933303833, + "tpp_threshold_20_total_metric": 0.015250012278556824, + "tpp_threshold_20_intended_diff_only": 0.017000019550323486, + "tpp_threshold_20_unintended_diff_only": 0.0017500072717666626, + "tpp_threshold_50_total_metric": 0.0429999977350235, + "tpp_threshold_50_intended_diff_only": 0.042999982833862305, + "tpp_threshold_50_unintended_diff_only": -1.4901161193847656e-08, + "tpp_threshold_100_total_metric": 0.10950000584125519, + "tpp_threshold_100_intended_diff_only": 0.11599999666213989, + "tpp_threshold_100_unintended_diff_only": 0.006499990820884705, + "tpp_threshold_500_total_metric": 0.3760000169277191, + "tpp_threshold_500_intended_diff_only": 0.3840000033378601, + "tpp_threshold_500_unintended_diff_only": 0.007999986410140991 }, "2": { - "tpp_threshold_2_total_metric": -0.0007499456405639648, - "tpp_threshold_2_intended_diff_only": 0.005000054836273193, - "tpp_threshold_2_unintended_diff_only": 0.005750000476837158, - "tpp_threshold_5_total_metric": 0.006250053644180298, - "tpp_threshold_5_intended_diff_only": 0.01500004529953003, - "tpp_threshold_5_unintended_diff_only": 0.008749991655349731, - "tpp_threshold_10_total_metric": 0.027500063180923462, - "tpp_threshold_10_intended_diff_only": 0.033000051975250244, - "tpp_threshold_10_unintended_diff_only": 0.005499988794326782 + "tpp_threshold_2_total_metric": 0.010000020265579224, + "tpp_threshold_2_intended_diff_only": 0.008000016212463379, + "tpp_threshold_2_unintended_diff_only": -0.0020000040531158447, + "tpp_threshold_5_total_metric": 0.0030000656843185425, + "tpp_threshold_5_intended_diff_only": 0.010000050067901611, + "tpp_threshold_5_unintended_diff_only": 0.006999984383583069, + "tpp_threshold_10_total_metric": 0.028750017285346985, + "tpp_threshold_10_intended_diff_only": 0.03600001335144043, + "tpp_threshold_10_unintended_diff_only": 0.007249996066093445, + "tpp_threshold_20_total_metric": 0.06525000929832458, + "tpp_threshold_20_intended_diff_only": 0.07400000095367432, + "tpp_threshold_20_unintended_diff_only": 0.008749991655349731, + "tpp_threshold_50_total_metric": 0.12475007772445679, + "tpp_threshold_50_intended_diff_only": 0.1350000500679016, + "tpp_threshold_50_unintended_diff_only": 0.010249972343444824, + "tpp_threshold_100_total_metric": 0.27225005626678467, + "tpp_threshold_100_intended_diff_only": 0.29700005054473877, + "tpp_threshold_100_unintended_diff_only": 0.0247499942779541, + "tpp_threshold_500_total_metric": 0.4017500728368759, + "tpp_threshold_500_intended_diff_only": 0.43500006198883057, + "tpp_threshold_500_unintended_diff_only": 0.03324998915195465 }, "3": { - "tpp_threshold_2_total_metric": 0.012750014662742615, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": -0.007750019431114197, - "tpp_threshold_5_total_metric": 0.017999976873397827, - "tpp_threshold_5_intended_diff_only": 0.015999972820281982, - "tpp_threshold_5_unintended_diff_only": -0.0020000040531158447, - "tpp_threshold_10_total_metric": 0.01724998652935028, - "tpp_threshold_10_intended_diff_only": 0.018999993801116943, - "tpp_threshold_10_unintended_diff_only": 0.0017500072717666626 + "tpp_threshold_2_total_metric": 0.0015000104904174805, + "tpp_threshold_2_intended_diff_only": 0.0, + "tpp_threshold_2_unintended_diff_only": -0.0015000104904174805, + "tpp_threshold_5_total_metric": 0.012749984860420227, + "tpp_threshold_5_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_5_unintended_diff_only": -0.006750002503395081, + "tpp_threshold_10_total_metric": 0.01849997043609619, + "tpp_threshold_10_intended_diff_only": 0.015999972820281982, + "tpp_threshold_10_unintended_diff_only": -0.002499997615814209, + "tpp_threshold_20_total_metric": 0.015750005841255188, + "tpp_threshold_20_intended_diff_only": 0.018000006675720215, + "tpp_threshold_20_unintended_diff_only": 0.002250000834465027, + "tpp_threshold_50_total_metric": 0.05575002729892731, + "tpp_threshold_50_intended_diff_only": 0.054000020027160645, + "tpp_threshold_50_unintended_diff_only": -0.0017500072717666626, + "tpp_threshold_100_total_metric": 0.10200001299381256, + "tpp_threshold_100_intended_diff_only": 0.10600000619888306, + "tpp_threshold_100_unintended_diff_only": 0.003999993205070496, + "tpp_threshold_500_total_metric": 0.33400000631809235, + "tpp_threshold_500_intended_diff_only": 0.34299999475479126, + "tpp_threshold_500_unintended_diff_only": 0.008999988436698914 }, "5": { - "tpp_threshold_2_total_metric": -0.00625002384185791, - "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 0.007250010967254639, - "tpp_threshold_5_total_metric": 0.0007499754428863525, - "tpp_threshold_5_intended_diff_only": 0.001999974250793457, - "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045, - "tpp_threshold_10_total_metric": 0.004749968647956848, - "tpp_threshold_10_intended_diff_only": 0.006999969482421875, - "tpp_threshold_10_unintended_diff_only": 0.002250000834465027 + "tpp_threshold_2_total_metric": 0.0024999529123306274, + "tpp_threshold_2_intended_diff_only": 0.0029999613761901855, + "tpp_threshold_2_unintended_diff_only": 0.0005000084638595581, + "tpp_threshold_5_total_metric": 0.009499937295913696, + "tpp_threshold_5_intended_diff_only": 0.008999943733215332, + "tpp_threshold_5_unintended_diff_only": -0.0004999935626983643, + "tpp_threshold_10_total_metric": -0.0004999935626983643, + "tpp_threshold_10_intended_diff_only": 0.004999995231628418, + "tpp_threshold_10_unintended_diff_only": 0.005499988794326782, + "tpp_threshold_20_total_metric": 0.021249964833259583, + "tpp_threshold_20_intended_diff_only": 0.02199995517730713, + "tpp_threshold_20_unintended_diff_only": 0.0007499903440475464, + "tpp_threshold_50_total_metric": 0.054749950766563416, + "tpp_threshold_50_intended_diff_only": 0.06299996376037598, + "tpp_threshold_50_unintended_diff_only": 0.008250012993812561, + "tpp_threshold_100_total_metric": 0.11424997448921204, + "tpp_threshold_100_intended_diff_only": 0.12199997901916504, + "tpp_threshold_100_unintended_diff_only": 0.007750004529953003, + "tpp_threshold_500_total_metric": 0.3189999759197235, + "tpp_threshold_500_intended_diff_only": 0.38499999046325684, + "tpp_threshold_500_unintended_diff_only": 0.06600001454353333 }, "6": { - "tpp_threshold_2_total_metric": 0.016499951481819153, - "tpp_threshold_2_intended_diff_only": 0.01699995994567871, - "tpp_threshold_2_unintended_diff_only": 0.0005000084638595581, - "tpp_threshold_5_total_metric": 0.02950000762939453, - "tpp_threshold_5_intended_diff_only": 0.03700000047683716, - "tpp_threshold_5_unintended_diff_only": 0.007499992847442627, - "tpp_threshold_10_total_metric": 0.054249972105026245, - "tpp_threshold_10_intended_diff_only": 0.05699998140335083, - "tpp_threshold_10_unintended_diff_only": 0.002750009298324585 + "tpp_threshold_2_total_metric": 0.005249977111816406, + "tpp_threshold_2_intended_diff_only": 0.010999977588653564, + "tpp_threshold_2_unintended_diff_only": 0.005750000476837158, + "tpp_threshold_5_total_metric": 0.030749991536140442, + "tpp_threshold_5_intended_diff_only": 0.03200000524520874, + "tpp_threshold_5_unintended_diff_only": 0.0012500137090682983, + "tpp_threshold_10_total_metric": 0.06624996662139893, + "tpp_threshold_10_intended_diff_only": 0.07199996709823608, + "tpp_threshold_10_unintended_diff_only": 0.005750000476837158, + "tpp_threshold_20_total_metric": 0.11699999868869781, + "tpp_threshold_20_intended_diff_only": 0.1290000081062317, + "tpp_threshold_20_unintended_diff_only": 0.012000009417533875, + "tpp_threshold_50_total_metric": 0.18549998104572296, + "tpp_threshold_50_intended_diff_only": 0.19699996709823608, + "tpp_threshold_50_unintended_diff_only": 0.011499986052513123, + "tpp_threshold_100_total_metric": 0.2697500139474869, + "tpp_threshold_100_intended_diff_only": 0.28600001335144043, + "tpp_threshold_100_unintended_diff_only": 0.016249999403953552, + "tpp_threshold_500_total_metric": 0.34425002336502075, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.02875000238418579 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-6103-tokens-25000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-6103-tokens-25000000/scr/saebench_scr_custom_sae_eval_results.json index b2f4679c31352cbd359db4c8aab6d37dc7978a75..a9f831892c782e018153acd0a63c88d72d825e7b 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-6103-tokens-25000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-6103-tokens-25000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "2ff001da-cfea-436f-8611-67607a0453db", - "datetime_epoch_millis": 1776886666503, + "eval_id": "24e5bcc8-e66e-4b73-a08d-524b591774b5", + "datetime_epoch_millis": 1777317728636, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.19620984140046563, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.2351352741338071, "scr_metric_threshold_10": 0.28484493965498264, "scr_dir2_threshold_10": 0.2821089879477948, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.20285064387076104, + "scr_metric_threshold_20": 0.326805875878005, + "scr_dir2_threshold_20": 0.32857903549932893, + "scr_dir1_threshold_50": 0.06344311799392961, + "scr_metric_threshold_50": 0.39347829473738455, + "scr_dir2_threshold_50": 0.39913178990759024, + "scr_dir1_threshold_100": 0.09300308309625738, + "scr_metric_threshold_100": 0.4048802664204696, + "scr_dir2_threshold_100": 0.40823974399288304, + "scr_dir1_threshold_500": 0.20965666674342906, + "scr_metric_threshold_500": 0.3054615213045736, + "scr_dir2_threshold_500": 0.3119463499548245 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.5384614208981502, "scr_metric_threshold_10": 0.08808283994795557, "scr_dir2_threshold_10": 0.08808283994795557, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.5384614208981502, + "scr_metric_threshold_20": 0.11139890852479135, + "scr_dir2_threshold_20": 0.11139890852479135, + "scr_dir1_threshold_50": 0.48718003580863245, + "scr_metric_threshold_50": 0.18911918858629842, + "scr_dir2_threshold_50": 0.18911918858629842, + "scr_dir1_threshold_100": 0.48718003580863245, + "scr_metric_threshold_100": 0.2616580342884179, + "scr_dir2_threshold_100": 0.2616580342884179, + "scr_dir1_threshold_500": 0.39743608357792887, + "scr_metric_threshold_500": 0.3108808114137016, + "scr_dir2_threshold_500": 0.3108808114137016 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.12121234014557077, "scr_metric_threshold_10": 0.30275238555271, "scr_dir2_threshold_10": 0.30275238555271, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2045454853329709, + "scr_metric_threshold_20": 0.35474002715930114, + "scr_dir2_threshold_20": 0.35474002715930114, + "scr_dir1_threshold_50": -0.4469697140738727, + "scr_metric_threshold_50": 0.4311927307857058, + "scr_dir2_threshold_50": 0.4311927307857058, + "scr_dir1_threshold_100": -0.5833329194122803, + "scr_metric_threshold_100": 0.48623861906801674, + "scr_dir2_threshold_100": 0.48623861906801674, + "scr_dir1_threshold_500": 0.2878790820706107, + "scr_metric_threshold_500": 0.08562689682199721, + "scr_dir2_threshold_500": 0.08562689682199721 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.3437501746229541, "scr_metric_threshold_10": 0.1482412214577287, "scr_dir2_threshold_10": 0.1482412214577287, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.031249592546440393, + "scr_metric_threshold_20": 0.18341709182393628, + "scr_dir2_threshold_20": 0.18341709182393628, + "scr_dir1_threshold_50": -0.6406250291038257, + "scr_metric_threshold_50": 0.28140711269005153, + "scr_dir2_threshold_50": 0.28140711269005153, + "scr_dir1_threshold_100": 0.06250011641530274, + "scr_metric_threshold_100": 0.06783921065491268, + "scr_dir2_threshold_100": 0.06783921065491268, + "scr_dir1_threshold_500": 0.04687532014208255, + "scr_metric_threshold_500": 0.10301508102112025, + "scr_dir2_threshold_500": 0.10301508102112025 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": 0.22818779864829455, "scr_metric_threshold_10": 0.30573252397568285, "scr_dir2_threshold_10": 0.30573252397568285, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.07382535837795348, + "scr_metric_threshold_20": 0.39171981413981577, + "scr_dir2_threshold_20": 0.39171981413981577, + "scr_dir1_threshold_50": 0.14093939327267527, + "scr_metric_threshold_50": 0.5414013089483273, + "scr_dir2_threshold_50": 0.5414013089483273, + "scr_dir1_threshold_100": 0.08724800534441686, + "scr_metric_threshold_100": 0.5987261690577493, + "scr_dir2_threshold_100": 0.5987261690577493, + "scr_dir1_threshold_500": -0.07382575840915591, + "scr_metric_threshold_500": 0.13694280573859832, + "scr_dir2_threshold_500": 0.13694280573859832 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.09473687842942387, "scr_metric_threshold_10": 0.595238157185469, "scr_dir2_threshold_10": 0.595238157185469, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.16842108235134678, + "scr_metric_threshold_20": 0.6587301042915575, + "scr_dir2_threshold_20": 0.6587301042915575, + "scr_dir1_threshold_50": 0.09473687842942387, + "scr_metric_threshold_50": 0.6587301042915575, + "scr_dir2_threshold_50": 0.6587301042915575, + "scr_dir1_threshold_100": 0.0631580235225028, + "scr_metric_threshold_100": 0.7063492420158761, + "scr_dir2_threshold_100": 0.7063492420158761, + "scr_dir1_threshold_500": 0.07894729412163301, + "scr_metric_threshold_500": 0.7023808031440972, + "scr_dir2_threshold_500": 0.7023808031440972 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.2896173902280937, "scr_metric_threshold_10": 0.17557251734733875, "scr_dir2_threshold_10": 0.17557251734733875, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3060109503196829, + "scr_metric_threshold_20": 0.22519087703221677, + "scr_dir2_threshold_20": 0.22519087703221677, + "scr_dir1_threshold_50": 0.3606557163469838, + "scr_metric_threshold_50": 0.28625946991455836, + "scr_dir2_threshold_50": 0.28625946991455836, + "scr_dir1_threshold_100": 0.4644808369405549, + "scr_metric_threshold_100": 0.297709930610645, + "scr_dir2_threshold_100": 0.297709930610645, + "scr_dir1_threshold_500": 0.502732368584764, + "scr_metric_threshold_500": 0.41984734387395123, + "scr_dir2_threshold_500": 0.41984734387395123 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.12558142242720055, "scr_metric_threshold_10": 0.5236051041022068, "scr_dir2_threshold_10": 0.5236051041022068, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.21860457508548414, + "scr_metric_threshold_20": 0.5450643129054811, + "scr_dir2_threshold_20": 0.5450643129054811, + "scr_dir1_threshold_50": 0.3441859975126847, + "scr_metric_threshold_50": 0.5922747769238424, + "scr_dir2_threshold_50": 0.5922747769238424, + "scr_dir1_threshold_100": -0.013953583791087694, + "scr_metric_threshold_100": 0.6437767757261215, + "scr_dir2_threshold_100": 0.6437767757261215, + "scr_dir1_threshold_500": 0.18139520312982554, + "scr_metric_threshold_500": 0.42918468769337936, + "scr_dir2_threshold_500": 0.42918468769337936 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.13953476767076942, "scr_metric_threshold_10": 0.13953476767076942, "scr_dir2_threshold_10": 0.11764715401326675, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.14418587114694043, + "scr_metric_threshold_20": 0.14418587114694043, + "scr_dir2_threshold_20": 0.15837114811753172, + "scr_dir1_threshold_50": 0.16744166575873534, + "scr_metric_threshold_50": 0.16744166575873534, + "scr_dir2_threshold_50": 0.21266962712038115, + "scr_dir1_threshold_100": 0.17674414994201715, + "scr_metric_threshold_100": 0.17674414994201715, + "scr_dir2_threshold_100": 0.20361997052132483, + "scr_dir1_threshold_500": 0.2558137407297439, + "scr_metric_threshold_500": 0.2558137407297439, + "scr_dir2_threshold_500": 0.30769236993175136 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-6103-tokens-25000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-6103-tokens-25000000/tpp/saebench_tpp_custom_sae_eval_results.json index ff3df0a1c8a88cf4a3b38be32bffde3fbcbc0f5e..c3d71d1c45c8282cf53feece7347c45603899b99 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-6103-tokens-25000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-6103-tokens-25000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "16eef46d-fb0b-4386-a598-d932973537b3", - "datetime_epoch_millis": 1776886817024, + "eval_id": "63f7ee90-d531-42e1-b7b1-985ab429c5c9", + "datetime_epoch_millis": 1777318009874, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.012399998307228087, - "tpp_threshold_2_intended_diff_only": 0.0125, - "tpp_threshold_2_unintended_diff_only": 0.00010000169277191162, - "tpp_threshold_5_total_metric": 0.01602499634027481, - "tpp_threshold_5_intended_diff_only": 0.018399995565414426, - "tpp_threshold_5_unintended_diff_only": 0.002374999225139618, - "tpp_threshold_10_total_metric": 0.020849990844726562, - "tpp_threshold_10_intended_diff_only": 0.02369999885559082, + "tpp_threshold_2_total_metric": 0.011625003814697266, + "tpp_threshold_2_intended_diff_only": 0.0127000093460083, + "tpp_threshold_2_unintended_diff_only": 0.001075005531311035, + "tpp_threshold_5_total_metric": 0.01549999862909317, + "tpp_threshold_5_intended_diff_only": 0.016700005531311034, + "tpp_threshold_5_unintended_diff_only": 0.001200006902217865, + "tpp_threshold_10_total_metric": 0.02384999394416809, + "tpp_threshold_10_intended_diff_only": 0.02670000195503235, "tpp_threshold_10_unintended_diff_only": 0.0028500080108642576, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_20_total_metric": 0.044850008189678194, + "tpp_threshold_20_intended_diff_only": 0.048200011253356934, + "tpp_threshold_20_unintended_diff_only": 0.0033500030636787414, + "tpp_threshold_50_total_metric": 0.09422501623630525, + "tpp_threshold_50_intended_diff_only": 0.0992000162601471, + "tpp_threshold_50_unintended_diff_only": 0.0049750000238418576, + "tpp_threshold_100_total_metric": 0.18182501643896104, + "tpp_threshold_100_intended_diff_only": 0.1898000180721283, + "tpp_threshold_100_unintended_diff_only": 0.007975001633167268, + "tpp_threshold_500_total_metric": 0.3744000270962715, + "tpp_threshold_500_intended_diff_only": 0.3871000289916992, + "tpp_threshold_500_unintended_diff_only": 0.012700001895427703 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.01654999554157257, - "tpp_threshold_2_intended_diff_only": 0.01640000343322754, - "tpp_threshold_2_unintended_diff_only": -0.00014999210834503173, - "tpp_threshold_5_total_metric": 0.017199993133544922, - "tpp_threshold_5_intended_diff_only": 0.019200003147125243, - "tpp_threshold_5_unintended_diff_only": 0.002000010013580322, - "tpp_threshold_10_total_metric": 0.020099982619285583, - "tpp_threshold_10_intended_diff_only": 0.02239999771118164, - "tpp_threshold_10_unintended_diff_only": 0.002300015091896057, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.015200001001358033, + "tpp_threshold_2_intended_diff_only": 0.01660001277923584, + "tpp_threshold_2_unintended_diff_only": 0.0014000117778778076, + "tpp_threshold_5_total_metric": 0.01814999580383301, + "tpp_threshold_5_intended_diff_only": 0.018800008296966552, + "tpp_threshold_5_unintended_diff_only": 0.0006500124931335449, + "tpp_threshold_10_total_metric": 0.02024998962879181, + "tpp_threshold_10_intended_diff_only": 0.022600007057189942, + "tpp_threshold_10_unintended_diff_only": 0.0023500174283981323, + "tpp_threshold_20_total_metric": 0.044200006127357486, + "tpp_threshold_20_intended_diff_only": 0.04660001993179321, + "tpp_threshold_20_unintended_diff_only": 0.00240001380443573, + "tpp_threshold_50_total_metric": 0.09805001616477967, + "tpp_threshold_50_intended_diff_only": 0.10240002870559692, + "tpp_threshold_50_unintended_diff_only": 0.004350012540817261, + "tpp_threshold_100_total_metric": 0.18940001726150513, + "tpp_threshold_100_intended_diff_only": 0.19420002698898314, + "tpp_threshold_100_unintended_diff_only": 0.0048000097274780275, + "tpp_threshold_500_total_metric": 0.4010000228881836, + "tpp_threshold_500_intended_diff_only": 0.40760003328323363, + "tpp_threshold_500_unintended_diff_only": 0.006600010395050049 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.008250001072883605, - "tpp_threshold_2_intended_diff_only": 0.008599996566772461, - "tpp_threshold_2_unintended_diff_only": 0.00034999549388885497, - "tpp_threshold_5_total_metric": 0.0148499995470047, - "tpp_threshold_5_intended_diff_only": 0.017599987983703613, - "tpp_threshold_5_unintended_diff_only": 0.0027499884366989137, - "tpp_threshold_10_total_metric": 0.02159999907016754, - "tpp_threshold_10_intended_diff_only": 0.025, - "tpp_threshold_10_unintended_diff_only": 0.0034000009298324587, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.0080500066280365, + "tpp_threshold_2_intended_diff_only": 0.008800005912780762, + "tpp_threshold_2_unintended_diff_only": 0.0007499992847442627, + "tpp_threshold_5_total_metric": 0.012850001454353333, + "tpp_threshold_5_intended_diff_only": 0.014600002765655517, + "tpp_threshold_5_unintended_diff_only": 0.0017500013113021851, + "tpp_threshold_10_total_metric": 0.027449998259544372, + "tpp_threshold_10_intended_diff_only": 0.030799996852874757, + "tpp_threshold_10_unintended_diff_only": 0.0033499985933303834, + "tpp_threshold_20_total_metric": 0.0455000102519989, + "tpp_threshold_20_intended_diff_only": 0.049800002574920656, + "tpp_threshold_20_unintended_diff_only": 0.004299992322921753, + "tpp_threshold_50_total_metric": 0.09040001630783082, + "tpp_threshold_50_intended_diff_only": 0.09600000381469727, + "tpp_threshold_50_unintended_diff_only": 0.005599987506866455, + "tpp_threshold_100_total_metric": 0.17425001561641693, + "tpp_threshold_100_intended_diff_only": 0.18540000915527344, + "tpp_threshold_100_unintended_diff_only": 0.011149993538856507, + "tpp_threshold_500_total_metric": 0.34780003130435944, + "tpp_threshold_500_intended_diff_only": 0.3666000247001648, + "tpp_threshold_500_unintended_diff_only": 0.018799993395805358 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.0017500072717666626, "tpp_threshold_10_total_metric": 0.003999948501586914, "tpp_threshold_10_intended_diff_only": 0.006999969482421875, - "tpp_threshold_10_unintended_diff_only": 0.003000020980834961 + "tpp_threshold_10_unintended_diff_only": 0.003000020980834961, + "tpp_threshold_20_total_metric": 0.033749982714653015, + "tpp_threshold_20_intended_diff_only": 0.03600001335144043, + "tpp_threshold_20_unintended_diff_only": 0.0022500306367874146, + "tpp_threshold_50_total_metric": 0.07224996387958527, + "tpp_threshold_50_intended_diff_only": 0.07499998807907104, + "tpp_threshold_50_unintended_diff_only": 0.002750024199485779, + "tpp_threshold_100_total_metric": 0.1887499988079071, + "tpp_threshold_100_intended_diff_only": 0.19300001859664917, + "tpp_threshold_100_unintended_diff_only": 0.004250019788742065, + "tpp_threshold_500_total_metric": 0.416499987244606, + "tpp_threshold_500_intended_diff_only": 0.42100000381469727, + "tpp_threshold_500_unintended_diff_only": 0.0045000165700912476 }, "1": { - "tpp_threshold_2_total_metric": -0.0005000084638595581, - "tpp_threshold_2_intended_diff_only": 0.0, - "tpp_threshold_2_unintended_diff_only": 0.0005000084638595581, - "tpp_threshold_5_total_metric": 0.0032499879598617554, - "tpp_threshold_5_intended_diff_only": 0.004999995231628418, - "tpp_threshold_5_unintended_diff_only": 0.0017500072717666626, - "tpp_threshold_10_total_metric": -0.0022500306367874146, - "tpp_threshold_10_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_10_unintended_diff_only": 0.003250017762184143 + "tpp_threshold_2_total_metric": 0.0034999847412109375, + "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_2_unintended_diff_only": 0.000500023365020752, + "tpp_threshold_5_total_metric": -0.0005000084638595581, + "tpp_threshold_5_intended_diff_only": 0.0, + "tpp_threshold_5_unintended_diff_only": 0.0005000084638595581, + "tpp_threshold_10_total_metric": 0.00449998676776886, + "tpp_threshold_10_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_10_unintended_diff_only": -0.0004999786615371704, + "tpp_threshold_20_total_metric": 0.009499981999397278, + "tpp_threshold_20_intended_diff_only": 0.013999998569488525, + "tpp_threshold_20_unintended_diff_only": 0.0045000165700912476, + "tpp_threshold_50_total_metric": 0.04450002312660217, + "tpp_threshold_50_intended_diff_only": 0.053000032901763916, + "tpp_threshold_50_unintended_diff_only": 0.008500009775161743, + "tpp_threshold_100_total_metric": 0.12999998033046722, + "tpp_threshold_100_intended_diff_only": 0.13499999046325684, + "tpp_threshold_100_unintended_diff_only": 0.005000010132789612, + "tpp_threshold_500_total_metric": 0.41425004601478577, + "tpp_threshold_500_intended_diff_only": 0.4180000424385071, + "tpp_threshold_500_unintended_diff_only": 0.0037499964237213135 }, "2": { - "tpp_threshold_2_total_metric": 0.06250004470348358, - "tpp_threshold_2_intended_diff_only": 0.061000049114227295, - "tpp_threshold_2_unintended_diff_only": -0.0014999955892562866, - "tpp_threshold_5_total_metric": 0.06450001895427704, - "tpp_threshold_5_intended_diff_only": 0.06700003147125244, - "tpp_threshold_5_unintended_diff_only": 0.002500012516975403, - "tpp_threshold_10_total_metric": 0.07500001788139343, - "tpp_threshold_10_intended_diff_only": 0.07600003480911255, - "tpp_threshold_10_unintended_diff_only": 0.0010000169277191162 + "tpp_threshold_2_total_metric": 0.05150005221366882, + "tpp_threshold_2_intended_diff_only": 0.05600005388259888, + "tpp_threshold_2_unintended_diff_only": 0.004500001668930054, + "tpp_threshold_5_total_metric": 0.061250001192092896, + "tpp_threshold_5_intended_diff_only": 0.06400001049041748, + "tpp_threshold_5_unintended_diff_only": 0.002750009298324585, + "tpp_threshold_10_total_metric": 0.0727500170469284, + "tpp_threshold_10_intended_diff_only": 0.07700002193450928, + "tpp_threshold_10_unintended_diff_only": 0.004250004887580872, + "tpp_threshold_20_total_metric": 0.1120000034570694, + "tpp_threshold_20_intended_diff_only": 0.11400002241134644, + "tpp_threshold_20_unintended_diff_only": 0.0020000189542770386, + "tpp_threshold_50_total_metric": 0.20750002562999725, + "tpp_threshold_50_intended_diff_only": 0.21000003814697266, + "tpp_threshold_50_unintended_diff_only": 0.002500012516975403, + "tpp_threshold_100_total_metric": 0.3072500675916672, + "tpp_threshold_100_intended_diff_only": 0.31000006198883057, + "tpp_threshold_100_unintended_diff_only": 0.002749994397163391, + "tpp_threshold_500_total_metric": 0.42875005304813385, + "tpp_threshold_500_intended_diff_only": 0.440000057220459, + "tpp_threshold_500_unintended_diff_only": 0.011250004172325134 }, "6": { - "tpp_threshold_2_total_metric": 0.010500013828277588, + "tpp_threshold_2_total_metric": 0.00849999487400055, "tpp_threshold_2_intended_diff_only": 0.008000016212463379, - "tpp_threshold_2_unintended_diff_only": -0.002499997615814209, - "tpp_threshold_5_total_metric": 0.002249985933303833, - "tpp_threshold_5_intended_diff_only": 0.004999995231628418, - "tpp_threshold_5_unintended_diff_only": 0.002750009298324585, - "tpp_threshold_10_total_metric": 0.007749974727630615, - "tpp_threshold_10_intended_diff_only": 0.009999990463256836, - "tpp_threshold_10_unintended_diff_only": 0.0022500157356262207 + "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704, + "tpp_threshold_5_total_metric": 0.00875002145767212, + "tpp_threshold_5_intended_diff_only": 0.006000041961669922, + "tpp_threshold_5_unintended_diff_only": -0.0027499794960021973, + "tpp_threshold_10_total_metric": 0.010500028729438782, + "tpp_threshold_10_intended_diff_only": 0.01100003719329834, + "tpp_threshold_10_unintended_diff_only": 0.0005000084638595581, + "tpp_threshold_20_total_metric": 0.01500004529953003, + "tpp_threshold_20_intended_diff_only": 0.01500004529953003, + "tpp_threshold_20_unintended_diff_only": 0.0, + "tpp_threshold_50_total_metric": 0.013500049710273743, + "tpp_threshold_50_intended_diff_only": 0.01500004529953003, + "tpp_threshold_50_unintended_diff_only": 0.0014999955892562866, + "tpp_threshold_100_total_metric": 0.021250024437904358, + "tpp_threshold_100_intended_diff_only": 0.025000035762786865, + "tpp_threshold_100_unintended_diff_only": 0.0037500113248825073, + "tpp_threshold_500_total_metric": 0.2887500077486038, + "tpp_threshold_500_intended_diff_only": 0.2900000214576721, + "tpp_threshold_500_unintended_diff_only": 0.0012500137090682983 }, "9": { - "tpp_threshold_2_total_metric": 0.0034999847412109375, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": 0.0015000104904174805, - "tpp_threshold_5_total_metric": 0.006750002503395081, - "tpp_threshold_5_intended_diff_only": 0.008000016212463379, - "tpp_threshold_5_unintended_diff_only": 0.0012500137090682983, - "tpp_threshold_10_total_metric": 0.01600000262260437, - "tpp_threshold_10_intended_diff_only": 0.018000006675720215, - "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447 + "tpp_threshold_2_total_metric": 0.005750030279159546, + "tpp_threshold_2_intended_diff_only": 0.00700002908706665, + "tpp_threshold_2_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_5_total_metric": 0.01199999451637268, + "tpp_threshold_5_intended_diff_only": 0.013000011444091797, + "tpp_threshold_5_unintended_diff_only": 0.0010000169277191162, + "tpp_threshold_10_total_metric": 0.009499967098236084, + "tpp_threshold_10_intended_diff_only": 0.013999998569488525, + "tpp_threshold_10_unintended_diff_only": 0.004500031471252441, + "tpp_threshold_20_total_metric": 0.050750017166137695, + "tpp_threshold_20_intended_diff_only": 0.054000020027160645, + "tpp_threshold_20_unintended_diff_only": 0.0032500028610229492, + "tpp_threshold_50_total_metric": 0.15250001847743988, + "tpp_threshold_50_intended_diff_only": 0.15900003910064697, + "tpp_threshold_50_unintended_diff_only": 0.006500020623207092, + "tpp_threshold_100_total_metric": 0.2997500151395798, + "tpp_threshold_100_intended_diff_only": 0.30800002813339233, + "tpp_threshold_100_unintended_diff_only": 0.008250012993812561, + "tpp_threshold_500_total_metric": 0.4567500203847885, + "tpp_threshold_500_intended_diff_only": 0.46900004148483276, + "tpp_threshold_500_unintended_diff_only": 0.01225002110004425 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.008500009775161743, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": -0.004500001668930054, - "tpp_threshold_5_total_metric": 0.007249996066093445, - "tpp_threshold_5_intended_diff_only": 0.004999995231628418, - "tpp_threshold_5_unintended_diff_only": -0.002250000834465027, - "tpp_threshold_10_total_metric": 0.002249971032142639, + "tpp_threshold_2_total_metric": 0.0034999698400497437, + "tpp_threshold_2_intended_diff_only": 0.006999969482421875, + "tpp_threshold_2_unintended_diff_only": 0.0034999996423721313, + "tpp_threshold_5_total_metric": 0.000250011682510376, + "tpp_threshold_5_intended_diff_only": 0.003000020980834961, + "tpp_threshold_5_unintended_diff_only": 0.002750009298324585, + "tpp_threshold_10_total_metric": 0.00424996018409729, "tpp_threshold_10_intended_diff_only": 0.006999969482421875, - "tpp_threshold_10_unintended_diff_only": 0.004749998450279236 + "tpp_threshold_10_unintended_diff_only": 0.002750009298324585, + "tpp_threshold_20_total_metric": 0.015000030398368835, + "tpp_threshold_20_intended_diff_only": 0.017000019550323486, + "tpp_threshold_20_unintended_diff_only": 0.001999989151954651, + "tpp_threshold_50_total_metric": 0.038750022649765015, + "tpp_threshold_50_intended_diff_only": 0.03600001335144043, + "tpp_threshold_50_unintended_diff_only": -0.002750009298324585, + "tpp_threshold_100_total_metric": 0.11225003004074097, + "tpp_threshold_100_intended_diff_only": 0.11800003051757812, + "tpp_threshold_100_unintended_diff_only": 0.005750000476837158, + "tpp_threshold_500_total_metric": 0.3412500023841858, + "tpp_threshold_500_intended_diff_only": 0.34200000762939453, + "tpp_threshold_500_unintended_diff_only": 0.0007500052452087402 }, "2": { - "tpp_threshold_2_total_metric": 0.011250019073486328, - "tpp_threshold_2_intended_diff_only": 0.017000019550323486, - "tpp_threshold_2_unintended_diff_only": 0.005750000476837158, - "tpp_threshold_5_total_metric": 0.01775003969669342, - "tpp_threshold_5_intended_diff_only": 0.026000022888183594, - "tpp_threshold_5_unintended_diff_only": 0.008249983191490173, - "tpp_threshold_10_total_metric": 0.03500005602836609, - "tpp_threshold_10_intended_diff_only": 0.03800004720687866, - "tpp_threshold_10_unintended_diff_only": 0.0029999911785125732 + "tpp_threshold_2_total_metric": 0.0205000638961792, + "tpp_threshold_2_intended_diff_only": 0.01900005340576172, + "tpp_threshold_2_unintended_diff_only": -0.0015000104904174805, + "tpp_threshold_5_total_metric": 0.014500051736831665, + "tpp_threshold_5_intended_diff_only": 0.024000048637390137, + "tpp_threshold_5_unintended_diff_only": 0.009499996900558472, + "tpp_threshold_10_total_metric": 0.045750051736831665, + "tpp_threshold_10_intended_diff_only": 0.05200004577636719, + "tpp_threshold_10_unintended_diff_only": 0.0062499940395355225, + "tpp_threshold_20_total_metric": 0.05725005269050598, + "tpp_threshold_20_intended_diff_only": 0.061000049114227295, + "tpp_threshold_20_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_50_total_metric": 0.1077500581741333, + "tpp_threshold_50_intended_diff_only": 0.11800003051757812, + "tpp_threshold_50_unintended_diff_only": 0.010249972343444824, + "tpp_threshold_100_total_metric": 0.3240000903606415, + "tpp_threshold_100_intended_diff_only": 0.3420000672340393, + "tpp_threshold_100_unintended_diff_only": 0.017999976873397827, + "tpp_threshold_500_total_metric": 0.39475010335445404, + "tpp_threshold_500_intended_diff_only": 0.43400007486343384, + "tpp_threshold_500_unintended_diff_only": 0.0392499715089798 }, "3": { - "tpp_threshold_2_total_metric": 0.011000007390975952, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": -0.006999999284744263, - "tpp_threshold_5_total_metric": 0.015499994158744812, - "tpp_threshold_5_intended_diff_only": 0.015999972820281982, - "tpp_threshold_5_unintended_diff_only": 0.0004999786615371704, - "tpp_threshold_10_total_metric": 0.011749982833862305, - "tpp_threshold_10_intended_diff_only": 0.014999985694885254, - "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492 + "tpp_threshold_2_total_metric": 0.004500031471252441, + "tpp_threshold_2_intended_diff_only": 0.003000020980834961, + "tpp_threshold_2_unintended_diff_only": -0.0015000104904174805, + "tpp_threshold_5_total_metric": 0.006499990820884705, + "tpp_threshold_5_intended_diff_only": 0.004999995231628418, + "tpp_threshold_5_unintended_diff_only": -0.0014999955892562866, + "tpp_threshold_10_total_metric": 0.014749988913536072, + "tpp_threshold_10_intended_diff_only": 0.013999998569488525, + "tpp_threshold_10_unintended_diff_only": -0.0007499903440475464, + "tpp_threshold_20_total_metric": 0.007499992847442627, + "tpp_threshold_20_intended_diff_only": 0.010999977588653564, + "tpp_threshold_20_unintended_diff_only": 0.0034999847412109375, + "tpp_threshold_50_total_metric": 0.03800000250339508, + "tpp_threshold_50_intended_diff_only": 0.042999982833862305, + "tpp_threshold_50_unintended_diff_only": 0.004999980330467224, + "tpp_threshold_100_total_metric": 0.08099998533725739, + "tpp_threshold_100_intended_diff_only": 0.08899998664855957, + "tpp_threshold_100_unintended_diff_only": 0.008000001311302185, + "tpp_threshold_500_total_metric": 0.3212500512599945, + "tpp_threshold_500_intended_diff_only": 0.33000004291534424, + "tpp_threshold_500_unintended_diff_only": 0.008749991655349731 }, "5": { - "tpp_threshold_2_total_metric": -0.004500031471252441, - "tpp_threshold_2_intended_diff_only": 0.0029999613761901855, - "tpp_threshold_2_unintended_diff_only": 0.007499992847442627, - "tpp_threshold_5_total_metric": 0.0007499754428863525, - "tpp_threshold_5_intended_diff_only": 0.0029999613761901855, - "tpp_threshold_5_unintended_diff_only": 0.002249985933303833, - "tpp_threshold_10_total_metric": 0.0009999722242355347, - "tpp_threshold_10_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_10_unintended_diff_only": 0.005000010132789612 + "tpp_threshold_2_total_metric": 0.006749972701072693, + "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_2_unintended_diff_only": -0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.0059999823570251465, + "tpp_threshold_5_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_5_unintended_diff_only": 0.0, + "tpp_threshold_10_total_metric": 0.0009999871253967285, + "tpp_threshold_10_intended_diff_only": 0.006999969482421875, + "tpp_threshold_10_unintended_diff_only": 0.0059999823570251465, + "tpp_threshold_20_total_metric": 0.031499966979026794, + "tpp_threshold_20_intended_diff_only": 0.034999966621398926, + "tpp_threshold_20_unintended_diff_only": 0.0034999996423721313, + "tpp_threshold_50_total_metric": 0.054499998688697815, + "tpp_threshold_50_intended_diff_only": 0.06099998950958252, + "tpp_threshold_50_unintended_diff_only": 0.006499990820884705, + "tpp_threshold_100_total_metric": 0.09499998390674591, + "tpp_threshold_100_intended_diff_only": 0.10399997234344482, + "tpp_threshold_100_unintended_diff_only": 0.008999988436698914, + "tpp_threshold_500_total_metric": 0.3355000168085098, + "tpp_threshold_500_intended_diff_only": 0.35500001907348633, + "tpp_threshold_500_unintended_diff_only": 0.0195000022649765 }, "6": { - "tpp_threshold_2_total_metric": 0.015000000596046448, - "tpp_threshold_2_intended_diff_only": 0.014999985694885254, - "tpp_threshold_2_unintended_diff_only": -1.4901161193847656e-08, - "tpp_threshold_5_total_metric": 0.03299999237060547, - "tpp_threshold_5_intended_diff_only": 0.03799998760223389, - "tpp_threshold_5_unintended_diff_only": 0.004999995231628418, - "tpp_threshold_10_total_metric": 0.05800001323223114, - "tpp_threshold_10_intended_diff_only": 0.05900001525878906, - "tpp_threshold_10_unintended_diff_only": 0.0010000020265579224 + "tpp_threshold_2_total_metric": 0.004999995231628418, + "tpp_threshold_2_intended_diff_only": 0.009000003337860107, + "tpp_threshold_2_unintended_diff_only": 0.0040000081062316895, + "tpp_threshold_5_total_metric": 0.03699997067451477, + "tpp_threshold_5_intended_diff_only": 0.034999966621398926, + "tpp_threshold_5_unintended_diff_only": -0.0020000040531158447, + "tpp_threshold_10_total_metric": 0.07150000333786011, + "tpp_threshold_10_intended_diff_only": 0.07400000095367432, + "tpp_threshold_10_unintended_diff_only": 0.002499997615814209, + "tpp_threshold_20_total_metric": 0.11625000834465027, + "tpp_threshold_20_intended_diff_only": 0.125, + "tpp_threshold_20_unintended_diff_only": 0.008749991655349731, + "tpp_threshold_50_total_metric": 0.21299999952316284, + "tpp_threshold_50_intended_diff_only": 0.22200000286102295, + "tpp_threshold_50_unintended_diff_only": 0.009000003337860107, + "tpp_threshold_100_total_metric": 0.2589999884366989, + "tpp_threshold_100_intended_diff_only": 0.27399998903274536, + "tpp_threshold_100_unintended_diff_only": 0.015000000596046448, + "tpp_threshold_500_total_metric": 0.346249982714653, + "tpp_threshold_500_intended_diff_only": 0.37199997901916504, + "tpp_threshold_500_unintended_diff_only": 0.025749996304512024 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-61035-tokens-250000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-61035-tokens-250000000/scr/saebench_scr_custom_sae_eval_results.json index 8d2484ec2525f0a82d245596c5919e9ec30d81af..6712922929b36db815f8f27f17964708a163185a 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-61035-tokens-250000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-61035-tokens-250000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "a5d7cd24-340e-4956-ac98-281f6d1ac108", - "datetime_epoch_millis": 1776886979309, + "eval_id": "b1e7419f-cf41-404c-b2a6-c90adc108176", + "datetime_epoch_millis": 1777317908900, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.2220859849509409, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.06718847095878272, "scr_metric_threshold_10": 0.252787738013544, "scr_dir2_threshold_10": 0.25925938989698294, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.01326544210468493, + "scr_metric_threshold_20": 0.3194196761427372, + "scr_dir2_threshold_20": 0.3199511023149136, + "scr_dir1_threshold_50": 0.08637048811969517, + "scr_metric_threshold_50": 0.37699152040636147, + "scr_dir2_threshold_50": 0.37554197934239436, + "scr_dir1_threshold_100": 0.02540944219925659, + "scr_metric_threshold_100": 0.3988576318437054, + "scr_dir2_threshold_100": 0.4111248056988769, + "scr_dir1_threshold_500": -0.534597704925256, + "scr_metric_threshold_500": 0.20945854174321957, + "scr_dir2_threshold_500": 0.21481212960555648 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.3333335880540079, "scr_metric_threshold_10": 0.10362698860995496, "scr_dir2_threshold_10": 0.10362698860995496, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.29487178507484585, + "scr_metric_threshold_20": 0.1968911085011348, + "scr_dir2_threshold_20": 0.1968911085011348, + "scr_dir1_threshold_50": 0.5512825313325533, + "scr_metric_threshold_50": 0.28497410286525365, + "scr_dir2_threshold_50": 0.28497410286525365, + "scr_dir1_threshold_100": 0.5384614208981502, + "scr_metric_threshold_100": 0.31347145138531374, + "scr_dir2_threshold_100": 0.31347145138531374, + "scr_dir1_threshold_500": -1.4871792716466088, + "scr_metric_threshold_500": 0.08549219997634344, + "scr_dir2_threshold_500": 0.08549219997634344 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.04545485332970889, "scr_metric_threshold_10": 0.3516819627607703, "scr_dir2_threshold_10": 0.3516819627607703, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.3181817155567637, + "scr_metric_threshold_20": 0.38532103569898746, + "scr_dir2_threshold_20": 0.38532103569898746, + "scr_dir1_threshold_50": -0.13636365688864727, + "scr_metric_threshold_50": 0.5412845073503276, + "scr_dir2_threshold_50": 0.5412845073503276, + "scr_dir1_threshold_100": -0.613636004448673, + "scr_metric_threshold_100": 0.5229357564047648, + "scr_dir2_threshold_100": 0.5229357564047648, + "scr_dir1_threshold_500": -1.8409084648962586, + "scr_metric_threshold_500": 0.07645270362640467, + "scr_dir2_threshold_500": 0.07645270362640467 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": -0.3749997671693945, "scr_metric_threshold_10": 0.1482412214577287, "scr_dir2_threshold_10": 0.1482412214577287, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.18750034924590825, + "scr_metric_threshold_20": 0.17587935183103792, + "scr_dir2_threshold_20": 0.17587935183103792, + "scr_dir1_threshold_50": 0.0, + "scr_metric_threshold_50": 0.21356790203513887, + "scr_dir2_threshold_50": 0.21356790203513887, + "scr_dir1_threshold_100": 0.32812537834973393, + "scr_metric_threshold_100": 0.04020108028160345, + "scr_dir2_threshold_100": 0.04020108028160345, + "scr_dir1_threshold_500": -0.3906245634426147, + "scr_metric_threshold_500": -0.13316574147193197, + "scr_dir2_threshold_500": -0.13316574147193197 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": -0.1208054228229802, "scr_metric_threshold_10": 0.24203831933130424, "scr_dir2_threshold_10": 0.24203831933130424, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.758389554385242, + "scr_metric_threshold_20": 0.3248407474042558, + "scr_dir2_threshold_20": 0.3248407474042558, + "scr_dir1_threshold_50": -0.5234900322225133, + "scr_metric_threshold_50": 0.3089171962431611, + "scr_dir2_threshold_50": 0.3089171962431611, + "scr_dir1_threshold_100": -0.8523492832440931, + "scr_metric_threshold_100": 0.5414013089483273, + "scr_dir2_threshold_100": 0.5414013089483273, + "scr_dir1_threshold_500": -1.2617452161268579, + "scr_metric_threshold_500": -0.40764336530091044, + "scr_dir2_threshold_500": -0.40764336530091044 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.08421069803000372, "scr_metric_threshold_10": 0.5476190194611504, "scr_dir2_threshold_10": 0.5476190194611504, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.11578955293692479, + "scr_metric_threshold_20": 0.6150794054390177, + "scr_dir2_threshold_20": 0.6150794054390177, + "scr_dir1_threshold_50": -0.15263149804355594, + "scr_metric_threshold_50": 0.6746031501996637, + "scr_dir2_threshold_50": 0.6746031501996637, + "scr_dir1_threshold_100": -0.2526314666726899, + "scr_metric_threshold_100": 0.7023808031440972, + "scr_dir2_threshold_100": 0.7023808031440972, + "scr_dir1_threshold_500": -0.2578945568723999, + "scr_metric_threshold_500": 0.6706349478542213, + "scr_dir2_threshold_500": 0.6706349478542213 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.3005465388586523, "scr_metric_threshold_10": 0.14122136275770192, "scr_dir2_threshold_10": 0.14122136275770192, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3333333333333333, + "scr_metric_threshold_20": 0.22137413263306224, + "scr_dir2_threshold_20": 0.22137413263306224, + "scr_dir1_threshold_50": 0.3661201278080144, + "scr_metric_threshold_50": 0.2557252872226991, + "scr_dir2_threshold_50": 0.2557252872226991, + "scr_dir1_threshold_100": 0.371584539269045, + "scr_metric_threshold_100": 0.36259549539076413, + "scr_dir2_threshold_100": 0.36259549539076413, + "scr_dir1_threshold_500": 0.4918032199542054, + "scr_metric_threshold_500": 0.5916031168221354, + "scr_dir2_threshold_500": 0.5916031168221354 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.17674410094308396, "scr_metric_threshold_10": 0.3948498512825616, "scr_dir2_threshold_10": 0.3948498512825616, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.23720926106331344, + "scr_metric_threshold_20": 0.45922747769238426, + "scr_dir2_threshold_20": 0.45922747769238426, + "scr_dir1_threshold_50": 0.3255813115348554, + "scr_metric_threshold_50": 0.4763948958977931, + "scr_dir2_threshold_50": 0.4763948958977931, + "scr_dir1_threshold_100": 0.46511631775314366, + "scr_metric_threshold_100": 0.4892705235053367, + "scr_dir2_threshold_100": 0.4892705235053367, + "scr_dir1_threshold_500": 0.21395347289874253, + "scr_metric_threshold_500": 0.5364807317097504, + "scr_dir2_threshold_500": 0.5364807317097504 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.09302317844717961, "scr_metric_threshold_10": 0.09302317844717961, "scr_dir2_threshold_10": 0.14479639351469145, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.17674414994201715, + "scr_metric_threshold_20": 0.17674414994201715, + "scr_dir2_threshold_20": 0.1809955593194283, + "scr_dir1_threshold_50": 0.2604651214368547, + "scr_metric_threshold_50": 0.2604651214368547, + "scr_dir2_threshold_50": 0.24886879292511796, + "scr_dir1_threshold_100": 0.21860463568943592, + "scr_metric_threshold_100": 0.21860463568943592, + "scr_dir2_threshold_100": 0.31674202653080763, + "scr_dir1_threshold_500": 0.2558137407297439, + "scr_metric_threshold_500": 0.2558137407297439, + "scr_dir2_threshold_500": 0.2986424436284392 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-61035-tokens-250000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-61035-tokens-250000000/tpp/saebench_tpp_custom_sae_eval_results.json index da7249680d92a096258f1340b1201adcd9184e14..1cc0fbc8fa56fbcbd211fc004f3d2e703fd179d0 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-61035-tokens-250000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-61035-tokens-250000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "833ebe19-6fed-4efb-a3d1-22ec19370e1d", - "datetime_epoch_millis": 1776887139171, + "eval_id": "2137d14e-0345-468e-be9a-d66f9982f700", + "datetime_epoch_millis": 1777318225949, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.01147499680519104, - "tpp_threshold_2_intended_diff_only": 0.01119999885559082, - "tpp_threshold_2_unintended_diff_only": -0.00027499794960021977, - "tpp_threshold_5_total_metric": 0.035474997758865354, - "tpp_threshold_5_intended_diff_only": 0.03950000405311584, - "tpp_threshold_5_unintended_diff_only": 0.004025006294250488, - "tpp_threshold_10_total_metric": 0.08065000027418137, - "tpp_threshold_10_intended_diff_only": 0.09100000858306885, - "tpp_threshold_10_unintended_diff_only": 0.01035000830888748, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.010399992763996124, + "tpp_threshold_2_intended_diff_only": 0.011299997568130493, + "tpp_threshold_2_unintended_diff_only": 0.0009000048041343688, + "tpp_threshold_5_total_metric": 0.035599997639656066, + "tpp_threshold_5_intended_diff_only": 0.03820000290870666, + "tpp_threshold_5_unintended_diff_only": 0.0026000052690505985, + "tpp_threshold_10_total_metric": 0.08202499598264694, + "tpp_threshold_10_intended_diff_only": 0.09260000586509703, + "tpp_threshold_10_unintended_diff_only": 0.010575009882450104, + "tpp_threshold_20_total_metric": 0.1260750025510788, + "tpp_threshold_20_intended_diff_only": 0.14160000681877136, + "tpp_threshold_20_unintended_diff_only": 0.015525004267692565, + "tpp_threshold_50_total_metric": 0.2001499980688095, + "tpp_threshold_50_intended_diff_only": 0.2352000057697296, + "tpp_threshold_50_unintended_diff_only": 0.03505000770092011, + "tpp_threshold_100_total_metric": 0.2484250172972679, + "tpp_threshold_100_intended_diff_only": 0.31520002484321596, + "tpp_threshold_100_unintended_diff_only": 0.06677500754594803, + "tpp_threshold_500_total_metric": 0.28007502406835555, + "tpp_threshold_500_intended_diff_only": 0.4279000341892243, + "tpp_threshold_500_unintended_diff_only": 0.14782501012086868 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.01654999554157257, - "tpp_threshold_2_intended_diff_only": 0.016200006008148193, - "tpp_threshold_2_unintended_diff_only": -0.00034998953342437746, - "tpp_threshold_5_total_metric": 0.059250009059906, - "tpp_threshold_5_intended_diff_only": 0.06480002403259277, - "tpp_threshold_5_unintended_diff_only": 0.005550014972686768, - "tpp_threshold_10_total_metric": 0.1446000099182129, - "tpp_threshold_10_intended_diff_only": 0.16200002431869506, - "tpp_threshold_10_unintended_diff_only": 0.017400014400482177, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.015249985456466674, + "tpp_threshold_2_intended_diff_only": 0.01640000343322754, + "tpp_threshold_2_unintended_diff_only": 0.0011500179767608642, + "tpp_threshold_5_total_metric": 0.06084998846054077, + "tpp_threshold_5_intended_diff_only": 0.06540000438690186, + "tpp_threshold_5_unintended_diff_only": 0.004550015926361084, + "tpp_threshold_10_total_metric": 0.14369999766349792, + "tpp_threshold_10_intended_diff_only": 0.16180001497268676, + "tpp_threshold_10_unintended_diff_only": 0.018100017309188844, + "tpp_threshold_20_total_metric": 0.2104000121355057, + "tpp_threshold_20_intended_diff_only": 0.23720002174377441, + "tpp_threshold_20_unintended_diff_only": 0.026800009608268737, + "tpp_threshold_50_total_metric": 0.2968500077724457, + "tpp_threshold_50_intended_diff_only": 0.3632000207901001, + "tpp_threshold_50_unintended_diff_only": 0.06635001301765442, + "tpp_threshold_100_total_metric": 0.3070500075817108, + "tpp_threshold_100_intended_diff_only": 0.4228000283241272, + "tpp_threshold_100_unintended_diff_only": 0.11575002074241639, + "tpp_threshold_500_total_metric": 0.19850002825260163, + "tpp_threshold_500_intended_diff_only": 0.46460005044937136, + "tpp_threshold_500_unintended_diff_only": 0.2661000221967697 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.006399998068809509, + "tpp_threshold_2_total_metric": 0.005550000071525574, "tpp_threshold_2_intended_diff_only": 0.006199991703033448, - "tpp_threshold_2_unintended_diff_only": -0.00020000636577606202, - "tpp_threshold_5_total_metric": 0.011699986457824708, - "tpp_threshold_5_intended_diff_only": 0.014199984073638917, - "tpp_threshold_5_unintended_diff_only": 0.002499997615814209, - "tpp_threshold_10_total_metric": 0.01669999063014984, - "tpp_threshold_10_intended_diff_only": 0.019999992847442628, - "tpp_threshold_10_unintended_diff_only": 0.0033000022172927856, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_unintended_diff_only": 0.0006499916315078735, + "tpp_threshold_5_total_metric": 0.010350006818771362, + "tpp_threshold_5_intended_diff_only": 0.011000001430511474, + "tpp_threshold_5_unintended_diff_only": 0.0006499946117401123, + "tpp_threshold_10_total_metric": 0.02034999430179596, + "tpp_threshold_10_intended_diff_only": 0.023399996757507324, + "tpp_threshold_10_unintended_diff_only": 0.003050002455711365, + "tpp_threshold_20_total_metric": 0.04174999296665192, + "tpp_threshold_20_intended_diff_only": 0.04599999189376831, + "tpp_threshold_20_unintended_diff_only": 0.004249998927116394, + "tpp_threshold_50_total_metric": 0.10344998836517334, + "tpp_threshold_50_intended_diff_only": 0.10719999074935913, + "tpp_threshold_50_unintended_diff_only": 0.003750002384185791, + "tpp_threshold_100_total_metric": 0.189800027012825, + "tpp_threshold_100_intended_diff_only": 0.20760002136230468, + "tpp_threshold_100_unintended_diff_only": 0.017799994349479674, + "tpp_threshold_500_total_metric": 0.3616500198841095, + "tpp_threshold_500_intended_diff_only": 0.39120001792907716, + "tpp_threshold_500_unintended_diff_only": 0.029549998044967652 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.0017500221729278564, "tpp_threshold_10_total_metric": 0.006999969482421875, "tpp_threshold_10_intended_diff_only": 0.009999990463256836, - "tpp_threshold_10_unintended_diff_only": 0.003000020980834961 + "tpp_threshold_10_unintended_diff_only": 0.003000020980834961, + "tpp_threshold_20_total_metric": 0.14774996042251587, + "tpp_threshold_20_intended_diff_only": 0.17399996519088745, + "tpp_threshold_20_unintended_diff_only": 0.026250004768371582, + "tpp_threshold_50_total_metric": 0.27424998581409454, + "tpp_threshold_50_intended_diff_only": 0.31800001859664917, + "tpp_threshold_50_unintended_diff_only": 0.043750032782554626, + "tpp_threshold_100_total_metric": 0.34174998104572296, + "tpp_threshold_100_intended_diff_only": 0.39800000190734863, + "tpp_threshold_100_unintended_diff_only": 0.05625002086162567, + "tpp_threshold_500_total_metric": 0.22049999237060547, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.2355000376701355 }, "1": { - "tpp_threshold_2_total_metric": -0.0007500201463699341, - "tpp_threshold_2_intended_diff_only": 0.0, - "tpp_threshold_2_unintended_diff_only": 0.0007500201463699341, - "tpp_threshold_5_total_metric": 0.07450002431869507, - "tpp_threshold_5_intended_diff_only": 0.08500003814697266, - "tpp_threshold_5_unintended_diff_only": 0.010500013828277588, - "tpp_threshold_10_total_metric": 0.14749999344348907, + "tpp_threshold_2_total_metric": 0.0037499815225601196, + "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_2_unintended_diff_only": 0.0002500265836715698, + "tpp_threshold_5_total_metric": 0.07524998486042023, + "tpp_threshold_5_intended_diff_only": 0.08300000429153442, + "tpp_threshold_5_unintended_diff_only": 0.007750019431114197, + "tpp_threshold_10_total_metric": 0.14999999105930328, "tpp_threshold_10_intended_diff_only": 0.17100000381469727, - "tpp_threshold_10_unintended_diff_only": 0.02350001037120819 + "tpp_threshold_10_unintended_diff_only": 0.021000012755393982, + "tpp_threshold_20_total_metric": 0.18675000965595245, + "tpp_threshold_20_intended_diff_only": 0.21100002527236938, + "tpp_threshold_20_unintended_diff_only": 0.02425001561641693, + "tpp_threshold_50_total_metric": 0.26650001108646393, + "tpp_threshold_50_intended_diff_only": 0.3270000219345093, + "tpp_threshold_50_unintended_diff_only": 0.06050001084804535, + "tpp_threshold_100_total_metric": 0.26374998688697815, + "tpp_threshold_100_intended_diff_only": 0.40700000524520874, + "tpp_threshold_100_unintended_diff_only": 0.1432500183582306, + "tpp_threshold_500_total_metric": 0.16225002706050873, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.2967500239610672 }, "2": { - "tpp_threshold_2_total_metric": 0.0572500079870224, - "tpp_threshold_2_intended_diff_only": 0.05500000715255737, - "tpp_threshold_2_unintended_diff_only": -0.002250000834465027, - "tpp_threshold_5_total_metric": 0.1715000420808792, + "tpp_threshold_2_total_metric": 0.04649999737739563, + "tpp_threshold_2_intended_diff_only": 0.050000011920928955, + "tpp_threshold_2_unintended_diff_only": 0.003500014543533325, + "tpp_threshold_5_total_metric": 0.17000003159046173, "tpp_threshold_5_intended_diff_only": 0.1770000457763672, - "tpp_threshold_5_unintended_diff_only": 0.005500003695487976, - "tpp_threshold_10_total_metric": 0.26225003600120544, - "tpp_threshold_10_intended_diff_only": 0.28300005197525024, - "tpp_threshold_10_unintended_diff_only": 0.0207500159740448 + "tpp_threshold_5_unintended_diff_only": 0.0070000141859054565, + "tpp_threshold_10_total_metric": 0.25950002670288086, + "tpp_threshold_10_intended_diff_only": 0.284000039100647, + "tpp_threshold_10_unintended_diff_only": 0.024500012397766113, + "tpp_threshold_20_total_metric": 0.2872500419616699, + "tpp_threshold_20_intended_diff_only": 0.3290000557899475, + "tpp_threshold_20_unintended_diff_only": 0.04175001382827759, + "tpp_threshold_50_total_metric": 0.27800001204013824, + "tpp_threshold_50_intended_diff_only": 0.37300002574920654, + "tpp_threshold_50_unintended_diff_only": 0.0950000137090683, + "tpp_threshold_100_total_metric": 0.24050000309944153, + "tpp_threshold_100_intended_diff_only": 0.4010000228881836, + "tpp_threshold_100_unintended_diff_only": 0.16050001978874207, + "tpp_threshold_500_total_metric": 0.109250009059906, + "tpp_threshold_500_intended_diff_only": 0.44600003957748413, + "tpp_threshold_500_unintended_diff_only": 0.3367500305175781 }, "6": { - "tpp_threshold_2_total_metric": 0.011750012636184692, + "tpp_threshold_2_total_metric": 0.009249985218048096, "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": -0.002750009298324585, - "tpp_threshold_5_total_metric": 0.02775000035762787, - "tpp_threshold_5_intended_diff_only": 0.03600001335144043, - "tpp_threshold_5_unintended_diff_only": 0.008250012993812561, - "tpp_threshold_10_total_metric": 0.19150002300739288, + "tpp_threshold_2_unintended_diff_only": -0.0002499818801879883, + "tpp_threshold_5_total_metric": 0.03350000083446503, + "tpp_threshold_5_intended_diff_only": 0.03700000047683716, + "tpp_threshold_5_unintended_diff_only": 0.0034999996423721313, + "tpp_threshold_10_total_metric": 0.19200001657009125, "tpp_threshold_10_intended_diff_only": 0.21000003814697266, - "tpp_threshold_10_unintended_diff_only": 0.018500015139579773 + "tpp_threshold_10_unintended_diff_only": 0.01800002157688141, + "tpp_threshold_20_total_metric": 0.2237500250339508, + "tpp_threshold_20_intended_diff_only": 0.23900002241134644, + "tpp_threshold_20_unintended_diff_only": 0.01524999737739563, + "tpp_threshold_50_total_metric": 0.3642500042915344, + "tpp_threshold_50_intended_diff_only": 0.4620000123977661, + "tpp_threshold_50_unintended_diff_only": 0.09775000810623169, + "tpp_threshold_100_total_metric": 0.33650003373622894, + "tpp_threshold_100_intended_diff_only": 0.47700005769729614, + "tpp_threshold_100_unintended_diff_only": 0.1405000239610672, + "tpp_threshold_500_total_metric": 0.24800005555152893, + "tpp_threshold_500_intended_diff_only": 0.48600006103515625, + "tpp_threshold_500_unintended_diff_only": 0.23800000548362732 }, "9": { - "tpp_threshold_2_total_metric": 0.0047500282526016235, - "tpp_threshold_2_intended_diff_only": 0.006000041961669922, - "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983, - "tpp_threshold_5_total_metric": 0.008250027894973755, - "tpp_threshold_5_intended_diff_only": 0.010000050067901611, - "tpp_threshold_5_unintended_diff_only": 0.0017500221729278564, - "tpp_threshold_10_total_metric": 0.11475002765655518, - "tpp_threshold_10_intended_diff_only": 0.13600003719329834, - "tpp_threshold_10_unintended_diff_only": 0.021250009536743164 + "tpp_threshold_2_total_metric": 0.0070000141859054565, + "tpp_threshold_2_intended_diff_only": 0.008000016212463379, + "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, + "tpp_threshold_5_total_metric": 0.011249974370002747, + "tpp_threshold_5_intended_diff_only": 0.013999998569488525, + "tpp_threshold_5_unintended_diff_only": 0.002750024199485779, + "tpp_threshold_10_total_metric": 0.10999998450279236, + "tpp_threshold_10_intended_diff_only": 0.1340000033378601, + "tpp_threshold_10_unintended_diff_only": 0.02400001883506775, + "tpp_threshold_20_total_metric": 0.20650002360343933, + "tpp_threshold_20_intended_diff_only": 0.2330000400543213, + "tpp_threshold_20_unintended_diff_only": 0.026500016450881958, + "tpp_threshold_50_total_metric": 0.30125002562999725, + "tpp_threshold_50_intended_diff_only": 0.3360000252723694, + "tpp_threshold_50_unintended_diff_only": 0.03474999964237213, + "tpp_threshold_100_total_metric": 0.3527500331401825, + "tpp_threshold_100_intended_diff_only": 0.4310000538825989, + "tpp_threshold_100_unintended_diff_only": 0.07825002074241638, + "tpp_threshold_500_total_metric": 0.252500057220459, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.22350001335144043 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.010999992489814758, - "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_2_unintended_diff_only": -0.005000010132789612, - "tpp_threshold_5_total_metric": 0.012499988079071045, - "tpp_threshold_5_intended_diff_only": 0.009999990463256836, - "tpp_threshold_5_unintended_diff_only": -0.002499997615814209, - "tpp_threshold_10_total_metric": 0.002249971032142639, - "tpp_threshold_10_intended_diff_only": 0.006999969482421875, - "tpp_threshold_10_unintended_diff_only": 0.004749998450279236 + "tpp_threshold_2_total_metric": 0.006750002503395081, + "tpp_threshold_2_intended_diff_only": 0.009999990463256836, + "tpp_threshold_2_unintended_diff_only": 0.0032499879598617554, + "tpp_threshold_5_total_metric": 0.0057499706745147705, + "tpp_threshold_5_intended_diff_only": 0.006999969482421875, + "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_10_total_metric": 0.0034999847412109375, + "tpp_threshold_10_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_10_unintended_diff_only": 0.002499997615814209, + "tpp_threshold_20_total_metric": 0.010250017046928406, + "tpp_threshold_20_intended_diff_only": 0.013000011444091797, + "tpp_threshold_20_unintended_diff_only": 0.002749994397163391, + "tpp_threshold_50_total_metric": 0.06399999558925629, + "tpp_threshold_50_intended_diff_only": 0.06099998950958252, + "tpp_threshold_50_unintended_diff_only": -0.003000006079673767, + "tpp_threshold_100_total_metric": 0.09025001525878906, + "tpp_threshold_100_intended_diff_only": 0.09700000286102295, + "tpp_threshold_100_unintended_diff_only": 0.006749987602233887, + "tpp_threshold_500_total_metric": 0.37650005519390106, + "tpp_threshold_500_intended_diff_only": 0.3810000419616699, + "tpp_threshold_500_unintended_diff_only": 0.00449998676776886 }, "2": { - "tpp_threshold_2_total_metric": 0.00025004148483276367, - "tpp_threshold_2_intended_diff_only": 0.006000041961669922, - "tpp_threshold_2_unintended_diff_only": 0.005750000476837158, - "tpp_threshold_5_total_metric": 0.00700002908706665, - "tpp_threshold_5_intended_diff_only": 0.013999998569488525, - "tpp_threshold_5_unintended_diff_only": 0.006999969482421875, - "tpp_threshold_10_total_metric": 0.020500048995018005, - "tpp_threshold_10_intended_diff_only": 0.024000048637390137, - "tpp_threshold_10_unintended_diff_only": 0.0034999996423721313 + "tpp_threshold_2_total_metric": 0.010000035166740417, + "tpp_threshold_2_intended_diff_only": 0.008000016212463379, + "tpp_threshold_2_unintended_diff_only": -0.0020000189542770386, + "tpp_threshold_5_total_metric": 0.0025000721216201782, + "tpp_threshold_5_intended_diff_only": 0.010000050067901611, + "tpp_threshold_5_unintended_diff_only": 0.007499977946281433, + "tpp_threshold_10_total_metric": 0.022000014781951904, + "tpp_threshold_10_intended_diff_only": 0.027000010013580322, + "tpp_threshold_10_unintended_diff_only": 0.004999995231628418, + "tpp_threshold_20_total_metric": 0.050000011920928955, + "tpp_threshold_20_intended_diff_only": 0.05500000715255737, + "tpp_threshold_20_unintended_diff_only": 0.004999995231628418, + "tpp_threshold_50_total_metric": 0.11925002932548523, + "tpp_threshold_50_intended_diff_only": 0.12700003385543823, + "tpp_threshold_50_unintended_diff_only": 0.007750004529953003, + "tpp_threshold_100_total_metric": 0.2665000557899475, + "tpp_threshold_100_intended_diff_only": 0.28800004720687866, + "tpp_threshold_100_unintended_diff_only": 0.021499991416931152, + "tpp_threshold_500_total_metric": 0.4045000374317169, + "tpp_threshold_500_intended_diff_only": 0.437000036239624, + "tpp_threshold_500_unintended_diff_only": 0.032499998807907104 }, "3": { - "tpp_threshold_2_total_metric": 0.013750001788139343, - "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_2_unintended_diff_only": -0.007750019431114197, - "tpp_threshold_5_total_metric": 0.01774996519088745, - "tpp_threshold_5_intended_diff_only": 0.015999972820281982, - "tpp_threshold_5_unintended_diff_only": -0.0017499923706054688, - "tpp_threshold_10_total_metric": 0.014500021934509277, + "tpp_threshold_2_total_metric": 0.0062499940395355225, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": -0.0012499988079071045, + "tpp_threshold_5_total_metric": 0.009250015020370483, + "tpp_threshold_5_intended_diff_only": 0.004999995231628418, + "tpp_threshold_5_unintended_diff_only": -0.004250019788742065, + "tpp_threshold_10_total_metric": 0.020500019192695618, "tpp_threshold_10_intended_diff_only": 0.017000019550323486, - "tpp_threshold_10_unintended_diff_only": 0.002499997615814209 + "tpp_threshold_10_unintended_diff_only": -0.0034999996423721313, + "tpp_threshold_20_total_metric": 0.015999987721443176, + "tpp_threshold_20_intended_diff_only": 0.018999993801116943, + "tpp_threshold_20_unintended_diff_only": 0.003000006079673767, + "tpp_threshold_50_total_metric": 0.061250001192092896, + "tpp_threshold_50_intended_diff_only": 0.06000000238418579, + "tpp_threshold_50_unintended_diff_only": -0.0012499988079071045, + "tpp_threshold_100_total_metric": 0.11550003290176392, + "tpp_threshold_100_intended_diff_only": 0.11800003051757812, + "tpp_threshold_100_unintended_diff_only": 0.002499997615814209, + "tpp_threshold_500_total_metric": 0.338249996304512, + "tpp_threshold_500_intended_diff_only": 0.3479999899864197, + "tpp_threshold_500_unintended_diff_only": 0.009749993681907654 }, "5": { - "tpp_threshold_2_total_metric": -0.006000012159347534, - "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 0.006999999284744263, - "tpp_threshold_5_total_metric": 0.005999937653541565, - "tpp_threshold_5_intended_diff_only": 0.008999943733215332, - "tpp_threshold_5_unintended_diff_only": 0.003000006079673767, - "tpp_threshold_10_total_metric": 0.003749936819076538, - "tpp_threshold_10_intended_diff_only": 0.007999956607818604, - "tpp_threshold_10_unintended_diff_only": 0.004250019788742065 + "tpp_threshold_2_total_metric": 0.0024999678134918213, + "tpp_threshold_2_intended_diff_only": 0.001999974250793457, + "tpp_threshold_2_unintended_diff_only": -0.0004999935626983643, + "tpp_threshold_5_total_metric": 0.014999985694885254, + "tpp_threshold_5_intended_diff_only": 0.013999998569488525, + "tpp_threshold_5_unintended_diff_only": -0.0009999871253967285, + "tpp_threshold_10_total_metric": 0.0037499815225601196, + "tpp_threshold_10_intended_diff_only": 0.009999990463256836, + "tpp_threshold_10_unintended_diff_only": 0.006250008940696716, + "tpp_threshold_20_total_metric": 0.028749972581863403, + "tpp_threshold_20_intended_diff_only": 0.029999971389770508, + "tpp_threshold_20_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_50_total_metric": 0.0939999520778656, + "tpp_threshold_50_intended_diff_only": 0.09999996423721313, + "tpp_threshold_50_unintended_diff_only": 0.006000012159347534, + "tpp_threshold_100_total_metric": 0.2227500081062317, + "tpp_threshold_100_intended_diff_only": 0.2680000066757202, + "tpp_threshold_100_unintended_diff_only": 0.045249998569488525, + "tpp_threshold_500_total_metric": 0.34474998712539673, + "tpp_threshold_500_intended_diff_only": 0.4169999957084656, + "tpp_threshold_500_unintended_diff_only": 0.07225000858306885 }, "6": { - "tpp_threshold_2_total_metric": 0.012999966740608215, - "tpp_threshold_2_intended_diff_only": 0.011999964714050293, - "tpp_threshold_2_unintended_diff_only": -0.0010000020265579224, - "tpp_threshold_5_total_metric": 0.015250012278556824, - "tpp_threshold_5_intended_diff_only": 0.022000014781951904, - "tpp_threshold_5_unintended_diff_only": 0.006750002503395081, - "tpp_threshold_10_total_metric": 0.04249997437000275, - "tpp_threshold_10_intended_diff_only": 0.04399996995925903, - "tpp_threshold_10_unintended_diff_only": 0.0014999955892562866 + "tpp_threshold_2_total_metric": 0.002250000834465027, + "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_2_unintended_diff_only": 0.0037499815225601196, + "tpp_threshold_5_total_metric": 0.019249990582466125, + "tpp_threshold_5_intended_diff_only": 0.018999993801116943, + "tpp_threshold_5_unintended_diff_only": -0.00024999678134918213, + "tpp_threshold_10_total_metric": 0.05199997127056122, + "tpp_threshold_10_intended_diff_only": 0.05699998140335083, + "tpp_threshold_10_unintended_diff_only": 0.005000010132789612, + "tpp_threshold_20_total_metric": 0.10374997556209564, + "tpp_threshold_20_intended_diff_only": 0.11299997568130493, + "tpp_threshold_20_unintended_diff_only": 0.00925000011920929, + "tpp_threshold_50_total_metric": 0.1787499636411667, + "tpp_threshold_50_intended_diff_only": 0.18799996376037598, + "tpp_threshold_50_unintended_diff_only": 0.00925000011920929, + "tpp_threshold_100_total_metric": 0.2540000230073929, + "tpp_threshold_100_intended_diff_only": 0.2670000195503235, + "tpp_threshold_100_unintended_diff_only": 0.012999996542930603, + "tpp_threshold_500_total_metric": 0.34425002336502075, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.02875000238418579 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-73242-tokens-300000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-73242-tokens-300000000/scr/saebench_scr_custom_sae_eval_results.json index be277f34c89fc5c1db8f5c5cbe268d91a36f2075..3a634f451706732f244bfaa342828be98934dc1c 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-73242-tokens-300000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-73242-tokens-300000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "fcbaf017-7a89-432c-8c57-304abafb3518", - "datetime_epoch_millis": 1776887399925, + "eval_id": "9cc6a034-b9ee-4680-b53c-4ccfce129bf9", + "datetime_epoch_millis": 1777318087227, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.23352789707970661, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.22100008812377056, "scr_metric_threshold_10": 0.24491371204719437, "scr_dir2_threshold_10": 0.2513853639306334, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.01992975659933531, + "scr_metric_threshold_20": 0.3208456422724625, + "scr_dir2_threshold_20": 0.3207956805101175, + "scr_dir1_threshold_50": 0.172858450777758, + "scr_metric_threshold_50": 0.36177235777600864, + "scr_dir2_threshold_50": 0.36159608524231784, + "scr_dir1_threshold_100": 0.12464093150734233, + "scr_metric_threshold_100": 0.40015076045340137, + "scr_dir2_threshold_100": 0.4021737972533496, + "scr_dir1_threshold_500": -0.5861198618105339, + "scr_metric_threshold_500": 0.2573234281377508, + "scr_dir2_threshold_500": 0.23634322350169643 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.38461573730554943, "scr_metric_threshold_10": 0.09326427430734313, "scr_dir2_threshold_10": 0.09326427430734313, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3333335880540079, + "scr_metric_threshold_20": 0.1968911085011348, + "scr_dir2_threshold_20": 0.1968911085011348, + "scr_dir1_threshold_50": 0.5256410746257707, + "scr_metric_threshold_50": 0.2797926685058661, + "scr_dir2_threshold_50": 0.2797926685058661, + "scr_dir1_threshold_100": 0.47435892537422925, + "scr_metric_threshold_100": 0.32383416568792556, + "scr_dir2_threshold_100": 0.32383416568792556, + "scr_dir1_threshold_500": -2.602563916422071, + "scr_metric_threshold_500": 0.11917098285579106, + "scr_dir2_threshold_500": 0.11917098285579106 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.24242422874090183, "scr_metric_threshold_10": 0.31192657874830254, "scr_dir2_threshold_10": 0.31192657874830254, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.1439393152601855, + "scr_metric_threshold_20": 0.37920490690192576, + "scr_dir2_threshold_20": 0.37920490690192576, + "scr_dir1_threshold_50": 0.23484857036936357, + "scr_metric_threshold_50": 0.498471058939329, + "scr_dir2_threshold_50": 0.498471058939329, + "scr_dir1_threshold_100": -0.4772723475600257, + "scr_metric_threshold_100": 0.5993884600311694, + "scr_dir2_threshold_100": 0.5993884600311694, + "scr_dir1_threshold_500": -0.7348483445942438, + "scr_metric_threshold_500": 0.3149846431468334, + "scr_dir2_threshold_500": 0.3149846431468334 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.2968748544808716, "scr_metric_threshold_10": 0.09799002086611526, "scr_dir2_threshold_10": 0.09799002086611526, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.21874994179234863, + "scr_metric_threshold_20": 0.15577896145062706, + "scr_dir2_threshold_20": 0.15577896145062706, + "scr_dir1_threshold_50": 0.23437473806556883, + "scr_metric_threshold_50": 0.19849242204934214, + "scr_dir2_threshold_50": 0.19849242204934214, + "scr_dir1_threshold_100": 0.250000465661211, + "scr_metric_threshold_100": 0.0025125300775025, + "scr_dir2_threshold_100": 0.0025125300775025, + "scr_dir1_threshold_500": -0.9843752037267798, + "scr_metric_threshold_500": -0.128140681316927, + "scr_dir2_threshold_500": -0.128140681316927 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": 0.1543624402703411, "scr_metric_threshold_10": 0.31847140286929915, "scr_dir2_threshold_10": 0.31847140286929915, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.6845641960072886, + "scr_metric_threshold_20": 0.37579626297872115, + "scr_dir2_threshold_20": 0.37579626297872115, + "scr_dir1_threshold_50": -0.5503357261866425, + "scr_metric_threshold_50": 0.24203831933130424, + "scr_dir2_threshold_50": 0.24203831933130424, + "scr_dir1_threshold_100": -0.0872484053756193, + "scr_metric_threshold_100": 0.47452243203647043, + "scr_dir2_threshold_100": 0.47452243203647043, + "scr_dir1_threshold_500": -1.3892619624330698, + "scr_metric_threshold_500": -0.43312093326444, + "scr_dir2_threshold_500": -0.43312093326444 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.07894729412163301, "scr_metric_threshold_10": 0.5119047252994956, "scr_dir2_threshold_10": 0.5119047252994956, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.08421069803000372, + "scr_metric_threshold_20": 0.6190476077844602, + "scr_dir2_threshold_20": 0.6190476077844602, + "scr_dir1_threshold_50": -0.13684191373576507, + "scr_metric_threshold_50": 0.6865079937623274, + "scr_dir2_threshold_50": 0.6865079937623274, + "scr_dir1_threshold_100": -0.2578945568723999, + "scr_metric_threshold_100": 0.7261904902694248, + "scr_dir2_threshold_100": 0.7261904902694248, + "scr_dir1_threshold_500": -0.18947375685884774, + "scr_metric_threshold_500": 0.7698411891219645, + "scr_dir2_threshold_500": 0.7698411891219645 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.3224041847027747, "scr_metric_threshold_10": 0.13358787395939287, "scr_dir2_threshold_10": 0.13358787395939287, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3333333333333333, + "scr_metric_threshold_20": 0.1908397224425799, + "scr_dir2_threshold_20": 0.1908397224425799, + "scr_dir1_threshold_50": 0.4426228653879351, + "scr_metric_threshold_50": 0.25954203162185363, + "scr_dir2_threshold_50": 0.25954203162185363, + "scr_dir1_threshold_100": 0.4207648938353153, + "scr_metric_threshold_100": 0.3396945739985909, + "scr_dir2_threshold_100": 0.3396945739985909, + "scr_dir1_threshold_500": 0.5464479859815062, + "scr_metric_threshold_500": 0.5305342964411708, + "scr_dir2_threshold_500": 0.5305342964411708 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.19534878692091323, "scr_metric_threshold_10": 0.39914164188042695, "scr_dir2_threshold_10": 0.39914164188042695, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.274418633018972, + "scr_metric_threshold_20": 0.46781131470206244, + "scr_dir2_threshold_20": 0.46781131470206244, + "scr_dir1_threshold_50": 0.4139533620063974, + "scr_metric_threshold_50": 0.5107297323086109, + "scr_dir2_threshold_50": 0.5107297323086109, + "scr_dir1_threshold_100": 0.4372091501709683, + "scr_metric_threshold_100": 0.49785410470106733, + "scr_dir2_threshold_100": 0.49785410470106733, + "scr_dir1_threshold_500": 0.3162791071613722, + "scr_metric_threshold_500": 0.5364807317097504, + "scr_dir2_threshold_500": 0.5364807317097504 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.09302317844717961, "scr_metric_threshold_10": 0.09302317844717961, "scr_dir2_threshold_10": 0.14479639351469145, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.18139525341818818, + "scr_metric_threshold_20": 0.18139525341818818, + "scr_dir2_threshold_20": 0.1809955593194283, + "scr_dir1_threshold_50": 0.21860463568943592, + "scr_metric_threshold_50": 0.21860463568943592, + "scr_dir2_threshold_50": 0.2171944554199093, + "scr_dir1_threshold_100": 0.2372093268250598, + "scr_metric_threshold_100": 0.2372093268250598, + "scr_dir2_threshold_100": 0.2533936212246461, + "scr_dir1_threshold_500": 0.34883719640786326, + "scr_metric_threshold_500": 0.34883719640786326, + "scr_dir2_threshold_500": 0.1809955593194283 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-73242-tokens-300000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-73242-tokens-300000000/tpp/saebench_tpp_custom_sae_eval_results.json index 449fa1c402e45a520628fa746d59a5955a64d178..19137789664105c12b1f19d0156d5c9d437b4d88 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-73242-tokens-300000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-73242-tokens-300000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "b30b6c14-44f2-4a91-982b-81fb5d645849", - "datetime_epoch_millis": 1776887556630, + "eval_id": "cb378589-fd87-4036-8105-a6fe86bab619", + "datetime_epoch_millis": 1777318368573, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.012474995851516724, - "tpp_threshold_2_intended_diff_only": 0.012699997425079346, - "tpp_threshold_2_unintended_diff_only": 0.0002250015735626221, - "tpp_threshold_5_total_metric": 0.04307500272989273, - "tpp_threshold_5_intended_diff_only": 0.04770000576972962, - "tpp_threshold_5_unintended_diff_only": 0.004625003039836884, - "tpp_threshold_10_total_metric": 0.08982499539852142, - "tpp_threshold_10_intended_diff_only": 0.10350000262260438, - "tpp_threshold_10_unintended_diff_only": 0.013675007224082948, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.011049994826316833, + "tpp_threshold_2_intended_diff_only": 0.0125, + "tpp_threshold_2_unintended_diff_only": 0.0014500051736831665, + "tpp_threshold_5_total_metric": 0.04295000284910202, + "tpp_threshold_5_intended_diff_only": 0.04610000252723694, + "tpp_threshold_5_unintended_diff_only": 0.003149999678134918, + "tpp_threshold_10_total_metric": 0.090575011074543, + "tpp_threshold_10_intended_diff_only": 0.10490001440048217, + "tpp_threshold_10_unintended_diff_only": 0.014325003325939178, + "tpp_threshold_20_total_metric": 0.13920000940561295, + "tpp_threshold_20_intended_diff_only": 0.16050001382827758, + "tpp_threshold_20_unintended_diff_only": 0.021300004422664644, + "tpp_threshold_50_total_metric": 0.21207500994205475, + "tpp_threshold_50_intended_diff_only": 0.25560001134872434, + "tpp_threshold_50_unintended_diff_only": 0.043525001406669615, + "tpp_threshold_100_total_metric": 0.25417500734329224, + "tpp_threshold_100_intended_diff_only": 0.327700012922287, + "tpp_threshold_100_unintended_diff_only": 0.07352500557899475, + "tpp_threshold_500_total_metric": 0.26705001294612885, + "tpp_threshold_500_intended_diff_only": 0.4312000274658203, + "tpp_threshold_500_unintended_diff_only": 0.16415001451969147 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.0164499968290329, - "tpp_threshold_2_intended_diff_only": 0.016200006008148193, - "tpp_threshold_2_unintended_diff_only": -0.00024999082088470457, - "tpp_threshold_5_total_metric": 0.07195001244544982, - "tpp_threshold_5_intended_diff_only": 0.0788000226020813, - "tpp_threshold_5_unintended_diff_only": 0.00685001015663147, - "tpp_threshold_10_total_metric": 0.15554999709129333, - "tpp_threshold_10_intended_diff_only": 0.17980000972747803, - "tpp_threshold_10_unintended_diff_only": 0.024250012636184693, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.015349987149238586, + "tpp_threshold_2_intended_diff_only": 0.016600000858306884, + "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983, + "tpp_threshold_5_total_metric": 0.0738500028848648, + "tpp_threshold_5_intended_diff_only": 0.07960001230239869, + "tpp_threshold_5_unintended_diff_only": 0.005750009417533874, + "tpp_threshold_10_total_metric": 0.15465000569820403, + "tpp_threshold_10_intended_diff_only": 0.17980002164840697, + "tpp_threshold_10_unintended_diff_only": 0.025150015950202942, + "tpp_threshold_20_total_metric": 0.22005001604557037, + "tpp_threshold_20_intended_diff_only": 0.25740002393722533, + "tpp_threshold_20_unintended_diff_only": 0.03735000789165497, + "tpp_threshold_50_total_metric": 0.2962500214576721, + "tpp_threshold_50_intended_diff_only": 0.3764000296592712, + "tpp_threshold_50_unintended_diff_only": 0.08015000820159912, + "tpp_threshold_100_total_metric": 0.29900001287460326, + "tpp_threshold_100_intended_diff_only": 0.43260003328323365, + "tpp_threshold_100_unintended_diff_only": 0.13360002040863037, + "tpp_threshold_500_total_metric": 0.17340001463890076, + "tpp_threshold_500_intended_diff_only": 0.46620004177093505, + "tpp_threshold_500_unintended_diff_only": 0.2928000271320343 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.00849999487400055, - "tpp_threshold_2_intended_diff_only": 0.009199988842010499, - "tpp_threshold_2_unintended_diff_only": 0.0006999939680099488, - "tpp_threshold_5_total_metric": 0.014199993014335633, - "tpp_threshold_5_intended_diff_only": 0.01659998893737793, - "tpp_threshold_5_unintended_diff_only": 0.0023999959230422974, - "tpp_threshold_10_total_metric": 0.02409999370574951, - "tpp_threshold_10_intended_diff_only": 0.027199995517730714, - "tpp_threshold_10_unintended_diff_only": 0.0031000018119812013, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.006750002503395081, + "tpp_threshold_2_intended_diff_only": 0.008399999141693116, + "tpp_threshold_2_unintended_diff_only": 0.0016499966382980346, + "tpp_threshold_5_total_metric": 0.012050002813339233, + "tpp_threshold_5_intended_diff_only": 0.012599992752075195, + "tpp_threshold_5_unintended_diff_only": 0.0005499899387359619, + "tpp_threshold_10_total_metric": 0.026500016450881958, + "tpp_threshold_10_intended_diff_only": 0.03000000715255737, + "tpp_threshold_10_unintended_diff_only": 0.003499990701675415, + "tpp_threshold_20_total_metric": 0.058350002765655516, + "tpp_threshold_20_intended_diff_only": 0.06360000371932983, + "tpp_threshold_20_unintended_diff_only": 0.005250000953674316, + "tpp_threshold_50_total_metric": 0.12789999842643737, + "tpp_threshold_50_intended_diff_only": 0.1347999930381775, + "tpp_threshold_50_unintended_diff_only": 0.006899994611740112, + "tpp_threshold_100_total_metric": 0.2093500018119812, + "tpp_threshold_100_intended_diff_only": 0.22279999256134034, + "tpp_threshold_100_unintended_diff_only": 0.01344999074935913, + "tpp_threshold_500_total_metric": 0.36070001125335693, + "tpp_threshold_500_intended_diff_only": 0.3962000131607056, + "tpp_threshold_500_unintended_diff_only": 0.03550000190734863 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.0012500137090682983, "tpp_threshold_10_total_metric": 0.004249945282936096, "tpp_threshold_10_intended_diff_only": 0.006999969482421875, - "tpp_threshold_10_unintended_diff_only": 0.002750024199485779 + "tpp_threshold_10_unintended_diff_only": 0.002750024199485779, + "tpp_threshold_20_total_metric": 0.15900000929832458, + "tpp_threshold_20_intended_diff_only": 0.18800002336502075, + "tpp_threshold_20_unintended_diff_only": 0.029000014066696167, + "tpp_threshold_50_total_metric": 0.28624996542930603, + "tpp_threshold_50_intended_diff_only": 0.343999981880188, + "tpp_threshold_50_unintended_diff_only": 0.05775001645088196, + "tpp_threshold_100_total_metric": 0.34974999725818634, + "tpp_threshold_100_intended_diff_only": 0.42000001668930054, + "tpp_threshold_100_unintended_diff_only": 0.0702500194311142, + "tpp_threshold_500_total_metric": 0.18699999153614044, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.2690000385046005 }, "1": { - "tpp_threshold_2_total_metric": -2.9802322387695312e-08, - "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162, - "tpp_threshold_5_total_metric": 0.11024999618530273, - "tpp_threshold_5_intended_diff_only": 0.12300002574920654, - "tpp_threshold_5_unintended_diff_only": 0.012750029563903809, - "tpp_threshold_10_total_metric": 0.17725001275539398, + "tpp_threshold_2_total_metric": 0.004499971866607666, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 0.000500023365020752, + "tpp_threshold_5_total_metric": 0.11124998331069946, + "tpp_threshold_5_intended_diff_only": 0.12099999189376831, + "tpp_threshold_5_unintended_diff_only": 0.009750008583068848, + "tpp_threshold_10_total_metric": 0.1785000115633011, "tpp_threshold_10_intended_diff_only": 0.2070000171661377, - "tpp_threshold_10_unintended_diff_only": 0.029750004410743713 + "tpp_threshold_10_unintended_diff_only": 0.02850000560283661, + "tpp_threshold_20_total_metric": 0.21050001680850983, + "tpp_threshold_20_intended_diff_only": 0.24300003051757812, + "tpp_threshold_20_unintended_diff_only": 0.0325000137090683, + "tpp_threshold_50_total_metric": 0.27650000154972076, + "tpp_threshold_50_intended_diff_only": 0.35100001096725464, + "tpp_threshold_50_unintended_diff_only": 0.07450000941753387, + "tpp_threshold_100_total_metric": 0.2892500013113022, + "tpp_threshold_100_intended_diff_only": 0.4190000295639038, + "tpp_threshold_100_unintended_diff_only": 0.12975002825260162, + "tpp_threshold_500_total_metric": 0.1510000228881836, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.30800002813339233 }, "2": { - "tpp_threshold_2_total_metric": 0.05925004184246063, - "tpp_threshold_2_intended_diff_only": 0.057000041007995605, - "tpp_threshold_2_unintended_diff_only": -0.002250000834465027, - "tpp_threshold_5_total_metric": 0.19150005280971527, - "tpp_threshold_5_intended_diff_only": 0.1990000605583191, - "tpp_threshold_5_unintended_diff_only": 0.007500007748603821, - "tpp_threshold_10_total_metric": 0.2620000094175339, - "tpp_threshold_10_intended_diff_only": 0.30000001192092896, - "tpp_threshold_10_unintended_diff_only": 0.03800000250339508 + "tpp_threshold_2_total_metric": 0.04725000262260437, + "tpp_threshold_2_intended_diff_only": 0.050999999046325684, + "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_5_total_metric": 0.18800000846385956, + "tpp_threshold_5_intended_diff_only": 0.1980000138282776, + "tpp_threshold_5_unintended_diff_only": 0.01000000536441803, + "tpp_threshold_10_total_metric": 0.2615000456571579, + "tpp_threshold_10_intended_diff_only": 0.30100005865097046, + "tpp_threshold_10_unintended_diff_only": 0.03950001299381256, + "tpp_threshold_20_total_metric": 0.2785000205039978, + "tpp_threshold_20_intended_diff_only": 0.3460000157356262, + "tpp_threshold_20_unintended_diff_only": 0.06749999523162842, + "tpp_threshold_50_total_metric": 0.2682500630617142, + "tpp_threshold_50_intended_diff_only": 0.37900006771087646, + "tpp_threshold_50_unintended_diff_only": 0.11075000464916229, + "tpp_threshold_100_total_metric": 0.19450002908706665, + "tpp_threshold_100_intended_diff_only": 0.41300004720687866, + "tpp_threshold_100_unintended_diff_only": 0.218500018119812, + "tpp_threshold_500_total_metric": 0.0885000079870224, + "tpp_threshold_500_intended_diff_only": 0.4520000219345093, + "tpp_threshold_500_unintended_diff_only": 0.3635000139474869 }, "6": { - "tpp_threshold_2_total_metric": 0.010750025510787964, + "tpp_threshold_2_total_metric": 0.00849999487400055, "tpp_threshold_2_intended_diff_only": 0.008000016212463379, - "tpp_threshold_2_unintended_diff_only": -0.002750009298324585, - "tpp_threshold_5_total_metric": 0.035999998450279236, - "tpp_threshold_5_intended_diff_only": 0.046999990940093994, - "tpp_threshold_5_unintended_diff_only": 0.010999992489814758, - "tpp_threshold_10_total_metric": 0.2017500251531601, + "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704, + "tpp_threshold_5_total_metric": 0.043000027537345886, + "tpp_threshold_5_intended_diff_only": 0.0480000376701355, + "tpp_threshold_5_unintended_diff_only": 0.005000010132789612, + "tpp_threshold_10_total_metric": 0.20200002193450928, "tpp_threshold_10_intended_diff_only": 0.22800004482269287, - "tpp_threshold_10_unintended_diff_only": 0.026250019669532776 + "tpp_threshold_10_unintended_diff_only": 0.026000022888183594, + "tpp_threshold_20_total_metric": 0.2329999953508377, + "tpp_threshold_20_intended_diff_only": 0.2590000033378601, + "tpp_threshold_20_unintended_diff_only": 0.0260000079870224, + "tpp_threshold_50_total_metric": 0.3465000092983246, + "tpp_threshold_50_intended_diff_only": 0.4570000171661377, + "tpp_threshold_50_unintended_diff_only": 0.11050000786781311, + "tpp_threshold_100_total_metric": 0.3227500021457672, + "tpp_threshold_100_intended_diff_only": 0.4750000238418579, + "tpp_threshold_100_unintended_diff_only": 0.1522500216960907, + "tpp_threshold_500_total_metric": 0.21649999916553497, + "tpp_threshold_500_intended_diff_only": 0.4880000352859497, + "tpp_threshold_500_unintended_diff_only": 0.27150003612041473 }, "9": { - "tpp_threshold_2_total_metric": 0.0029999911785125732, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162, - "tpp_threshold_5_total_metric": 0.008250042796134949, - "tpp_threshold_5_intended_diff_only": 0.010000050067901611, - "tpp_threshold_5_unintended_diff_only": 0.0017500072717666626, - "tpp_threshold_10_total_metric": 0.13249999284744263, - "tpp_threshold_10_intended_diff_only": 0.15700000524520874, - "tpp_threshold_10_unintended_diff_only": 0.024500012397766113 + "tpp_threshold_2_total_metric": 0.007250010967254639, + "tpp_threshold_2_intended_diff_only": 0.008000016212463379, + "tpp_threshold_2_unintended_diff_only": 0.0007500052452087402, + "tpp_threshold_5_total_metric": 0.013250023126602173, + "tpp_threshold_5_intended_diff_only": 0.016000032424926758, + "tpp_threshold_5_unintended_diff_only": 0.002750009298324585, + "tpp_threshold_10_total_metric": 0.12700000405311584, + "tpp_threshold_10_intended_diff_only": 0.156000018119812, + "tpp_threshold_10_unintended_diff_only": 0.029000014066696167, + "tpp_threshold_20_total_metric": 0.21925003826618195, + "tpp_threshold_20_intended_diff_only": 0.2510000467300415, + "tpp_threshold_20_unintended_diff_only": 0.03175000846385956, + "tpp_threshold_50_total_metric": 0.30375006794929504, + "tpp_threshold_50_intended_diff_only": 0.3510000705718994, + "tpp_threshold_50_unintended_diff_only": 0.04725000262260437, + "tpp_threshold_100_total_metric": 0.33875003457069397, + "tpp_threshold_100_intended_diff_only": 0.4360000491142273, + "tpp_threshold_100_unintended_diff_only": 0.09725001454353333, + "tpp_threshold_500_total_metric": 0.22400005161762238, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.25200001895427704 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.009499996900558472, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": -0.004500001668930054, - "tpp_threshold_5_total_metric": 0.01124998927116394, - "tpp_threshold_5_intended_diff_only": 0.009999990463256836, - "tpp_threshold_5_unintended_diff_only": -0.0012499988079071045, - "tpp_threshold_10_total_metric": 0.004250004887580872, - "tpp_threshold_10_intended_diff_only": 0.009000003337860107, - "tpp_threshold_10_unintended_diff_only": 0.004749998450279236 + "tpp_threshold_2_total_metric": 0.004000023007392883, + "tpp_threshold_2_intended_diff_only": 0.008000016212463379, + "tpp_threshold_2_unintended_diff_only": 0.003999993205070496, + "tpp_threshold_5_total_metric": 0.0037499964237213135, + "tpp_threshold_5_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_5_unintended_diff_only": 0.002249985933303833, + "tpp_threshold_10_total_metric": 0.005000025033950806, + "tpp_threshold_10_intended_diff_only": 0.008000016212463379, + "tpp_threshold_10_unintended_diff_only": 0.0029999911785125732, + "tpp_threshold_20_total_metric": 0.023500025272369385, + "tpp_threshold_20_intended_diff_only": 0.027000010013580322, + "tpp_threshold_20_unintended_diff_only": 0.0034999847412109375, + "tpp_threshold_50_total_metric": 0.0910000205039978, + "tpp_threshold_50_intended_diff_only": 0.09200000762939453, + "tpp_threshold_50_unintended_diff_only": 0.0009999871253967285, + "tpp_threshold_100_total_metric": 0.13824999332427979, + "tpp_threshold_100_intended_diff_only": 0.14499998092651367, + "tpp_threshold_100_unintended_diff_only": 0.006749987602233887, + "tpp_threshold_500_total_metric": 0.4074999839067459, + "tpp_threshold_500_intended_diff_only": 0.421999990940094, + "tpp_threshold_500_unintended_diff_only": 0.014500007033348083 }, "2": { - "tpp_threshold_2_total_metric": 0.002750009298324585, - "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": 0.0062499940395355225, - "tpp_threshold_5_total_metric": 0.006750032305717468, - "tpp_threshold_5_intended_diff_only": 0.013000011444091797, - "tpp_threshold_5_unintended_diff_only": 0.006249979138374329, - "tpp_threshold_10_total_metric": 0.03275001049041748, - "tpp_threshold_10_intended_diff_only": 0.03600001335144043, - "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492 + "tpp_threshold_2_total_metric": 0.012250050902366638, + "tpp_threshold_2_intended_diff_only": 0.01100003719329834, + "tpp_threshold_2_unintended_diff_only": -0.0012500137090682983, + "tpp_threshold_5_total_metric": 0.0015000402927398682, + "tpp_threshold_5_intended_diff_only": 0.008000016212463379, + "tpp_threshold_5_unintended_diff_only": 0.006499975919723511, + "tpp_threshold_10_total_metric": 0.03575006127357483, + "tpp_threshold_10_intended_diff_only": 0.04300004243850708, + "tpp_threshold_10_unintended_diff_only": 0.007249981164932251, + "tpp_threshold_20_total_metric": 0.10075002908706665, + "tpp_threshold_20_intended_diff_only": 0.10800004005432129, + "tpp_threshold_20_unintended_diff_only": 0.007250010967254639, + "tpp_threshold_50_total_metric": 0.20775003731250763, + "tpp_threshold_50_intended_diff_only": 0.2250000238418579, + "tpp_threshold_50_unintended_diff_only": 0.01724998652935028, + "tpp_threshold_100_total_metric": 0.37000003457069397, + "tpp_threshold_100_intended_diff_only": 0.4010000228881836, + "tpp_threshold_100_unintended_diff_only": 0.030999988317489624, + "tpp_threshold_500_total_metric": 0.3917500227689743, + "tpp_threshold_500_intended_diff_only": 0.43800002336502075, + "tpp_threshold_500_unintended_diff_only": 0.04625000059604645 }, "3": { - "tpp_threshold_2_total_metric": 0.01199999451637268, - "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_2_unintended_diff_only": -0.006000012159347534, - "tpp_threshold_5_total_metric": 0.017499983310699463, - "tpp_threshold_5_intended_diff_only": 0.014999985694885254, - "tpp_threshold_5_unintended_diff_only": -0.002499997615814209, - "tpp_threshold_10_total_metric": 0.016249999403953552, - "tpp_threshold_10_intended_diff_only": 0.018000006675720215, - "tpp_threshold_10_unintended_diff_only": 0.0017500072717666626 + "tpp_threshold_2_total_metric": 0.003250017762184143, + "tpp_threshold_2_intended_diff_only": 0.003000020980834961, + "tpp_threshold_2_unintended_diff_only": -0.00024999678134918213, + "tpp_threshold_5_total_metric": 0.008250042796134949, + "tpp_threshold_5_intended_diff_only": 0.003000020980834961, + "tpp_threshold_5_unintended_diff_only": -0.005250021815299988, + "tpp_threshold_10_total_metric": 0.015750005841255188, + "tpp_threshold_10_intended_diff_only": 0.013999998569488525, + "tpp_threshold_10_unintended_diff_only": -0.0017500072717666626, + "tpp_threshold_20_total_metric": 0.01124998927116394, + "tpp_threshold_20_intended_diff_only": 0.013999998569488525, + "tpp_threshold_20_unintended_diff_only": 0.002750009298324585, + "tpp_threshold_50_total_metric": 0.040999963879585266, + "tpp_threshold_50_intended_diff_only": 0.038999974727630615, + "tpp_threshold_50_unintended_diff_only": -0.001999989151954651, + "tpp_threshold_100_total_metric": 0.09475001692771912, + "tpp_threshold_100_intended_diff_only": 0.09799998998641968, + "tpp_threshold_100_unintended_diff_only": 0.0032499730587005615, + "tpp_threshold_500_total_metric": 0.31800004839897156, + "tpp_threshold_500_intended_diff_only": 0.33000004291534424, + "tpp_threshold_500_unintended_diff_only": 0.01199999451637268 }, "5": { - "tpp_threshold_2_total_metric": -0.005750015377998352, - "tpp_threshold_2_intended_diff_only": 0.001999974250793457, - "tpp_threshold_2_unintended_diff_only": 0.007749989628791809, - "tpp_threshold_5_total_metric": 0.004749968647956848, - "tpp_threshold_5_intended_diff_only": 0.006999969482421875, - "tpp_threshold_5_unintended_diff_only": 0.002250000834465027, - "tpp_threshold_10_total_metric": 0.012499973177909851, + "tpp_threshold_2_total_metric": 0.0029999613761901855, + "tpp_threshold_2_intended_diff_only": 0.0029999613761901855, + "tpp_threshold_2_unintended_diff_only": 0.0, + "tpp_threshold_5_total_metric": 0.013499945402145386, + "tpp_threshold_5_intended_diff_only": 0.012999951839447021, + "tpp_threshold_5_unintended_diff_only": -0.0004999935626983643, + "tpp_threshold_10_total_metric": 0.010749980807304382, "tpp_threshold_10_intended_diff_only": 0.015999972820281982, - "tpp_threshold_10_unintended_diff_only": 0.0034999996423721313 + "tpp_threshold_10_unintended_diff_only": 0.0052499920129776, + "tpp_threshold_20_total_metric": 0.03550000488758087, + "tpp_threshold_20_intended_diff_only": 0.03700000047683716, + "tpp_threshold_20_unintended_diff_only": 0.0014999955892562866, + "tpp_threshold_50_total_metric": 0.10649995505809784, + "tpp_threshold_50_intended_diff_only": 0.11499994993209839, + "tpp_threshold_50_unintended_diff_only": 0.00849999487400055, + "tpp_threshold_100_total_metric": 0.17649997770786285, + "tpp_threshold_100_intended_diff_only": 0.18699997663497925, + "tpp_threshold_100_unintended_diff_only": 0.010499998927116394, + "tpp_threshold_500_total_metric": 0.3427499830722809, + "tpp_threshold_500_intended_diff_only": 0.4179999828338623, + "tpp_threshold_500_unintended_diff_only": 0.07524999976158142 }, "6": { - "tpp_threshold_2_total_metric": 0.02399998903274536, - "tpp_threshold_2_intended_diff_only": 0.02399998903274536, - "tpp_threshold_2_unintended_diff_only": 0.0, - "tpp_threshold_5_total_metric": 0.030749991536140442, - "tpp_threshold_5_intended_diff_only": 0.03799998760223389, - "tpp_threshold_5_unintended_diff_only": 0.007249996066093445, - "tpp_threshold_10_total_metric": 0.0547499805688858, - "tpp_threshold_10_intended_diff_only": 0.05699998140335083, - "tpp_threshold_10_unintended_diff_only": 0.002250000834465027 + "tpp_threshold_2_total_metric": 0.011249959468841553, + "tpp_threshold_2_intended_diff_only": 0.01699995994567871, + "tpp_threshold_2_unintended_diff_only": 0.005750000476837158, + "tpp_threshold_5_total_metric": 0.03324998915195465, + "tpp_threshold_5_intended_diff_only": 0.03299999237060547, + "tpp_threshold_5_unintended_diff_only": -0.00024999678134918213, + "tpp_threshold_10_total_metric": 0.06525000929832458, + "tpp_threshold_10_intended_diff_only": 0.0690000057220459, + "tpp_threshold_10_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_20_total_metric": 0.12074996531009674, + "tpp_threshold_20_intended_diff_only": 0.13199996948242188, + "tpp_threshold_20_unintended_diff_only": 0.011250004172325134, + "tpp_threshold_50_total_metric": 0.19325001537799835, + "tpp_threshold_50_intended_diff_only": 0.203000009059906, + "tpp_threshold_50_unintended_diff_only": 0.009749993681907654, + "tpp_threshold_100_total_metric": 0.2672499865293503, + "tpp_threshold_100_intended_diff_only": 0.28299999237060547, + "tpp_threshold_100_unintended_diff_only": 0.015750005841255188, + "tpp_threshold_500_total_metric": 0.343500018119812, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.02950000762939453 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-85449-tokens-350000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-85449-tokens-350000000/scr/saebench_scr_custom_sae_eval_results.json index 71e653dbd88709b1de22f72a50c3beafe14cf32b..71dd8f1e6cf6752c03f12b32e1d098bcbf8cf981 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-85449-tokens-350000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-85449-tokens-350000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "c088c71e-6651-4896-8939-bf512c0efaab", - "datetime_epoch_millis": 1776886948169, + "eval_id": "d0915c16-1395-4225-9f5d-c44b9b139e67", + "datetime_epoch_millis": 1777318183720, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.25788004958257216, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.09602766224957492, "scr_metric_threshold_10": 0.23316250700532673, "scr_dir2_threshold_10": 0.2286350036494662, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.09827302483631974, + "scr_metric_threshold_20": 0.2807216968715855, + "scr_dir2_threshold_20": 0.2703012872826715, + "scr_dir1_threshold_50": 0.2255021979564478, + "scr_metric_threshold_50": 0.3608777384298916, + "scr_dir2_threshold_50": 0.34901043006114646, + "scr_dir1_threshold_100": 0.2811808167584396, + "scr_metric_threshold_100": 0.4057409996421379, + "scr_dir2_threshold_100": 0.39424987922336285, + "scr_dir1_threshold_500": -0.36252936226044397, + "scr_metric_threshold_500": 0.22941208167954583, + "scr_dir2_threshold_500": 0.2192258332390753 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.2820514388024664, "scr_metric_threshold_10": 0.11139890852479135, "scr_dir2_threshold_10": 0.11139890852479135, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.30769289550924905, + "scr_metric_threshold_20": 0.15284976573523867, + "scr_dir2_threshold_20": 0.15284976573523867, + "scr_dir1_threshold_50": 0.47435892537422925, + "scr_metric_threshold_50": 0.24093260568319425, + "scr_dir2_threshold_50": 0.24093260568319425, + "scr_dir1_threshold_100": 0.4102564298503083, + "scr_metric_threshold_100": 0.27720202853425396, + "scr_dir2_threshold_100": 0.27720202853425396, + "scr_dir1_threshold_500": -1.9999992358379763, + "scr_metric_threshold_500": 0.11658034288417893, + "scr_dir2_threshold_500": 0.11658034288417893 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.2878790820706107, "scr_metric_threshold_10": 0.2293577463248362, "scr_dir2_threshold_10": 0.2293577463248362, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.06060617007278538, + "scr_metric_threshold_20": 0.1620796004484019, + "scr_dir2_threshold_20": 0.1620796004484019, + "scr_dir1_threshold_50": 0.3333334838500799, + "scr_metric_threshold_50": 0.41590222651586267, + "scr_dir2_threshold_50": 0.41590222651586267, + "scr_dir1_threshold_100": 0.454545372445411, + "scr_metric_threshold_100": 0.6360855973679174, + "scr_dir2_threshold_100": 0.6360855973679174, + "scr_dir1_threshold_500": -0.871212001482891, + "scr_metric_threshold_500": 0.2660550659387731, + "scr_dir2_threshold_500": 0.2660550659387731 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": -0.23437473806556883, "scr_metric_threshold_10": 0.11809041124652613, "scr_dir2_threshold_10": 0.11809041124652613, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.45312467985791743, + "scr_metric_threshold_20": 0.18844230173933216, + "scr_dir2_threshold_20": 0.18844230173933216, + "scr_dir1_threshold_50": 0.1093754365573853, + "scr_metric_threshold_50": 0.21859296219014387, + "scr_dir2_threshold_50": 0.21859296219014387, + "scr_dir1_threshold_100": 0.5312495925464404, + "scr_metric_threshold_100": 0.06532668057741017, + "scr_dir2_threshold_100": 0.06532668057741017, + "scr_dir1_threshold_500": 0.18750034924590825, + "scr_metric_threshold_500": -0.12311547140153112, + "scr_dir2_threshold_500": -0.12311547140153112 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": -0.4362420268780965, "scr_metric_threshold_10": 0.2611465427598772, "scr_dir2_threshold_10": 0.2611465427598772, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.2147651516818312, + "scr_metric_threshold_20": 0.28343943845592845, + "scr_dir2_threshold_20": 0.28343943845592845, + "scr_dir1_threshold_50": -0.2550338926436262, + "scr_metric_threshold_50": 0.3312102817629155, + "scr_dir2_threshold_50": 0.3312102817629155, + "scr_dir1_threshold_100": -0.3825506389498381, + "scr_metric_threshold_100": 0.32165607513677746, + "scr_dir2_threshold_100": 0.32165607513677746, + "scr_dir1_threshold_500": -1.3020139570886529, + "scr_metric_threshold_500": -0.4840764488389054, + "scr_dir2_threshold_500": -0.4840764488389054 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.07894729412163301, "scr_metric_threshold_10": 0.5119047252994956, "scr_dir2_threshold_10": 0.5119047252994956, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.12105264313663487, + "scr_metric_threshold_20": 0.6150794054390177, + "scr_dir2_threshold_20": 0.6150794054390177, + "scr_dir1_threshold_50": -0.11052614902855408, + "scr_metric_threshold_50": 0.6666667455087788, + "scr_dir2_threshold_50": 0.6666667455087788, + "scr_dir1_threshold_100": -0.2473683764729798, + "scr_metric_threshold_100": 0.742063536177531, + "scr_dir2_threshold_100": 0.742063536177531, + "scr_dir1_threshold_500": -0.22631570196547887, + "scr_metric_threshold_500": 0.7499999408684159, + "scr_dir2_threshold_500": 0.7499999408684159 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.3387977447943639, "scr_metric_threshold_10": 0.1183206688641517, "scr_dir2_threshold_10": 0.1183206688641517, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3879780993606342, + "scr_metric_threshold_20": 0.19465646684173443, + "scr_dir2_threshold_20": 0.19465646684173443, + "scr_dir1_threshold_50": 0.4153004823742847, + "scr_metric_threshold_50": 0.2671755204201627, + "scr_dir2_threshold_50": 0.2671755204201627, + "scr_dir1_threshold_100": 0.497267631415236, + "scr_metric_threshold_100": 0.39694664998040097, + "scr_dir2_threshold_100": 0.39694664998040097, + "scr_dir1_threshold_500": 0.4699452484015855, + "scr_metric_threshold_500": 0.5114503469467752, + "scr_dir2_threshold_500": 0.5114503469467752 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.28372083739245524, "scr_metric_threshold_10": 0.3476393872642003, "scr_dir2_threshold_10": 0.3476393872642003, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3395348953259431, + "scr_metric_threshold_20": 0.41201726948797046, + "scr_dir2_threshold_20": 0.41201726948797046, + "scr_dir1_threshold_50": 0.5069767918955439, + "scr_metric_threshold_50": 0.4163090600858358, + "scr_dir2_threshold_50": 0.4163090600858358, + "scr_dir1_threshold_100": 0.5999999445538274, + "scr_metric_threshold_100": 0.4206008506837011, + "scr_dir2_threshold_100": 0.4206008506837011, + "scr_dir1_threshold_500": 0.5069767918955439, + "scr_metric_threshold_500": 0.4635192682902496, + "scr_dir2_threshold_500": 0.4635192682902496 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.16744166575873534, "scr_metric_threshold_10": 0.16744166575873534, "scr_dir2_threshold_10": 0.1312216389118512, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2372093268250598, + "scr_metric_threshold_20": 0.2372093268250598, + "scr_dir2_threshold_20": 0.15384605011374777, + "scr_dir1_threshold_50": 0.3302325052722394, + "scr_metric_threshold_50": 0.3302325052722394, + "scr_dir2_threshold_50": 0.23529403832227772, + "scr_dir1_threshold_100": 0.386046578679111, + "scr_metric_threshold_100": 0.386046578679111, + "scr_dir2_threshold_100": 0.29411761532891106, + "scr_dir1_threshold_500": 0.33488360874841044, + "scr_metric_threshold_500": 0.33488360874841044, + "scr_dir2_threshold_500": 0.2533936212246461 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-85449-tokens-350000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-85449-tokens-350000000/tpp/saebench_tpp_custom_sae_eval_results.json index 268bbc9e8e4739e001a0c8c443371ef9c33e4c4d..a1bef5f7f3c95fe2ed7a127546e3a11041021f9b 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-85449-tokens-350000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-85449-tokens-350000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "60d28ef7-797a-4994-b57e-887239d53790", - "datetime_epoch_millis": 1776887094404, + "eval_id": "a65f4a04-2992-47bb-bcea-f7e63ab10dc8", + "datetime_epoch_millis": 1777318461717, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.01734999269247055, - "tpp_threshold_2_intended_diff_only": 0.018299996852874756, - "tpp_threshold_2_unintended_diff_only": 0.0009500041604042053, - "tpp_threshold_5_total_metric": 0.04620000720024109, - "tpp_threshold_5_intended_diff_only": 0.053400009870529175, - "tpp_threshold_5_unintended_diff_only": 0.007200002670288086, - "tpp_threshold_10_total_metric": 0.09057499915361404, - "tpp_threshold_10_intended_diff_only": 0.10970000624656677, - "tpp_threshold_10_unintended_diff_only": 0.01912500709295273, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.01579999029636383, + "tpp_threshold_2_intended_diff_only": 0.017899996042251586, + "tpp_threshold_2_unintended_diff_only": 0.0021000057458877563, + "tpp_threshold_5_total_metric": 0.0470500037074089, + "tpp_threshold_5_intended_diff_only": 0.05280000567436218, + "tpp_threshold_5_unintended_diff_only": 0.005750001966953278, + "tpp_threshold_10_total_metric": 0.09127501398324966, + "tpp_threshold_10_intended_diff_only": 0.11090001463890076, + "tpp_threshold_10_unintended_diff_only": 0.019625000655651093, + "tpp_threshold_20_total_metric": 0.13517500013113024, + "tpp_threshold_20_intended_diff_only": 0.1615000069141388, + "tpp_threshold_20_unintended_diff_only": 0.026325006783008576, + "tpp_threshold_50_total_metric": 0.19572501331567765, + "tpp_threshold_50_intended_diff_only": 0.2491000235080719, + "tpp_threshold_50_unintended_diff_only": 0.053375010192394254, + "tpp_threshold_100_total_metric": 0.23380002230405808, + "tpp_threshold_100_intended_diff_only": 0.32700003385543824, + "tpp_threshold_100_unintended_diff_only": 0.09320001155138015, + "tpp_threshold_500_total_metric": 0.26150002181529997, + "tpp_threshold_500_intended_diff_only": 0.4305000364780426, + "tpp_threshold_500_unintended_diff_only": 0.16900001466274261 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.02659999132156372, - "tpp_threshold_2_intended_diff_only": 0.028600001335144044, - "tpp_threshold_2_unintended_diff_only": 0.002000010013580322, - "tpp_threshold_5_total_metric": 0.07955000400543213, - "tpp_threshold_5_intended_diff_only": 0.09200001955032348, - "tpp_threshold_5_unintended_diff_only": 0.012450015544891358, - "tpp_threshold_10_total_metric": 0.15970000624656677, + "tpp_threshold_2_total_metric": 0.024899989366531372, + "tpp_threshold_2_intended_diff_only": 0.027799999713897704, + "tpp_threshold_2_unintended_diff_only": 0.002900010347366333, + "tpp_threshold_5_total_metric": 0.08175000250339508, + "tpp_threshold_5_intended_diff_only": 0.09320001602172852, + "tpp_threshold_5_unintended_diff_only": 0.011450013518333435, + "tpp_threshold_10_total_metric": 0.158950012922287, "tpp_threshold_10_intended_diff_only": 0.19560002088546752, - "tpp_threshold_10_unintended_diff_only": 0.03590001463890076, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_10_unintended_diff_only": 0.03665000796318054, + "tpp_threshold_20_total_metric": 0.22034999430179597, + "tpp_threshold_20_intended_diff_only": 0.26880000829696654, + "tpp_threshold_20_unintended_diff_only": 0.04845001399517059, + "tpp_threshold_50_total_metric": 0.28165002465248107, + "tpp_threshold_50_intended_diff_only": 0.38360004425048827, + "tpp_threshold_50_unintended_diff_only": 0.1019500195980072, + "tpp_threshold_100_total_metric": 0.2662500232458115, + "tpp_threshold_100_intended_diff_only": 0.43460004329681395, + "tpp_threshold_100_unintended_diff_only": 0.1683500200510025, + "tpp_threshold_500_total_metric": 0.15755001902580262, + "tpp_threshold_500_intended_diff_only": 0.46600004434585574, + "tpp_threshold_500_unintended_diff_only": 0.3084500253200531 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.00809999406337738, + "tpp_threshold_2_total_metric": 0.006699991226196289, "tpp_threshold_2_intended_diff_only": 0.007999992370605469, - "tpp_threshold_2_unintended_diff_only": -0.00010000169277191162, - "tpp_threshold_5_total_metric": 0.012850010395050048, - "tpp_threshold_5_intended_diff_only": 0.014800000190734863, - "tpp_threshold_5_unintended_diff_only": 0.0019499897956848145, - "tpp_threshold_10_total_metric": 0.021449992060661317, - "tpp_threshold_10_intended_diff_only": 0.023799991607666014, - "tpp_threshold_10_unintended_diff_only": 0.0023499995470046995, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_unintended_diff_only": 0.0013000011444091798, + "tpp_threshold_5_total_metric": 0.01235000491142273, + "tpp_threshold_5_intended_diff_only": 0.01239999532699585, + "tpp_threshold_5_unintended_diff_only": 4.999041557312012e-05, + "tpp_threshold_10_total_metric": 0.02360001504421234, + "tpp_threshold_10_intended_diff_only": 0.026200008392333985, + "tpp_threshold_10_unintended_diff_only": 0.002599993348121643, + "tpp_threshold_20_total_metric": 0.05000000596046448, + "tpp_threshold_20_intended_diff_only": 0.05420000553131103, + "tpp_threshold_20_unintended_diff_only": 0.004199999570846558, + "tpp_threshold_50_total_metric": 0.10980000197887421, + "tpp_threshold_50_intended_diff_only": 0.11460000276565552, + "tpp_threshold_50_unintended_diff_only": 0.004800000786781311, + "tpp_threshold_100_total_metric": 0.20135002136230468, + "tpp_threshold_100_intended_diff_only": 0.2194000244140625, + "tpp_threshold_100_unintended_diff_only": 0.018050003051757812, + "tpp_threshold_500_total_metric": 0.36545002460479736, + "tpp_threshold_500_intended_diff_only": 0.3950000286102295, + "tpp_threshold_500_unintended_diff_only": 0.029550004005432128 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.0012500137090682983, "tpp_threshold_10_total_metric": 0.0029999613761901855, "tpp_threshold_10_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_10_unintended_diff_only": 0.003000020980834961 + "tpp_threshold_10_unintended_diff_only": 0.003000020980834961, + "tpp_threshold_20_total_metric": 0.1839999556541443, + "tpp_threshold_20_intended_diff_only": 0.21999996900558472, + "tpp_threshold_20_unintended_diff_only": 0.03600001335144043, + "tpp_threshold_50_total_metric": 0.28849999606609344, + "tpp_threshold_50_intended_diff_only": 0.3500000238418579, + "tpp_threshold_50_unintended_diff_only": 0.061500027775764465, + "tpp_threshold_100_total_metric": 0.3292499929666519, + "tpp_threshold_100_intended_diff_only": 0.42500001192092896, + "tpp_threshold_100_unintended_diff_only": 0.09575001895427704, + "tpp_threshold_500_total_metric": 0.18050000071525574, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.27550002932548523 }, "1": { - "tpp_threshold_2_total_metric": 0.06400002539157867, - "tpp_threshold_2_intended_diff_only": 0.07600003480911255, - "tpp_threshold_2_unintended_diff_only": 0.012000009417533875, - "tpp_threshold_5_total_metric": 0.1745000034570694, + "tpp_threshold_2_total_metric": 0.06699997186660767, + "tpp_threshold_2_intended_diff_only": 0.07499998807907104, + "tpp_threshold_2_unintended_diff_only": 0.008000016212463379, + "tpp_threshold_5_total_metric": 0.1770000010728836, "tpp_threshold_5_intended_diff_only": 0.2160000205039978, - "tpp_threshold_5_unintended_diff_only": 0.041500017046928406, - "tpp_threshold_10_total_metric": 0.2072499841451645, + "tpp_threshold_5_unintended_diff_only": 0.0390000194311142, + "tpp_threshold_10_total_metric": 0.20799998939037323, "tpp_threshold_10_intended_diff_only": 0.27799999713897705, - "tpp_threshold_10_unintended_diff_only": 0.07075001299381256 + "tpp_threshold_10_unintended_diff_only": 0.07000000774860382, + "tpp_threshold_20_total_metric": 0.2319999784231186, + "tpp_threshold_20_intended_diff_only": 0.3059999942779541, + "tpp_threshold_20_unintended_diff_only": 0.07400001585483551, + "tpp_threshold_50_total_metric": 0.25325000286102295, + "tpp_threshold_50_intended_diff_only": 0.37400001287460327, + "tpp_threshold_50_unintended_diff_only": 0.12075001001358032, + "tpp_threshold_100_total_metric": 0.24450001120567322, + "tpp_threshold_100_intended_diff_only": 0.4280000329017639, + "tpp_threshold_100_unintended_diff_only": 0.1835000216960907, + "tpp_threshold_500_total_metric": 0.13875001668930054, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.3202500343322754 }, "2": { - "tpp_threshold_2_total_metric": 0.044499993324279785, - "tpp_threshold_2_intended_diff_only": 0.041999995708465576, - "tpp_threshold_2_unintended_diff_only": -0.002499997615814209, - "tpp_threshold_5_total_metric": 0.17100001871585846, - "tpp_threshold_5_intended_diff_only": 0.17800003290176392, - "tpp_threshold_5_unintended_diff_only": 0.0070000141859054565, - "tpp_threshold_10_total_metric": 0.24350005388259888, - "tpp_threshold_10_intended_diff_only": 0.29200005531311035, - "tpp_threshold_10_unintended_diff_only": 0.048500001430511475 + "tpp_threshold_2_total_metric": 0.032000020146369934, + "tpp_threshold_2_intended_diff_only": 0.03600001335144043, + "tpp_threshold_2_unintended_diff_only": 0.003999993205070496, + "tpp_threshold_5_total_metric": 0.16725003719329834, + "tpp_threshold_5_intended_diff_only": 0.1770000457763672, + "tpp_threshold_5_unintended_diff_only": 0.009750008583068848, + "tpp_threshold_10_total_metric": 0.24400004744529724, + "tpp_threshold_10_intended_diff_only": 0.2930000424385071, + "tpp_threshold_10_unintended_diff_only": 0.04899999499320984, + "tpp_threshold_20_total_metric": 0.25600001215934753, + "tpp_threshold_20_intended_diff_only": 0.3320000171661377, + "tpp_threshold_20_unintended_diff_only": 0.07600000500679016, + "tpp_threshold_50_total_metric": 0.23575003445148468, + "tpp_threshold_50_intended_diff_only": 0.36600005626678467, + "tpp_threshold_50_unintended_diff_only": 0.1302500218153, + "tpp_threshold_100_total_metric": 0.1612500548362732, + "tpp_threshold_100_intended_diff_only": 0.4020000696182251, + "tpp_threshold_100_unintended_diff_only": 0.2407500147819519, + "tpp_threshold_500_total_metric": 0.07950001955032349, + "tpp_threshold_500_intended_diff_only": 0.45100003480911255, + "tpp_threshold_500_unintended_diff_only": 0.37150001525878906 }, "6": { - "tpp_threshold_2_total_metric": 0.010500013828277588, + "tpp_threshold_2_total_metric": 0.00849999487400055, "tpp_threshold_2_intended_diff_only": 0.008000016212463379, - "tpp_threshold_2_unintended_diff_only": -0.002499997615814209, - "tpp_threshold_5_total_metric": 0.0337500125169754, - "tpp_threshold_5_intended_diff_only": 0.04400002956390381, - "tpp_threshold_5_unintended_diff_only": 0.010250017046928406, - "tpp_threshold_10_total_metric": 0.20775000751018524, + "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704, + "tpp_threshold_5_total_metric": 0.04025000333786011, + "tpp_threshold_5_intended_diff_only": 0.04500001668930054, + "tpp_threshold_5_unintended_diff_only": 0.00475001335144043, + "tpp_threshold_10_total_metric": 0.2070000171661377, "tpp_threshold_10_intended_diff_only": 0.2380000352859497, - "tpp_threshold_10_unintended_diff_only": 0.030250027775764465 + "tpp_threshold_10_unintended_diff_only": 0.03100001811981201, + "tpp_threshold_20_total_metric": 0.239500030875206, + "tpp_threshold_20_intended_diff_only": 0.27000004053115845, + "tpp_threshold_20_unintended_diff_only": 0.030500009655952454, + "tpp_threshold_50_total_metric": 0.33225004374980927, + "tpp_threshold_50_intended_diff_only": 0.4630000591278076, + "tpp_threshold_50_unintended_diff_only": 0.13075001537799835, + "tpp_threshold_100_total_metric": 0.2847500443458557, + "tpp_threshold_100_intended_diff_only": 0.48100006580352783, + "tpp_threshold_100_unintended_diff_only": 0.19625002145767212, + "tpp_threshold_500_total_metric": 0.19950000941753387, + "tpp_threshold_500_intended_diff_only": 0.4880000352859497, + "tpp_threshold_500_unintended_diff_only": 0.28850002586841583 }, "9": { - "tpp_threshold_2_total_metric": 0.0037499815225601196, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983, - "tpp_threshold_5_total_metric": 0.00475001335144043, - "tpp_threshold_5_intended_diff_only": 0.00700002908706665, - "tpp_threshold_5_unintended_diff_only": 0.0022500157356262207, - "tpp_threshold_10_total_metric": 0.13700002431869507, - "tpp_threshold_10_intended_diff_only": 0.1640000343322754, - "tpp_threshold_10_unintended_diff_only": 0.027000010013580322 + "tpp_threshold_2_total_metric": 0.006750017404556274, + "tpp_threshold_2_intended_diff_only": 0.008000016212463379, + "tpp_threshold_2_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_5_total_metric": 0.010499998927116394, + "tpp_threshold_5_intended_diff_only": 0.013000011444091797, + "tpp_threshold_5_unintended_diff_only": 0.002500012516975403, + "tpp_threshold_10_total_metric": 0.13275004923343658, + "tpp_threshold_10_intended_diff_only": 0.16300004720687866, + "tpp_threshold_10_unintended_diff_only": 0.030249997973442078, + "tpp_threshold_20_total_metric": 0.1902499943971634, + "tpp_threshold_20_intended_diff_only": 0.2160000205039978, + "tpp_threshold_20_unintended_diff_only": 0.02575002610683441, + "tpp_threshold_50_total_metric": 0.29850004613399506, + "tpp_threshold_50_intended_diff_only": 0.36500006914138794, + "tpp_threshold_50_unintended_diff_only": 0.06650002300739288, + "tpp_threshold_100_total_metric": 0.31150001287460327, + "tpp_threshold_100_intended_diff_only": 0.437000036239624, + "tpp_threshold_100_unintended_diff_only": 0.12550002336502075, + "tpp_threshold_500_total_metric": 0.18950004875659943, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.2865000218153 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.008750006556510925, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": -0.004749998450279236, - "tpp_threshold_5_total_metric": 0.011000022292137146, - "tpp_threshold_5_intended_diff_only": 0.008000016212463379, - "tpp_threshold_5_unintended_diff_only": -0.003000006079673767, - "tpp_threshold_10_total_metric": 0.0027499645948410034, - "tpp_threshold_10_intended_diff_only": 0.006999969482421875, - "tpp_threshold_10_unintended_diff_only": 0.004250004887580872 + "tpp_threshold_2_total_metric": 0.0029999762773513794, + "tpp_threshold_2_intended_diff_only": 0.006999969482421875, + "tpp_threshold_2_unintended_diff_only": 0.003999993205070496, + "tpp_threshold_5_total_metric": 0.004999995231628418, + "tpp_threshold_5_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_5_unintended_diff_only": 0.0009999871253967285, + "tpp_threshold_10_total_metric": 0.003999993205070496, + "tpp_threshold_10_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_10_unintended_diff_only": 0.001999989151954651, + "tpp_threshold_20_total_metric": 0.01875002682209015, + "tpp_threshold_20_intended_diff_only": 0.021000027656555176, + "tpp_threshold_20_unintended_diff_only": 0.002250000834465027, + "tpp_threshold_50_total_metric": 0.07500000298023224, + "tpp_threshold_50_intended_diff_only": 0.07400000095367432, + "tpp_threshold_50_unintended_diff_only": -0.0010000020265579224, + "tpp_threshold_100_total_metric": 0.12475001811981201, + "tpp_threshold_100_intended_diff_only": 0.13200002908706665, + "tpp_threshold_100_unintended_diff_only": 0.007250010967254639, + "tpp_threshold_500_total_metric": 0.3970000296831131, + "tpp_threshold_500_intended_diff_only": 0.40400004386901855, + "tpp_threshold_500_unintended_diff_only": 0.0070000141859054565 }, "2": { - "tpp_threshold_2_total_metric": 0.00025004148483276367, - "tpp_threshold_2_intended_diff_only": 0.006000041961669922, - "tpp_threshold_2_unintended_diff_only": 0.005750000476837158, - "tpp_threshold_5_total_metric": 0.009500056505203247, - "tpp_threshold_5_intended_diff_only": 0.016000032424926758, - "tpp_threshold_5_unintended_diff_only": 0.006499975919723511, - "tpp_threshold_10_total_metric": 0.02575002610683441, - "tpp_threshold_10_intended_diff_only": 0.026000022888183594, - "tpp_threshold_10_unintended_diff_only": 0.00024999678134918213 + "tpp_threshold_2_total_metric": 0.010499998927116394, + "tpp_threshold_2_intended_diff_only": 0.009000003337860107, + "tpp_threshold_2_unintended_diff_only": -0.0014999955892562866, + "tpp_threshold_5_total_metric": 0.0075000375509262085, + "tpp_threshold_5_intended_diff_only": 0.013000011444091797, + "tpp_threshold_5_unintended_diff_only": 0.005499973893165588, + "tpp_threshold_10_total_metric": 0.02525004744529724, + "tpp_threshold_10_intended_diff_only": 0.029000043869018555, + "tpp_threshold_10_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_20_total_metric": 0.0752500593662262, + "tpp_threshold_20_intended_diff_only": 0.07900005578994751, + "tpp_threshold_20_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_50_total_metric": 0.12075003981590271, + "tpp_threshold_50_intended_diff_only": 0.13100004196166992, + "tpp_threshold_50_unintended_diff_only": 0.010250002145767212, + "tpp_threshold_100_total_metric": 0.29475006461143494, + "tpp_threshold_100_intended_diff_only": 0.3190000653266907, + "tpp_threshold_100_unintended_diff_only": 0.024250000715255737, + "tpp_threshold_500_total_metric": 0.39900006353855133, + "tpp_threshold_500_intended_diff_only": 0.43500006198883057, + "tpp_threshold_500_unintended_diff_only": 0.035999998450279236 }, "3": { - "tpp_threshold_2_total_metric": 0.01299998164176941, - "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_2_unintended_diff_only": -0.006999999284744263, - "tpp_threshold_5_total_metric": 0.015250012278556824, - "tpp_threshold_5_intended_diff_only": 0.013999998569488525, - "tpp_threshold_5_unintended_diff_only": -0.0012500137090682983, - "tpp_threshold_10_total_metric": 0.015000015497207642, - "tpp_threshold_10_intended_diff_only": 0.017000019550323486, - "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447 + "tpp_threshold_2_total_metric": 0.004999995231628418, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 0.0, + "tpp_threshold_5_total_metric": 0.007750034332275391, + "tpp_threshold_5_intended_diff_only": 0.003000020980834961, + "tpp_threshold_5_unintended_diff_only": -0.00475001335144043, + "tpp_threshold_10_total_metric": 0.015250027179718018, + "tpp_threshold_10_intended_diff_only": 0.013000011444091797, + "tpp_threshold_10_unintended_diff_only": -0.0022500157356262207, + "tpp_threshold_20_total_metric": 0.015250012278556824, + "tpp_threshold_20_intended_diff_only": 0.018999993801116943, + "tpp_threshold_20_unintended_diff_only": 0.0037499815225601196, + "tpp_threshold_50_total_metric": 0.047250017523765564, + "tpp_threshold_50_intended_diff_only": 0.04500001668930054, + "tpp_threshold_50_unintended_diff_only": -0.002250000834465027, + "tpp_threshold_100_total_metric": 0.10475002229213715, + "tpp_threshold_100_intended_diff_only": 0.10900002717971802, + "tpp_threshold_100_unintended_diff_only": 0.004250004887580872, + "tpp_threshold_500_total_metric": 0.3367500305175781, + "tpp_threshold_500_intended_diff_only": 0.3500000238418579, + "tpp_threshold_500_unintended_diff_only": 0.013249993324279785 }, "5": { - "tpp_threshold_2_total_metric": -0.0077500492334365845, - "tpp_threshold_2_intended_diff_only": -0.001000046730041504, - "tpp_threshold_2_unintended_diff_only": 0.006750002503395081, - "tpp_threshold_5_total_metric": -0.0025000423192977905, - "tpp_threshold_5_intended_diff_only": -0.001000046730041504, - "tpp_threshold_5_unintended_diff_only": 0.0014999955892562866, - "tpp_threshold_10_total_metric": 0.007249966263771057, - "tpp_threshold_10_intended_diff_only": 0.010999977588653564, - "tpp_threshold_10_unintended_diff_only": 0.0037500113248825073 + "tpp_threshold_2_total_metric": 0.00024999678134918213, + "tpp_threshold_2_intended_diff_only": 0.0, + "tpp_threshold_2_unintended_diff_only": -0.00024999678134918213, + "tpp_threshold_5_total_metric": 0.008499950170516968, + "tpp_threshold_5_intended_diff_only": 0.007999956607818604, + "tpp_threshold_5_unintended_diff_only": -0.0004999935626983643, + "tpp_threshold_10_total_metric": 0.0037499964237213135, + "tpp_threshold_10_intended_diff_only": 0.009999990463256836, + "tpp_threshold_10_unintended_diff_only": 0.0062499940395355225, + "tpp_threshold_20_total_metric": 0.019999966025352478, + "tpp_threshold_20_intended_diff_only": 0.0209999680519104, + "tpp_threshold_20_unintended_diff_only": 0.0010000020265579224, + "tpp_threshold_50_total_metric": 0.10199996829032898, + "tpp_threshold_50_intended_diff_only": 0.10799998044967651, + "tpp_threshold_50_unintended_diff_only": 0.006000012159347534, + "tpp_threshold_100_total_metric": 0.20149998366832733, + "tpp_threshold_100_intended_diff_only": 0.2459999918937683, + "tpp_threshold_100_unintended_diff_only": 0.04450000822544098, + "tpp_threshold_500_total_metric": 0.35074998438358307, + "tpp_threshold_500_intended_diff_only": 0.4129999876022339, + "tpp_threshold_500_unintended_diff_only": 0.06225000321865082 }, "6": { - "tpp_threshold_2_total_metric": 0.026249989867210388, - "tpp_threshold_2_intended_diff_only": 0.02499997615814209, - "tpp_threshold_2_unintended_diff_only": -0.0012500137090682983, - "tpp_threshold_5_total_metric": 0.031000003218650818, - "tpp_threshold_5_intended_diff_only": 0.03700000047683716, - "tpp_threshold_5_unintended_diff_only": 0.00599999725818634, - "tpp_threshold_10_total_metric": 0.056499987840652466, - "tpp_threshold_10_intended_diff_only": 0.05799996852874756, - "tpp_threshold_10_unintended_diff_only": 0.0014999806880950928 + "tpp_threshold_2_total_metric": 0.014749988913536072, + "tpp_threshold_2_intended_diff_only": 0.018999993801116943, + "tpp_threshold_2_unintended_diff_only": 0.004250004887580872, + "tpp_threshold_5_total_metric": 0.03300000727176666, + "tpp_threshold_5_intended_diff_only": 0.03200000524520874, + "tpp_threshold_5_unintended_diff_only": -0.0010000020265579224, + "tpp_threshold_10_total_metric": 0.06975001096725464, + "tpp_threshold_10_intended_diff_only": 0.07300001382827759, + "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492, + "tpp_threshold_20_total_metric": 0.12074996531009674, + "tpp_threshold_20_intended_diff_only": 0.13099998235702515, + "tpp_threshold_20_unintended_diff_only": 0.010250017046928406, + "tpp_threshold_50_total_metric": 0.20399998128414154, + "tpp_threshold_50_intended_diff_only": 0.2149999737739563, + "tpp_threshold_50_unintended_diff_only": 0.010999992489814758, + "tpp_threshold_100_total_metric": 0.281000018119812, + "tpp_threshold_100_intended_diff_only": 0.29100000858306885, + "tpp_threshold_100_unintended_diff_only": 0.009999990463256836, + "tpp_threshold_500_total_metric": 0.3437500149011612, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.02925001084804535 } } } diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-97656-tokens-400000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-97656-tokens-400000000/scr/saebench_scr_custom_sae_eval_results.json index 60f857ff052640c4287f566a183f3ae173d2714a..ba8749db0911a2db073bcf5c150705a82f98a762 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-97656-tokens-400000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-97656-tokens-400000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "6effae0b-edbd-4b6e-87e0-22713a45acca", - "datetime_epoch_millis": 1776887709102, + "eval_id": "dffa54ee-eada-4a87-9f4b-5303122b6852", + "datetime_epoch_millis": 1777318207079, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.2998913967182217, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.24770712302484355, "scr_metric_threshold_10": 0.24589716877147033, "scr_dir2_threshold_10": 0.23154907044243342, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.07599601986418246, + "scr_metric_threshold_20": 0.2910799065884113, + "scr_dir2_threshold_20": 0.27253578100554343, + "scr_dir1_threshold_50": 0.25203918374420137, + "scr_metric_threshold_50": 0.35282294166443295, + "scr_dir2_threshold_50": 0.32778873751690984, + "scr_dir1_threshold_100": 0.19898691623141082, + "scr_metric_threshold_100": 0.3808640415954885, + "scr_dir2_threshold_100": 0.3481454132792711, + "scr_dir1_threshold_500": -0.5184301925444443, + "scr_metric_threshold_500": 0.23800222434871088, + "scr_dir2_threshold_500": 0.21375199954915639 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.3333335880540079, "scr_metric_threshold_10": 0.08549219997634344, "scr_dir2_threshold_10": 0.08549219997634344, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.34615393432638736, + "scr_metric_threshold_20": 0.16580312000946262, + "scr_dir2_threshold_20": 0.16580312000946262, + "scr_dir1_threshold_50": 0.5512825313325533, + "scr_metric_threshold_50": 0.222797971465746, + "scr_dir2_threshold_50": 0.222797971465746, + "scr_dir1_threshold_100": 0.47435892537422925, + "scr_metric_threshold_100": 0.3005180971110898, + "scr_dir2_threshold_100": 0.3005180971110898, + "scr_dir1_threshold_500": -2.2948710209128222, + "scr_metric_threshold_500": 0.11658034288417893, + "scr_dir2_threshold_500": 0.11658034288417893 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.2878790820706107, "scr_metric_threshold_10": 0.2538226260674608, "scr_dir2_threshold_10": 0.2538226260674608, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.12121188859533107, + "scr_metric_threshold_20": 0.25076456166892996, + "scr_dir2_threshold_20": 0.25076456166892996, + "scr_dir1_threshold_50": -0.03030308503639269, + "scr_metric_threshold_50": 0.44954129945407983, + "scr_dir2_threshold_50": 0.44954129945407983, + "scr_dir1_threshold_100": 0.10606057185225456, + "scr_metric_threshold_100": 0.6177370286995434, + "scr_dir2_threshold_100": 0.6177370286995434, + "scr_dir1_threshold_500": 0.3181817155567637, + "scr_metric_threshold_500": 0.2293577463248362, + "scr_dir2_threshold_500": 0.2293577463248362 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.28125005820765137, "scr_metric_threshold_10": 0.12311562116192198, "scr_dir2_threshold_10": 0.12311562116192198, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.24999953433878902, + "scr_metric_threshold_20": 0.21356790203513887, + "scr_dir2_threshold_20": 0.21356790203513887, + "scr_dir1_threshold_50": 0.4843752037267798, + "scr_metric_threshold_50": 0.23366829241554973, + "scr_dir2_threshold_50": 0.23366829241554973, + "scr_dir1_threshold_100": -0.14062502910382568, + "scr_metric_threshold_100": 0.08291454088031854, + "scr_dir2_threshold_100": 0.08291454088031854, + "scr_dir1_threshold_500": -1.6249993015081836, + "scr_metric_threshold_500": -0.0929648109507194, + "scr_dir2_threshold_500": -0.0929648109507194 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": -0.04698006444502671, "scr_metric_threshold_10": 0.24203831933130424, "scr_dir2_threshold_10": 0.24203831933130424, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.7181212134546494, + "scr_metric_threshold_20": 0.19108280375683886, + "scr_dir2_threshold_20": 0.19108280375683886, + "scr_dir1_threshold_50": -0.2885909100909871, + "scr_metric_threshold_50": 0.27070074938601524, + "scr_dir2_threshold_50": 0.27070074938601524, + "scr_dir1_threshold_100": -0.322147927538348, + "scr_metric_threshold_100": 0.14012747800607664, + "scr_dir2_threshold_100": 0.14012747800607664, + "scr_dir1_threshold_500": -1.6107384375981326, + "scr_metric_threshold_500": -0.4745222422127673, + "scr_dir2_threshold_500": -0.4745222422127673 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.08947378822971379, "scr_metric_threshold_10": 0.4920634770459469, "scr_dir2_threshold_10": 0.4920634770459469, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.12631573333634494, + "scr_metric_threshold_20": 0.5515872218065929, + "scr_dir2_threshold_20": 0.5515872218065929, + "scr_dir1_threshold_50": -0.14736840784384586, + "scr_metric_threshold_50": 0.6190476077844602, + "scr_dir2_threshold_50": 0.6190476077844602, + "scr_dir1_threshold_100": -0.17894726275076694, + "scr_metric_threshold_100": 0.6904761961077699, + "scr_dir2_threshold_100": 0.6904761961077699, + "scr_dir1_threshold_500": -0.210526117657688, + "scr_metric_threshold_500": 0.6944443984532124, + "scr_dir2_threshold_500": 0.6944443984532124 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.371584539269045, "scr_metric_threshold_10": 0.12213741326330622, "scr_dir2_threshold_10": 0.12213741326330622, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.4480872768489657, + "scr_metric_threshold_20": 0.1603053122520976, + "scr_dir2_threshold_20": 0.1603053122520976, + "scr_dir1_threshold_50": 0.502732368584764, + "scr_metric_threshold_50": 0.22519087703221677, + "scr_dir2_threshold_50": 0.22519087703221677, + "scr_dir1_threshold_100": 0.5136611915068252, + "scr_metric_threshold_100": 0.29007644181233594, + "scr_dir2_threshold_100": 0.29007644181233594, + "scr_dir1_threshold_500": 0.6010930777173046, + "scr_metric_threshold_500": 0.45419849846358806, + "scr_dir2_threshold_500": 0.45419849846358806 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.4372091501709683, "scr_metric_threshold_10": 0.4206008506837011, "scr_dir2_threshold_10": 0.4206008506837011, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.47906962431336847, + "scr_metric_threshold_20": 0.49785410470106733, + "scr_dir2_threshold_20": 0.49785410470106733, + "scr_dir1_threshold_50": 0.5674416747849105, + "scr_metric_threshold_50": 0.4248926412815665, + "scr_dir2_threshold_50": 0.4248926412815665, + "scr_dir1_threshold_100": 0.6651162068607985, + "scr_metric_threshold_100": 0.4506438964966536, + "scr_dir2_threshold_100": 0.4506438964966536, + "scr_dir1_threshold_500": 0.19534878692091323, + "scr_metric_threshold_500": 0.49785410470106733, + "scr_dir2_threshold_500": 0.49785410470106733 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.227906842641778, "scr_metric_threshold_10": 0.227906842641778, "scr_dir2_threshold_10": 0.1131220560094828, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2976742264771627, + "scr_metric_threshold_20": 0.2976742264771627, + "scr_dir2_threshold_20": 0.1493212218142196, + "scr_dir1_threshold_50": 0.3767440944958292, + "scr_metric_threshold_50": 0.3767440944958292, + "scr_dir2_threshold_50": 0.17647046131564434, + "scr_dir1_threshold_100": 0.4744186536501196, + "scr_metric_threshold_100": 0.4744186536501196, + "scr_dir2_threshold_100": 0.21266962712038115, + "scr_dir1_threshold_500": 0.47906975712629063, + "scr_metric_threshold_500": 0.47906975712629063, + "scr_dir2_threshold_500": 0.2850679587298548 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-97656-tokens-400000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-97656-tokens-400000000/tpp/saebench_tpp_custom_sae_eval_results.json index 0ab4479df648d01b3fa6aefc643eb0b332df0f43..bf30f4f0c9fd0b5714d0018ad352bd59e6272441 100644 --- a/gemma-2-2b-btk/k-100/seed-0/snapshots/step-97656-tokens-400000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-100/seed-0/snapshots/step-97656-tokens-400000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "d031b888-66db-4259-b982-2d23aa4394ff", - "datetime_epoch_millis": 1776887860742, + "eval_id": "201fe9d9-0666-4159-8614-0c77ad95bb48", + "datetime_epoch_millis": 1777318481219, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.023950007557868955, - "tpp_threshold_2_intended_diff_only": 0.0252000093460083, - "tpp_threshold_2_unintended_diff_only": 0.0012500017881393432, - "tpp_threshold_5_total_metric": 0.05390000492334366, - "tpp_threshold_5_intended_diff_only": 0.06120001077651978, - "tpp_threshold_5_unintended_diff_only": 0.007300005853176117, - "tpp_threshold_10_total_metric": 0.09737499803304672, - "tpp_threshold_10_intended_diff_only": 0.12040000557899476, - "tpp_threshold_10_unintended_diff_only": 0.023025007545948026, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.022350001335144042, + "tpp_threshold_2_intended_diff_only": 0.025000005960464478, + "tpp_threshold_2_unintended_diff_only": 0.0026500046253204344, + "tpp_threshold_5_total_metric": 0.05437501072883606, + "tpp_threshold_5_intended_diff_only": 0.060700017213821414, + "tpp_threshold_5_unintended_diff_only": 0.0063250064849853516, + "tpp_threshold_10_total_metric": 0.09972499907016753, + "tpp_threshold_10_intended_diff_only": 0.1228000044822693, + "tpp_threshold_10_unintended_diff_only": 0.023075005412101748, + "tpp_threshold_20_total_metric": 0.13872499465942384, + "tpp_threshold_20_intended_diff_only": 0.17320000529289245, + "tpp_threshold_20_unintended_diff_only": 0.03447501063346863, + "tpp_threshold_50_total_metric": 0.18587500751018524, + "tpp_threshold_50_intended_diff_only": 0.24930001497268675, + "tpp_threshold_50_unintended_diff_only": 0.06342500746250153, + "tpp_threshold_100_total_metric": 0.2121750146150589, + "tpp_threshold_100_intended_diff_only": 0.31560002565383916, + "tpp_threshold_100_unintended_diff_only": 0.10342501103878021, + "tpp_threshold_500_total_metric": 0.2530000120401382, + "tpp_threshold_500_intended_diff_only": 0.43230003118515015, + "tpp_threshold_500_unintended_diff_only": 0.1793000191450119 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.0385000079870224, - "tpp_threshold_2_intended_diff_only": 0.040600013732910153, - "tpp_threshold_2_unintended_diff_only": 0.0021000057458877563, - "tpp_threshold_5_total_metric": 0.09150001704692841, - "tpp_threshold_5_intended_diff_only": 0.1036000370979309, - "tpp_threshold_5_unintended_diff_only": 0.012100020051002502, - "tpp_threshold_10_total_metric": 0.17200001180171967, - "tpp_threshold_10_intended_diff_only": 0.2138000249862671, - "tpp_threshold_10_unintended_diff_only": 0.04180001318454742, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.03770000338554382, + "tpp_threshold_2_intended_diff_only": 0.041400015354156494, + "tpp_threshold_2_unintended_diff_only": 0.003700011968612671, + "tpp_threshold_5_total_metric": 0.09260002076625824, + "tpp_threshold_5_intended_diff_only": 0.10480003356933594, + "tpp_threshold_5_unintended_diff_only": 0.012200012803077698, + "tpp_threshold_10_total_metric": 0.17240000069141387, + "tpp_threshold_10_intended_diff_only": 0.2136000156402588, + "tpp_threshold_10_unintended_diff_only": 0.04120001494884491, + "tpp_threshold_20_total_metric": 0.23765000700950623, + "tpp_threshold_20_intended_diff_only": 0.3006000280380249, + "tpp_threshold_20_unintended_diff_only": 0.06295002102851868, + "tpp_threshold_50_total_metric": 0.27325000464916227, + "tpp_threshold_50_intended_diff_only": 0.3918000221252441, + "tpp_threshold_50_unintended_diff_only": 0.11855001747608185, + "tpp_threshold_100_total_metric": 0.24900001287460327, + "tpp_threshold_100_intended_diff_only": 0.43540003299713137, + "tpp_threshold_100_unintended_diff_only": 0.18640002012252807, + "tpp_threshold_500_total_metric": 0.14020000994205475, + "tpp_threshold_500_intended_diff_only": 0.46680004596710206, + "tpp_threshold_500_unintended_diff_only": 0.3266000360250473 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.009400007128715516, - "tpp_threshold_2_intended_diff_only": 0.009800004959106445, - "tpp_threshold_2_unintended_diff_only": 0.0003999978303909302, - "tpp_threshold_5_total_metric": 0.01629999279975891, - "tpp_threshold_5_intended_diff_only": 0.018799984455108644, - "tpp_threshold_5_unintended_diff_only": 0.0024999916553497315, - "tpp_threshold_10_total_metric": 0.02274998426437378, - "tpp_threshold_10_intended_diff_only": 0.02699998617172241, - "tpp_threshold_10_unintended_diff_only": 0.004250001907348633, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.006999999284744263, + "tpp_threshold_2_intended_diff_only": 0.008599996566772461, + "tpp_threshold_2_unintended_diff_only": 0.0015999972820281982, + "tpp_threshold_5_total_metric": 0.01615000069141388, + "tpp_threshold_5_intended_diff_only": 0.016600000858306884, + "tpp_threshold_5_unintended_diff_only": 0.00045000016689300537, + "tpp_threshold_10_total_metric": 0.027049997448921205, + "tpp_threshold_10_intended_diff_only": 0.03199999332427979, + "tpp_threshold_10_unintended_diff_only": 0.004949995875358581, + "tpp_threshold_20_total_metric": 0.03979998230934143, + "tpp_threshold_20_intended_diff_only": 0.04579998254776001, + "tpp_threshold_20_unintended_diff_only": 0.006000000238418579, + "tpp_threshold_50_total_metric": 0.09850001037120819, + "tpp_threshold_50_intended_diff_only": 0.1068000078201294, + "tpp_threshold_50_unintended_diff_only": 0.008299997448921204, + "tpp_threshold_100_total_metric": 0.17535001635551453, + "tpp_threshold_100_intended_diff_only": 0.19580001831054689, + "tpp_threshold_100_unintended_diff_only": 0.020450001955032347, + "tpp_threshold_500_total_metric": 0.3658000141382217, + "tpp_threshold_500_intended_diff_only": 0.39780001640319823, + "tpp_threshold_500_unintended_diff_only": 0.0320000022649765 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.0015000253915786743, "tpp_threshold_10_total_metric": 0.003749936819076538, "tpp_threshold_10_intended_diff_only": 0.006999969482421875, - "tpp_threshold_10_unintended_diff_only": 0.003250032663345337 + "tpp_threshold_10_unintended_diff_only": 0.003250032663345337, + "tpp_threshold_20_total_metric": 0.21599997580051422, + "tpp_threshold_20_intended_diff_only": 0.2540000081062317, + "tpp_threshold_20_unintended_diff_only": 0.03800003230571747, + "tpp_threshold_50_total_metric": 0.2774999439716339, + "tpp_threshold_50_intended_diff_only": 0.3539999723434448, + "tpp_threshold_50_unintended_diff_only": 0.07650002837181091, + "tpp_threshold_100_total_metric": 0.31074997782707214, + "tpp_threshold_100_intended_diff_only": 0.4259999990463257, + "tpp_threshold_100_unintended_diff_only": 0.11525002121925354, + "tpp_threshold_500_total_metric": 0.16324999928474426, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.2927500307559967 }, "1": { - "tpp_threshold_2_total_metric": -1.4901161193847656e-08, - "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_5_total_metric": 0.14275000989437103, + "tpp_threshold_2_total_metric": 0.004749983549118042, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 0.000250011682510376, + "tpp_threshold_5_total_metric": 0.14150004088878632, "tpp_threshold_5_intended_diff_only": 0.15900003910064697, - "tpp_threshold_5_unintended_diff_only": 0.01625002920627594, - "tpp_threshold_10_total_metric": 0.20299997925758362, + "tpp_threshold_5_unintended_diff_only": 0.017499998211860657, + "tpp_threshold_10_total_metric": 0.20349998772144318, "tpp_threshold_10_intended_diff_only": 0.25, - "tpp_threshold_10_unintended_diff_only": 0.04700002074241638 + "tpp_threshold_10_unintended_diff_only": 0.046500012278556824, + "tpp_threshold_20_total_metric": 0.23199999332427979, + "tpp_threshold_20_intended_diff_only": 0.28200000524520874, + "tpp_threshold_20_unintended_diff_only": 0.050000011920928955, + "tpp_threshold_50_total_metric": 0.2659999877214432, + "tpp_threshold_50_intended_diff_only": 0.3659999966621399, + "tpp_threshold_50_unintended_diff_only": 0.10000000894069672, + "tpp_threshold_100_total_metric": 0.24424998462200165, + "tpp_threshold_100_intended_diff_only": 0.41200000047683716, + "tpp_threshold_100_unintended_diff_only": 0.1677500158548355, + "tpp_threshold_500_total_metric": 0.13100001215934753, + "tpp_threshold_500_intended_diff_only": 0.4590000510215759, + "tpp_threshold_500_unintended_diff_only": 0.3280000388622284 }, "2": { - "tpp_threshold_2_total_metric": 0.17125006020069122, - "tpp_threshold_2_intended_diff_only": 0.18100005388259888, - "tpp_threshold_2_unintended_diff_only": 0.009749993681907654, - "tpp_threshold_5_total_metric": 0.2537500411272049, - "tpp_threshold_5_intended_diff_only": 0.28300005197525024, - "tpp_threshold_5_unintended_diff_only": 0.02925001084804535, - "tpp_threshold_10_total_metric": 0.24375005066394806, - "tpp_threshold_10_intended_diff_only": 0.3340000510215759, - "tpp_threshold_10_unintended_diff_only": 0.09025000035762787 + "tpp_threshold_2_total_metric": 0.1615000218153, + "tpp_threshold_2_intended_diff_only": 0.17800003290176392, + "tpp_threshold_2_unintended_diff_only": 0.016500011086463928, + "tpp_threshold_5_total_metric": 0.2507500499486923, + "tpp_threshold_5_intended_diff_only": 0.2820000648498535, + "tpp_threshold_5_unintended_diff_only": 0.031250014901161194, + "tpp_threshold_10_total_metric": 0.24525001645088196, + "tpp_threshold_10_intended_diff_only": 0.33500003814697266, + "tpp_threshold_10_unintended_diff_only": 0.0897500216960907, + "tpp_threshold_20_total_metric": 0.23250000178813934, + "tpp_threshold_20_intended_diff_only": 0.36400002241134644, + "tpp_threshold_20_unintended_diff_only": 0.1315000206232071, + "tpp_threshold_50_total_metric": 0.20775002241134644, + "tpp_threshold_50_intended_diff_only": 0.39900004863739014, + "tpp_threshold_50_unintended_diff_only": 0.1912500262260437, + "tpp_threshold_100_total_metric": 0.14250002801418304, + "tpp_threshold_100_intended_diff_only": 0.42200005054473877, + "tpp_threshold_100_unintended_diff_only": 0.2795000225305557, + "tpp_threshold_500_total_metric": 0.06624998152256012, + "tpp_threshold_500_intended_diff_only": 0.4520000219345093, + "tpp_threshold_500_unintended_diff_only": 0.38575004041194916 }, "6": { - "tpp_threshold_2_total_metric": 0.011500000953674316, + "tpp_threshold_2_total_metric": 0.009499981999397278, "tpp_threshold_2_intended_diff_only": 0.009000003337860107, - "tpp_threshold_2_unintended_diff_only": -0.002499997615814209, - "tpp_threshold_5_total_metric": 0.04075004160404205, - "tpp_threshold_5_intended_diff_only": 0.05200004577636719, - "tpp_threshold_5_unintended_diff_only": 0.011250004172325134, - "tpp_threshold_10_total_metric": 0.25400005280971527, + "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704, + "tpp_threshold_5_total_metric": 0.045500025153160095, + "tpp_threshold_5_intended_diff_only": 0.053000032901763916, + "tpp_threshold_5_unintended_diff_only": 0.007500007748603821, + "tpp_threshold_10_total_metric": 0.25700004398822784, "tpp_threshold_10_intended_diff_only": 0.29200005531311035, - "tpp_threshold_10_unintended_diff_only": 0.03800000250339508 + "tpp_threshold_10_unintended_diff_only": 0.03500001132488251, + "tpp_threshold_20_total_metric": 0.3035000413656235, + "tpp_threshold_20_intended_diff_only": 0.37000006437301636, + "tpp_threshold_20_unintended_diff_only": 0.06650002300739288, + "tpp_threshold_50_total_metric": 0.3230000287294388, + "tpp_threshold_50_intended_diff_only": 0.4740000367164612, + "tpp_threshold_50_unintended_diff_only": 0.1510000079870224, + "tpp_threshold_100_total_metric": 0.25550004839897156, + "tpp_threshold_100_intended_diff_only": 0.48100006580352783, + "tpp_threshold_100_unintended_diff_only": 0.22550001740455627, + "tpp_threshold_500_total_metric": 0.1757500320672989, + "tpp_threshold_500_intended_diff_only": 0.49100005626678467, + "tpp_threshold_500_unintended_diff_only": 0.3152500241994858 }, "9": { - "tpp_threshold_2_total_metric": 0.0022500157356262207, - "tpp_threshold_2_intended_diff_only": 0.003000020980834961, - "tpp_threshold_2_unintended_diff_only": 0.0007500052452087402, - "tpp_threshold_5_total_metric": 0.007750019431114197, - "tpp_threshold_5_intended_diff_only": 0.010000050067901611, - "tpp_threshold_5_unintended_diff_only": 0.0022500306367874146, - "tpp_threshold_10_total_metric": 0.15550003945827484, - "tpp_threshold_10_intended_diff_only": 0.1860000491142273, - "tpp_threshold_10_unintended_diff_only": 0.030500009655952454 + "tpp_threshold_2_total_metric": 0.0052500516176223755, + "tpp_threshold_2_intended_diff_only": 0.006000041961669922, + "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.012750014662742615, + "tpp_threshold_5_intended_diff_only": 0.016000032424926758, + "tpp_threshold_5_unintended_diff_only": 0.003250017762184143, + "tpp_threshold_10_total_metric": 0.15250001847743988, + "tpp_threshold_10_intended_diff_only": 0.18400001525878906, + "tpp_threshold_10_unintended_diff_only": 0.03149999678134918, + "tpp_threshold_20_total_metric": 0.2042500227689743, + "tpp_threshold_20_intended_diff_only": 0.2330000400543213, + "tpp_threshold_20_unintended_diff_only": 0.028750017285346985, + "tpp_threshold_50_total_metric": 0.29200004041194916, + "tpp_threshold_50_intended_diff_only": 0.36600005626678467, + "tpp_threshold_50_unintended_diff_only": 0.07400001585483551, + "tpp_threshold_100_total_metric": 0.29200002551078796, + "tpp_threshold_100_intended_diff_only": 0.4360000491142273, + "tpp_threshold_100_unintended_diff_only": 0.14400002360343933, + "tpp_threshold_500_total_metric": 0.16475002467632294, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.3112500458955765 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.009749993681907654, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": -0.004749998450279236, - "tpp_threshold_5_total_metric": 0.012499973177909851, - "tpp_threshold_5_intended_diff_only": 0.010999977588653564, - "tpp_threshold_5_unintended_diff_only": -0.0014999955892562866, - "tpp_threshold_10_total_metric": 0.0009999871253967285, + "tpp_threshold_2_total_metric": 0.002249985933303833, + "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_5_total_metric": 0.005499973893165588, + "tpp_threshold_5_intended_diff_only": 0.006999969482421875, + "tpp_threshold_5_unintended_diff_only": 0.0014999955892562866, + "tpp_threshold_10_total_metric": 0.0029999911785125732, "tpp_threshold_10_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_10_unintended_diff_only": 0.004999995231628418 + "tpp_threshold_10_unintended_diff_only": 0.0029999911785125732, + "tpp_threshold_20_total_metric": 0.011749982833862305, + "tpp_threshold_20_intended_diff_only": 0.014999985694885254, + "tpp_threshold_20_unintended_diff_only": 0.0032500028610229492, + "tpp_threshold_50_total_metric": 0.06650000810623169, + "tpp_threshold_50_intended_diff_only": 0.06499999761581421, + "tpp_threshold_50_unintended_diff_only": -0.0015000104904174805, + "tpp_threshold_100_total_metric": 0.12775000929832458, + "tpp_threshold_100_intended_diff_only": 0.13300001621246338, + "tpp_threshold_100_unintended_diff_only": 0.005250006914138794, + "tpp_threshold_500_total_metric": 0.39875003695487976, + "tpp_threshold_500_intended_diff_only": 0.409000039100647, + "tpp_threshold_500_unintended_diff_only": 0.010250002145767212 }, "2": { - "tpp_threshold_2_total_metric": 0.00025004148483276367, - "tpp_threshold_2_intended_diff_only": 0.006000041961669922, - "tpp_threshold_2_unintended_diff_only": 0.005750000476837158, - "tpp_threshold_5_total_metric": 0.008250027894973755, - "tpp_threshold_5_intended_diff_only": 0.013999998569488525, - "tpp_threshold_5_unintended_diff_only": 0.0057499706745147705, - "tpp_threshold_10_total_metric": 0.01950003206729889, - "tpp_threshold_10_intended_diff_only": 0.030000030994415283, - "tpp_threshold_10_unintended_diff_only": 0.010499998927116394 + "tpp_threshold_2_total_metric": 0.0102500319480896, + "tpp_threshold_2_intended_diff_only": 0.008000016212463379, + "tpp_threshold_2_unintended_diff_only": -0.0022500157356262207, + "tpp_threshold_5_total_metric": 0.00575004518032074, + "tpp_threshold_5_intended_diff_only": 0.01100003719329834, + "tpp_threshold_5_unintended_diff_only": 0.0052499920129776, + "tpp_threshold_10_total_metric": 0.03275001049041748, + "tpp_threshold_10_intended_diff_only": 0.046000003814697266, + "tpp_threshold_10_unintended_diff_only": 0.013249993324279785, + "tpp_threshold_20_total_metric": 0.04975001513957977, + "tpp_threshold_20_intended_diff_only": 0.05900001525878906, + "tpp_threshold_20_unintended_diff_only": 0.00925000011920929, + "tpp_threshold_50_total_metric": 0.10050001740455627, + "tpp_threshold_50_intended_diff_only": 0.12000000476837158, + "tpp_threshold_50_unintended_diff_only": 0.019499987363815308, + "tpp_threshold_100_total_metric": 0.2080000638961792, + "tpp_threshold_100_intended_diff_only": 0.24100005626678467, + "tpp_threshold_100_unintended_diff_only": 0.03299999237060547, + "tpp_threshold_500_total_metric": 0.4035000652074814, + "tpp_threshold_500_intended_diff_only": 0.4360000491142273, + "tpp_threshold_500_unintended_diff_only": 0.03249998390674591 }, "3": { - "tpp_threshold_2_total_metric": 0.013500049710273743, - "tpp_threshold_2_intended_diff_only": 0.00700002908706665, - "tpp_threshold_2_unintended_diff_only": -0.006500020623207092, - "tpp_threshold_5_total_metric": 0.017249971628189087, - "tpp_threshold_5_intended_diff_only": 0.015999972820281982, - "tpp_threshold_5_unintended_diff_only": -0.0012499988079071045, - "tpp_threshold_10_total_metric": 0.01899997889995575, - "tpp_threshold_10_intended_diff_only": 0.019999980926513672, - "tpp_threshold_10_unintended_diff_only": 0.0010000020265579224 + "tpp_threshold_2_total_metric": 0.004999995231628418, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 0.0, + "tpp_threshold_5_total_metric": 0.014250010251998901, + "tpp_threshold_5_intended_diff_only": 0.009000003337860107, + "tpp_threshold_5_unintended_diff_only": -0.005250006914138794, + "tpp_threshold_10_total_metric": 0.021749988198280334, + "tpp_threshold_10_intended_diff_only": 0.018999993801116943, + "tpp_threshold_10_unintended_diff_only": -0.002749994397163391, + "tpp_threshold_20_total_metric": 0.012249976396560669, + "tpp_threshold_20_intended_diff_only": 0.015999972820281982, + "tpp_threshold_20_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_50_total_metric": 0.06350001692771912, + "tpp_threshold_50_intended_diff_only": 0.06400001049041748, + "tpp_threshold_50_unintended_diff_only": 0.0004999935626983643, + "tpp_threshold_100_total_metric": 0.11650000512599945, + "tpp_threshold_100_intended_diff_only": 0.12000000476837158, + "tpp_threshold_100_unintended_diff_only": 0.0034999996423721313, + "tpp_threshold_500_total_metric": 0.34599998593330383, + "tpp_threshold_500_intended_diff_only": 0.3659999966621399, + "tpp_threshold_500_unintended_diff_only": 0.02000001072883606 }, "5": { - "tpp_threshold_2_total_metric": -0.008000046014785767, - "tpp_threshold_2_intended_diff_only": -0.001000046730041504, - "tpp_threshold_2_unintended_diff_only": 0.006999999284744263, - "tpp_threshold_5_total_metric": -0.0007500052452087402, - "tpp_threshold_5_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_5_unintended_diff_only": 0.0017499923706054688, - "tpp_threshold_10_total_metric": 0.0004999488592147827, - "tpp_threshold_10_intended_diff_only": 0.0029999613761901855, - "tpp_threshold_10_unintended_diff_only": 0.002500012516975403 + "tpp_threshold_2_total_metric": 0.0009999722242355347, + "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, + "tpp_threshold_2_unintended_diff_only": 1.4901161193847656e-08, + "tpp_threshold_5_total_metric": 0.006499975919723511, + "tpp_threshold_5_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_5_unintended_diff_only": -0.0004999935626983643, + "tpp_threshold_10_total_metric": -0.00475001335144043, + "tpp_threshold_10_intended_diff_only": 0.001999974250793457, + "tpp_threshold_10_unintended_diff_only": 0.006749987602233887, + "tpp_threshold_20_total_metric": 0.008249938488006592, + "tpp_threshold_20_intended_diff_only": 0.008999943733215332, + "tpp_threshold_20_unintended_diff_only": 0.0007500052452087402, + "tpp_threshold_50_total_metric": 0.06199999153614044, + "tpp_threshold_50_intended_diff_only": 0.0690000057220459, + "tpp_threshold_50_unintended_diff_only": 0.0070000141859054565, + "tpp_threshold_100_total_metric": 0.1537499874830246, + "tpp_threshold_100_intended_diff_only": 0.19900000095367432, + "tpp_threshold_100_unintended_diff_only": 0.04525001347064972, + "tpp_threshold_500_total_metric": 0.33949996531009674, + "tpp_threshold_500_intended_diff_only": 0.4049999713897705, + "tpp_threshold_500_unintended_diff_only": 0.06550000607967377 }, "6": { - "tpp_threshold_2_total_metric": 0.03149999678134918, - "tpp_threshold_2_intended_diff_only": 0.03200000524520874, - "tpp_threshold_2_unintended_diff_only": 0.0005000084638595581, - "tpp_threshold_5_total_metric": 0.0442499965429306, - "tpp_threshold_5_intended_diff_only": 0.05199998617172241, - "tpp_threshold_5_unintended_diff_only": 0.007749989628791809, - "tpp_threshold_10_total_metric": 0.07374997437000275, - "tpp_threshold_10_intended_diff_only": 0.07599997520446777, - "tpp_threshold_10_unintended_diff_only": 0.002250000834465027 + "tpp_threshold_2_total_metric": 0.016500011086463928, + "tpp_threshold_2_intended_diff_only": 0.023000001907348633, + "tpp_threshold_2_unintended_diff_only": 0.006499990820884705, + "tpp_threshold_5_total_metric": 0.04874999821186066, + "tpp_threshold_5_intended_diff_only": 0.050000011920928955, + "tpp_threshold_5_unintended_diff_only": 0.0012500137090682983, + "tpp_threshold_10_total_metric": 0.08250001072883606, + "tpp_threshold_10_intended_diff_only": 0.08700001239776611, + "tpp_threshold_10_unintended_diff_only": 0.004500001668930054, + "tpp_threshold_20_total_metric": 0.11699999868869781, + "tpp_threshold_20_intended_diff_only": 0.12999999523162842, + "tpp_threshold_20_unintended_diff_only": 0.012999996542930603, + "tpp_threshold_50_total_metric": 0.20000001788139343, + "tpp_threshold_50_intended_diff_only": 0.2160000205039978, + "tpp_threshold_50_unintended_diff_only": 0.01600000262260437, + "tpp_threshold_100_total_metric": 0.2707500159740448, + "tpp_threshold_100_intended_diff_only": 0.28600001335144043, + "tpp_threshold_100_unintended_diff_only": 0.01524999737739563, + "tpp_threshold_500_total_metric": 0.341250017285347, + "tpp_threshold_500_intended_diff_only": 0.37300002574920654, + "tpp_threshold_500_unintended_diff_only": 0.03175000846385956 } } } diff --git a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-1-tokens-0/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-1-tokens-0/scr/saebench_scr_custom_sae_eval_results.json index 9202cacfd9343a5f6fdb8476dd2cc62ea0c37751..c300d092baed287a5e19405026de72f390150b2e 100644 --- a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-1-tokens-0/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-1-tokens-0/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "98f78e53-f3e7-4bc4-9634-c9bb7b870698", - "datetime_epoch_millis": 1776887423878, + "eval_id": "6a908c3a-7ecc-490a-9975-f787bb480372", + "datetime_epoch_millis": 1777318563703, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.0, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": -0.0006578862749637588, "scr_metric_threshold_10": 0.0005267100879694628, "scr_dir2_threshold_10": 0.0005267100879694628, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.00036336475599393935, + "scr_metric_threshold_20": 0.0015627090975924374, + "scr_dir2_threshold_20": 0.0021283126350334566, + "scr_dir1_threshold_50": -0.0011627755466854, + "scr_metric_threshold_50": 0.002366738403605829, + "scr_dir2_threshold_50": 0.002932341941046848, + "scr_dir1_threshold_100": -0.001953099211795167, + "scr_metric_threshold_100": 0.006587582747192433, + "scr_dir2_threshold_100": 0.0059904104155906945, + "scr_dir1_threshold_500": 0.0068615762247188575, + "scr_metric_threshold_500": 0.015912912334432375, + "scr_dir2_threshold_500": 0.01644698079074465 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.0, "scr_metric_threshold_10": -0.0025906399716121324, "scr_dir2_threshold_10": -0.0025906399716121324, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.012821110434403185, + "scr_metric_threshold_20": 0.0, + "scr_dir2_threshold_20": 0.0, + "scr_dir1_threshold_50": 0.0, + "scr_metric_threshold_50": 0.0025906399716121324, + "scr_dir2_threshold_50": 0.0025906399716121324, + "scr_dir1_threshold_100": 0.0, + "scr_metric_threshold_100": 0.0, + "scr_dir2_threshold_100": 0.0, + "scr_dir1_threshold_500": 0.012821110434403185, + "scr_metric_threshold_500": 0.010362714302611827, + "scr_dir2_threshold_500": 0.010362714302611827 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.0, "scr_metric_threshold_10": 0.0, "scr_dir2_threshold_10": 0.0, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.0, + "scr_metric_threshold_20": 0.0, + "scr_dir2_threshold_20": 0.0, + "scr_dir1_threshold_50": 0.0, + "scr_metric_threshold_50": 0.003058064398530846, + "scr_dir2_threshold_50": 0.003058064398530846, + "scr_dir1_threshold_100": 0.0, + "scr_metric_threshold_100": 0.006116311074250613, + "scr_dir2_threshold_100": 0.006116311074250613, + "scr_dir1_threshold_500": 0.015151768293316199, + "scr_metric_threshold_500": 0.015290504269843151, + "scr_dir2_threshold_500": 0.015290504269843151 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.0, "scr_metric_threshold_10": 0.0025125300775025, "scr_dir2_threshold_10": 0.0025125300775025, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.0, + "scr_metric_threshold_20": 0.0050252099153958625, + "scr_dir2_threshold_20": 0.0050252099153958625, + "scr_dir1_threshold_50": 0.0, + "scr_metric_threshold_50": 0.0050252099153958625, + "scr_dir2_threshold_50": 0.0050252099153958625, + "scr_dir1_threshold_100": -0.015624796273220196, + "scr_metric_threshold_100": 0.0050252099153958625, + "scr_dir2_threshold_100": 0.0050252099153958625, + "scr_dir1_threshold_500": 0.0, + "scr_metric_threshold_500": 0.010050270070400862, + "scr_dir2_threshold_500": 0.010050270070400862 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": 0.0, "scr_metric_threshold_10": 0.0, "scr_dir2_threshold_10": 0.0, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.0, + "scr_metric_threshold_20": 0.003184672267478302, + "scr_dir2_threshold_20": 0.003184672267478302, + "scr_dir1_threshold_50": 0.0, + "scr_metric_threshold_50": 0.0, + "scr_dir2_threshold_50": 0.0, + "scr_dir1_threshold_100": 0.0, + "scr_metric_threshold_100": 0.003184672267478302, + "scr_dir2_threshold_100": 0.003184672267478302, + "scr_dir1_threshold_500": 0.0, + "scr_metric_threshold_500": 0.00955420662613804, + "scr_dir2_threshold_500": 0.00955420662613804 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": -0.00526309019971007, "scr_metric_threshold_10": 0.0, "scr_dir2_threshold_10": 0.0, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.00526309019971007, + "scr_metric_threshold_20": 0.0, + "scr_dir2_threshold_20": 0.0, + "scr_dir1_threshold_50": 0.0, + "scr_metric_threshold_50": 0.003968202345442456, + "scr_dir2_threshold_50": 0.003968202345442456, + "scr_dir1_threshold_100": 0.0, + "scr_metric_threshold_100": 0.01190484356266375, + "scr_dir2_threshold_100": 0.01190484356266375, + "scr_dir1_threshold_500": 0.01052649410808079, + "scr_metric_threshold_500": 0.043650698852539783, + "scr_dir2_threshold_500": 0.043650698852539783 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.0, "scr_metric_threshold_10": 0.0, "scr_dir2_threshold_10": 0.0, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.0, + "scr_metric_threshold_20": 0.0, + "scr_dir2_threshold_20": 0.0, + "scr_dir1_threshold_50": 0.0, + "scr_metric_threshold_50": 0.0, + "scr_dir2_threshold_50": 0.0, + "scr_dir1_threshold_100": 0.0, + "scr_metric_threshold_100": 0.0, + "scr_dir2_threshold_100": 0.0, + "scr_dir1_threshold_500": 0.016393234383091822, + "scr_metric_threshold_500": 0.007633488798309048, + "scr_dir2_threshold_500": 0.007633488798309048 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.0, "scr_metric_threshold_10": 0.004291790597865335, "scr_dir2_threshold_10": 0.004291790597865335, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.0046511021867416, + "scr_metric_threshold_20": 0.004291790597865335, + "scr_dir2_threshold_20": 0.004291790597865335, + "scr_dir1_threshold_50": -0.0093022043734832, + "scr_metric_threshold_50": 0.004291790597865335, + "scr_dir2_threshold_50": 0.004291790597865335, + "scr_dir1_threshold_100": -0.0093022043734832, + "scr_metric_threshold_100": 0.017167418205408878, + "scr_dir2_threshold_100": 0.017167418205408878, + "scr_dir1_threshold_500": -0.0093022043734832, + "scr_metric_threshold_500": 0.021459208803274215, + "scr_dir2_threshold_500": 0.021459208803274215 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.0, "scr_metric_threshold_10": 0.0, "scr_dir2_threshold_10": 0.0, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.0, + "scr_metric_threshold_20": 0.0, + "scr_dir2_threshold_20": 0.004524828299528152, + "scr_dir1_threshold_50": 0.0, + "scr_metric_threshold_50": 0.0, + "scr_dir2_threshold_50": 0.004524828299528152, + "scr_dir1_threshold_100": 0.009302206952342061, + "scr_metric_threshold_100": 0.009302206952342061, + "scr_dir2_threshold_100": 0.004524828299528152, + "scr_dir1_threshold_500": 0.009302206952342061, + "scr_metric_threshold_500": 0.009302206952342061, + "scr_dir2_threshold_500": 0.013574754602840255 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-1-tokens-0/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-1-tokens-0/tpp/saebench_tpp_custom_sae_eval_results.json index 7baf5b7c3ac72fb28f564e98b3f0211f9e81ab81..66f64cb28c966bf3c2d821b247b0392434624127 100644 --- a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-1-tokens-0/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-1-tokens-0/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "e15be7e9-222d-434c-871b-033992544375", - "datetime_epoch_millis": 1776887591121, + "eval_id": "459b7cd6-4d4c-4116-bc95-8f48ca8416b1", + "datetime_epoch_millis": 1777318843820, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.002500006556510925, - "tpp_threshold_2_intended_diff_only": 0.0011000096797943114, - "tpp_threshold_2_unintended_diff_only": -0.0013999968767166137, - "tpp_threshold_5_total_metric": -0.00037499964237213134, - "tpp_threshold_5_intended_diff_only": 0.0003000020980834961, - "tpp_threshold_5_unintended_diff_only": 0.0006750017404556275, - "tpp_threshold_10_total_metric": -0.001299990713596344, - "tpp_threshold_10_intended_diff_only": -0.0011999845504760743, - "tpp_threshold_10_unintended_diff_only": 0.00010000616312026976, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.0011250093579292298, + "tpp_threshold_2_intended_diff_only": 0.000800013542175293, + "tpp_threshold_2_unintended_diff_only": -0.00032499581575393675, + "tpp_threshold_5_total_metric": -0.00014999955892562868, + "tpp_threshold_5_intended_diff_only": -0.000999993085861206, + "tpp_threshold_5_unintended_diff_only": -0.0008499935269355774, + "tpp_threshold_10_total_metric": 0.0003750026226043701, + "tpp_threshold_10_intended_diff_only": 0.0009000062942504882, + "tpp_threshold_10_unintended_diff_only": 0.0005250036716461182, + "tpp_threshold_20_total_metric": 0.0008250057697296142, + "tpp_threshold_20_intended_diff_only": 0.000800013542175293, + "tpp_threshold_20_unintended_diff_only": -2.4992227554321273e-05, + "tpp_threshold_50_total_metric": 0.003724999725818634, + "tpp_threshold_50_intended_diff_only": 0.0027000069618225097, + "tpp_threshold_50_unintended_diff_only": -0.0010249927639961244, + "tpp_threshold_100_total_metric": 0.0005250081419944763, + "tpp_threshold_100_intended_diff_only": 0.0021000087261199953, + "tpp_threshold_100_unintended_diff_only": 0.0015750005841255188, + "tpp_threshold_500_total_metric": 0.02664999067783356, + "tpp_threshold_500_intended_diff_only": 0.02789999842643738, + "tpp_threshold_500_unintended_diff_only": 0.0012500077486038209 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.002450016140937805, - "tpp_threshold_2_intended_diff_only": 0.0008000254631042481, - "tpp_threshold_2_unintended_diff_only": -0.0016499906778335571, - "tpp_threshold_5_total_metric": -0.001600000262260437, + "tpp_threshold_2_total_metric": 0.0015500128269195558, + "tpp_threshold_2_intended_diff_only": 0.0012000203132629395, + "tpp_threshold_2_unintended_diff_only": -0.0003499925136566162, + "tpp_threshold_5_total_metric": -0.00030000805854797364, "tpp_threshold_5_intended_diff_only": -0.0007999897003173828, - "tpp_threshold_5_unintended_diff_only": 0.0008000105619430542, - "tpp_threshold_10_total_metric": -0.0017499953508377075, - "tpp_threshold_10_intended_diff_only": -0.0015999794006347657, - "tpp_threshold_10_unintended_diff_only": 0.0001500159502029419, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_5_unintended_diff_only": -0.0004999816417694091, + "tpp_threshold_10_total_metric": -0.0012999981641769408, + "tpp_threshold_10_intended_diff_only": -0.0011999845504760743, + "tpp_threshold_10_unintended_diff_only": 0.0001000136137008667, + "tpp_threshold_20_total_metric": 0.0032000094652175903, + "tpp_threshold_20_intended_diff_only": 0.002800023555755615, + "tpp_threshold_20_unintended_diff_only": -0.0003999859094619751, + "tpp_threshold_50_total_metric": 0.0027500003576278686, + "tpp_threshold_50_intended_diff_only": 0.0026000142097473145, + "tpp_threshold_50_unintended_diff_only": -0.0001499861478805542, + "tpp_threshold_100_total_metric": -0.0010999828577041627, + "tpp_threshold_100_intended_diff_only": -0.00019997358322143555, + "tpp_threshold_100_unintended_diff_only": 0.000900009274482727, + "tpp_threshold_500_total_metric": 0.01184999942779541, + "tpp_threshold_500_intended_diff_only": 0.012000012397766113, + "tpp_threshold_500_unintended_diff_only": 0.00015001296997070314 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.0025499969720840453, - "tpp_threshold_2_intended_diff_only": 0.001399993896484375, - "tpp_threshold_2_unintended_diff_only": -0.0011500030755996703, - "tpp_threshold_5_total_metric": 0.0008500009775161743, - "tpp_threshold_5_intended_diff_only": 0.001399993896484375, - "tpp_threshold_5_unintended_diff_only": 0.0005499929189682007, - "tpp_threshold_10_total_metric": -0.0008499860763549804, - "tpp_threshold_10_intended_diff_only": -0.0007999897003173828, - "tpp_threshold_10_unintended_diff_only": 4.9996376037597654e-05, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.0007000058889389038, + "tpp_threshold_2_intended_diff_only": 0.0004000067710876465, + "tpp_threshold_2_unintended_diff_only": -0.0002999991178512573, + "tpp_threshold_5_total_metric": 8.940696716308593e-09, + "tpp_threshold_5_intended_diff_only": -0.0011999964714050292, + "tpp_threshold_5_unintended_diff_only": -0.0012000054121017456, + "tpp_threshold_10_total_metric": 0.002050003409385681, + "tpp_threshold_10_intended_diff_only": 0.0029999971389770507, + "tpp_threshold_10_unintended_diff_only": 0.0009499937295913696, + "tpp_threshold_20_total_metric": -0.001549997925758362, + "tpp_threshold_20_intended_diff_only": -0.0011999964714050292, + "tpp_threshold_20_unintended_diff_only": 0.00035000145435333253, + "tpp_threshold_50_total_metric": 0.004699999094009399, + "tpp_threshold_50_intended_diff_only": 0.002799999713897705, + "tpp_threshold_50_unintended_diff_only": -0.0018999993801116944, + "tpp_threshold_100_total_metric": 0.0021499991416931152, + "tpp_threshold_100_intended_diff_only": 0.004399991035461426, + "tpp_threshold_100_unintended_diff_only": 0.0022499918937683105, + "tpp_threshold_500_total_metric": 0.04144998192787171, + "tpp_threshold_500_intended_diff_only": 0.043799984455108645, + "tpp_threshold_500_unintended_diff_only": 0.0023500025272369385 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.0005000084638595581, "tpp_threshold_10_total_metric": -0.004000037908554077, "tpp_threshold_10_intended_diff_only": -0.0040000081062316895, - "tpp_threshold_10_unintended_diff_only": 2.9802322387695312e-08 + "tpp_threshold_10_unintended_diff_only": 2.9802322387695312e-08, + "tpp_threshold_20_total_metric": 0.0032499581575393677, + "tpp_threshold_20_intended_diff_only": 0.001999974250793457, + "tpp_threshold_20_unintended_diff_only": -0.0012499839067459106, + "tpp_threshold_50_total_metric": 0.001249954104423523, + "tpp_threshold_50_intended_diff_only": 0.001999974250793457, + "tpp_threshold_50_unintended_diff_only": 0.0007500201463699341, + "tpp_threshold_100_total_metric": -0.0010000020265579224, + "tpp_threshold_100_intended_diff_only": -0.0009999871253967285, + "tpp_threshold_100_unintended_diff_only": 1.4901161193847656e-08, + "tpp_threshold_500_total_metric": 0.021249964833259583, + "tpp_threshold_500_intended_diff_only": 0.019999980926513672, + "tpp_threshold_500_unintended_diff_only": -0.0012499839067459106 }, "1": { - "tpp_threshold_2_total_metric": -0.0014999806880950928, - "tpp_threshold_2_intended_diff_only": -0.001999974250793457, - "tpp_threshold_2_unintended_diff_only": -0.0004999935626983643, - "tpp_threshold_5_total_metric": -0.0010000169277191162, - "tpp_threshold_5_intended_diff_only": 0.0, - "tpp_threshold_5_unintended_diff_only": 0.0010000169277191162, - "tpp_threshold_10_total_metric": -0.007499992847442627, - "tpp_threshold_10_intended_diff_only": -0.0059999823570251465, - "tpp_threshold_10_unintended_diff_only": 0.0015000104904174805 + "tpp_threshold_2_total_metric": 0.003250032663345337, + "tpp_threshold_2_intended_diff_only": 0.0020000338554382324, + "tpp_threshold_2_unintended_diff_only": -0.0012499988079071045, + "tpp_threshold_5_total_metric": -0.004250019788742065, + "tpp_threshold_5_intended_diff_only": -0.004999995231628418, + "tpp_threshold_5_unintended_diff_only": -0.0007499754428863525, + "tpp_threshold_10_total_metric": -0.0025000274181365967, + "tpp_threshold_10_intended_diff_only": -0.0040000081062316895, + "tpp_threshold_10_unintended_diff_only": -0.0014999806880950928, + "tpp_threshold_20_total_metric": -0.0015000253915786743, + "tpp_threshold_20_intended_diff_only": 0.0, + "tpp_threshold_20_unintended_diff_only": 0.0015000253915786743, + "tpp_threshold_50_total_metric": 0.000250011682510376, + "tpp_threshold_50_intended_diff_only": 0.0020000338554382324, + "tpp_threshold_50_unintended_diff_only": 0.0017500221729278564, + "tpp_threshold_100_total_metric": -0.008249998092651367, + "tpp_threshold_100_intended_diff_only": -0.004999995231628418, + "tpp_threshold_100_unintended_diff_only": 0.0032500028610229492, + "tpp_threshold_500_total_metric": 0.016750022768974304, + "tpp_threshold_500_intended_diff_only": 0.016000032424926758, + "tpp_threshold_500_unintended_diff_only": -0.0007499903440475464 }, "2": { - "tpp_threshold_2_total_metric": 0.006000027060508728, - "tpp_threshold_2_intended_diff_only": 0.0020000338554382324, - "tpp_threshold_2_unintended_diff_only": -0.003999993205070496, - "tpp_threshold_5_total_metric": 0.0007499903440475464, - "tpp_threshold_5_intended_diff_only": 0.0, - "tpp_threshold_5_unintended_diff_only": -0.0007499903440475464, - "tpp_threshold_10_total_metric": 0.0045000165700912476, - "tpp_threshold_10_intended_diff_only": 0.0020000338554382324, - "tpp_threshold_10_unintended_diff_only": -0.002499982714653015 + "tpp_threshold_2_total_metric": -0.0014999955892562866, + "tpp_threshold_2_intended_diff_only": -0.0009999871253967285, + "tpp_threshold_2_unintended_diff_only": 0.0005000084638595581, + "tpp_threshold_5_total_metric": -0.0010000020265579224, + "tpp_threshold_5_intended_diff_only": -0.0009999871253967285, + "tpp_threshold_5_unintended_diff_only": 1.4901161193847656e-08, + "tpp_threshold_10_total_metric": 0.0050000399351119995, + "tpp_threshold_10_intended_diff_only": 0.006000041961669922, + "tpp_threshold_10_unintended_diff_only": 0.0010000020265579224, + "tpp_threshold_20_total_metric": 0.01100003719329834, + "tpp_threshold_20_intended_diff_only": 0.010000050067901611, + "tpp_threshold_20_unintended_diff_only": -0.0009999871253967285, + "tpp_threshold_50_total_metric": 0.00599999725818634, + "tpp_threshold_50_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_50_unintended_diff_only": -0.001999989151954651, + "tpp_threshold_100_total_metric": 0.003000035881996155, + "tpp_threshold_100_intended_diff_only": 0.001000046730041504, + "tpp_threshold_100_unintended_diff_only": -0.001999989151954651, + "tpp_threshold_500_total_metric": 0.009749993681907654, + "tpp_threshold_500_intended_diff_only": 0.009000003337860107, + "tpp_threshold_500_unintended_diff_only": -0.0007499903440475464 }, "6": { - "tpp_threshold_2_total_metric": 0.004000052809715271, + "tpp_threshold_2_total_metric": 0.0017500370740890503, "tpp_threshold_2_intended_diff_only": 0.001000046730041504, - "tpp_threshold_2_unintended_diff_only": -0.003000006079673767, - "tpp_threshold_5_total_metric": -0.005249977111816406, - "tpp_threshold_5_intended_diff_only": -0.0029999613761901855, - "tpp_threshold_5_unintended_diff_only": 0.0022500157356262207, - "tpp_threshold_10_total_metric": 2.9802322387695312e-08, - "tpp_threshold_10_intended_diff_only": 0.001000046730041504, - "tpp_threshold_10_unintended_diff_only": 0.0010000169277191162 + "tpp_threshold_2_unintended_diff_only": -0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.0007500052452087402, + "tpp_threshold_5_intended_diff_only": -0.001999974250793457, + "tpp_threshold_5_unintended_diff_only": -0.0027499794960021973, + "tpp_threshold_10_total_metric": 0.002750024199485779, + "tpp_threshold_10_intended_diff_only": 0.0020000338554382324, + "tpp_threshold_10_unintended_diff_only": -0.0007499903440475464, + "tpp_threshold_20_total_metric": 0.003500029444694519, + "tpp_threshold_20_intended_diff_only": 0.001000046730041504, + "tpp_threshold_20_unintended_diff_only": -0.002499982714653015, + "tpp_threshold_50_total_metric": 0.0065000057220458984, + "tpp_threshold_50_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_50_unintended_diff_only": -0.002499997615814209, + "tpp_threshold_100_total_metric": 0.000500023365020752, + "tpp_threshold_100_intended_diff_only": 0.0020000338554382324, + "tpp_threshold_100_unintended_diff_only": 0.0015000104904174805, + "tpp_threshold_500_total_metric": 0.006000012159347534, + "tpp_threshold_500_intended_diff_only": 0.00700002908706665, + "tpp_threshold_500_unintended_diff_only": 0.0010000169277191162 }, "9": { - "tpp_threshold_2_total_metric": -0.0010000020265579224, - "tpp_threshold_2_intended_diff_only": -0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 1.4901161193847656e-08, - "tpp_threshold_5_total_metric": -0.00599999725818634, - "tpp_threshold_5_intended_diff_only": -0.004999995231628418, - "tpp_threshold_5_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_10_total_metric": -0.0017499923706054688, - "tpp_threshold_10_intended_diff_only": -0.0009999871253967285, - "tpp_threshold_10_unintended_diff_only": 0.0007500052452087402 + "tpp_threshold_2_total_metric": -0.0004999935626983643, + "tpp_threshold_2_intended_diff_only": 0.0, + "tpp_threshold_2_unintended_diff_only": 0.0004999935626983643, + "tpp_threshold_5_total_metric": -0.000500023365020752, + "tpp_threshold_5_intended_diff_only": 0.0, + "tpp_threshold_5_unintended_diff_only": 0.000500023365020752, + "tpp_threshold_10_total_metric": -0.007749989628791809, + "tpp_threshold_10_intended_diff_only": -0.0059999823570251465, + "tpp_threshold_10_unintended_diff_only": 0.0017500072717666626, + "tpp_threshold_20_total_metric": -0.0002499520778656006, + "tpp_threshold_20_intended_diff_only": 0.001000046730041504, + "tpp_threshold_20_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_50_total_metric": -0.00024996697902679443, + "tpp_threshold_50_intended_diff_only": 0.001000046730041504, + "tpp_threshold_50_unintended_diff_only": 0.0012500137090682983, + "tpp_threshold_100_total_metric": 0.0002500265836715698, + "tpp_threshold_100_intended_diff_only": 0.0020000338554382324, + "tpp_threshold_100_unintended_diff_only": 0.0017500072717666626, + "tpp_threshold_500_total_metric": 0.005500003695487976, + "tpp_threshold_500_intended_diff_only": 0.008000016212463379, + "tpp_threshold_500_unintended_diff_only": 0.002500012516975403 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.007249981164932251, - "tpp_threshold_2_intended_diff_only": 0.001999974250793457, - "tpp_threshold_2_unintended_diff_only": -0.005250006914138794, - "tpp_threshold_5_total_metric": 0.010249987244606018, - "tpp_threshold_5_intended_diff_only": 0.006999969482421875, - "tpp_threshold_5_unintended_diff_only": -0.003250017762184143, - "tpp_threshold_10_total_metric": -0.006999984383583069, - "tpp_threshold_10_intended_diff_only": -0.004999995231628418, - "tpp_threshold_10_unintended_diff_only": 0.001999989151954651 + "tpp_threshold_2_total_metric": 0.000250011682510376, + "tpp_threshold_2_intended_diff_only": 0.003000020980834961, + "tpp_threshold_2_unintended_diff_only": 0.002750009298324585, + "tpp_threshold_5_total_metric": 0.0020000189542770386, + "tpp_threshold_5_intended_diff_only": 0.003000020980834961, + "tpp_threshold_5_unintended_diff_only": 0.0010000020265579224, + "tpp_threshold_10_total_metric": -0.001499965786933899, + "tpp_threshold_10_intended_diff_only": -0.0009999871253967285, + "tpp_threshold_10_unintended_diff_only": 0.0004999786615371704, + "tpp_threshold_20_total_metric": -0.0004999786615371704, + "tpp_threshold_20_intended_diff_only": -0.0009999871253967285, + "tpp_threshold_20_unintended_diff_only": -0.0005000084638595581, + "tpp_threshold_50_total_metric": 0.002749994397163391, + "tpp_threshold_50_intended_diff_only": -0.0040000081062316895, + "tpp_threshold_50_unintended_diff_only": -0.006750002503395081, + "tpp_threshold_100_total_metric": -0.0012500137090682983, + "tpp_threshold_100_intended_diff_only": 0.0009999871253967285, + "tpp_threshold_100_unintended_diff_only": 0.002250000834465027, + "tpp_threshold_500_total_metric": 0.02024996280670166, + "tpp_threshold_500_intended_diff_only": 0.015999972820281982, + "tpp_threshold_500_unintended_diff_only": -0.004249989986419678 }, "2": { - "tpp_threshold_2_total_metric": -0.005999967455863953, - "tpp_threshold_2_intended_diff_only": -0.001999974250793457, - "tpp_threshold_2_unintended_diff_only": 0.003999993205070496, - "tpp_threshold_5_total_metric": -0.008249923586845398, - "tpp_threshold_5_intended_diff_only": -0.003999948501586914, - "tpp_threshold_5_unintended_diff_only": 0.004249975085258484, - "tpp_threshold_10_total_metric": -0.0009999573230743408, - "tpp_threshold_10_intended_diff_only": -0.0029999613761901855, - "tpp_threshold_10_unintended_diff_only": -0.0020000040531158447 + "tpp_threshold_2_total_metric": 0.0040000081062316895, + "tpp_threshold_2_intended_diff_only": 0.0, + "tpp_threshold_2_unintended_diff_only": -0.0040000081062316895, + "tpp_threshold_5_total_metric": -0.010999947786331177, + "tpp_threshold_5_intended_diff_only": -0.006999969482421875, + "tpp_threshold_5_unintended_diff_only": 0.003999978303909302, + "tpp_threshold_10_total_metric": 0.003000020980834961, + "tpp_threshold_10_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_10_unintended_diff_only": 0.0009999871253967285, + "tpp_threshold_20_total_metric": -0.0019999444484710693, + "tpp_threshold_20_intended_diff_only": -0.003999948501586914, + "tpp_threshold_20_unintended_diff_only": -0.0020000040531158447, + "tpp_threshold_50_total_metric": 0.0015000253915786743, + "tpp_threshold_50_intended_diff_only": 0.003000020980834961, + "tpp_threshold_50_unintended_diff_only": 0.0014999955892562866, + "tpp_threshold_100_total_metric": -0.0027499347925186157, + "tpp_threshold_100_intended_diff_only": 0.001000046730041504, + "tpp_threshold_100_unintended_diff_only": 0.0037499815225601196, + "tpp_threshold_500_total_metric": 0.05125002562999725, + "tpp_threshold_500_intended_diff_only": 0.054000020027160645, + "tpp_threshold_500_unintended_diff_only": 0.002749994397163391 }, "3": { - "tpp_threshold_2_total_metric": 0.010500013828277588, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": -0.0065000057220458984, - "tpp_threshold_5_total_metric": 0.014500021934509277, - "tpp_threshold_5_intended_diff_only": 0.012000024318695068, - "tpp_threshold_5_unintended_diff_only": -0.002499997615814209, - "tpp_threshold_10_total_metric": 0.011250019073486328, - "tpp_threshold_10_intended_diff_only": 0.012000024318695068, - "tpp_threshold_10_unintended_diff_only": 0.0007500052452087402 + "tpp_threshold_2_total_metric": 0.0007500052452087402, + "tpp_threshold_2_intended_diff_only": 0.0, + "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402, + "tpp_threshold_5_total_metric": 0.006999999284744263, + "tpp_threshold_5_intended_diff_only": 0.0009999871253967285, + "tpp_threshold_5_unintended_diff_only": -0.006000012159347534, + "tpp_threshold_10_total_metric": 0.008750036358833313, + "tpp_threshold_10_intended_diff_only": 0.00700002908706665, + "tpp_threshold_10_unintended_diff_only": -0.0017500072717666626, + "tpp_threshold_20_total_metric": 0.0002499818801879883, + "tpp_threshold_20_intended_diff_only": 0.0009999871253967285, + "tpp_threshold_20_unintended_diff_only": 0.0007500052452087402, + "tpp_threshold_50_total_metric": 0.01124998927116394, + "tpp_threshold_50_intended_diff_only": 0.004999995231628418, + "tpp_threshold_50_unintended_diff_only": -0.0062499940395355225, + "tpp_threshold_100_total_metric": 0.020500004291534424, + "tpp_threshold_100_intended_diff_only": 0.018999993801116943, + "tpp_threshold_100_unintended_diff_only": -0.0015000104904174805, + "tpp_threshold_500_total_metric": 0.030999988317489624, + "tpp_threshold_500_intended_diff_only": 0.03299999237060547, + "tpp_threshold_500_unintended_diff_only": 0.0020000040531158447 }, "5": { - "tpp_threshold_2_total_metric": -0.006250053644180298, - "tpp_threshold_2_intended_diff_only": -0.001000046730041504, - "tpp_threshold_2_unintended_diff_only": 0.005250006914138794, - "tpp_threshold_5_total_metric": -0.0055000633001327515, - "tpp_threshold_5_intended_diff_only": -0.005000054836273193, - "tpp_threshold_5_unintended_diff_only": 0.0005000084638595581, - "tpp_threshold_10_total_metric": -0.00475001335144043, - "tpp_threshold_10_intended_diff_only": -0.0040000081062316895, - "tpp_threshold_10_unintended_diff_only": 0.0007500052452087402 + "tpp_threshold_2_total_metric": 0.0012499839067459106, + "tpp_threshold_2_intended_diff_only": 0.0, + "tpp_threshold_2_unintended_diff_only": -0.0012499839067459106, + "tpp_threshold_5_total_metric": 0.002249985933303833, + "tpp_threshold_5_intended_diff_only": 0.0009999871253967285, + "tpp_threshold_5_unintended_diff_only": -0.0012499988079071045, + "tpp_threshold_10_total_metric": -0.010750040411949158, + "tpp_threshold_10_intended_diff_only": -0.00700002908706665, + "tpp_threshold_10_unintended_diff_only": 0.0037500113248825073, + "tpp_threshold_20_total_metric": -0.006500035524368286, + "tpp_threshold_20_intended_diff_only": -0.00700002908706665, + "tpp_threshold_20_unintended_diff_only": -0.0004999935626983643, + "tpp_threshold_50_total_metric": 0.0032500028610229492, + "tpp_threshold_50_intended_diff_only": 0.004999995231628418, + "tpp_threshold_50_unintended_diff_only": 0.0017499923706054688, + "tpp_threshold_100_total_metric": -0.003750026226043701, + "tpp_threshold_100_intended_diff_only": -0.0020000338554382324, + "tpp_threshold_100_unintended_diff_only": 0.0017499923706054688, + "tpp_threshold_500_total_metric": 0.02299995720386505, + "tpp_threshold_500_intended_diff_only": 0.029999971389770508, + "tpp_threshold_500_unintended_diff_only": 0.0070000141859054565 }, "6": { - "tpp_threshold_2_total_metric": 0.007250010967254639, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": -0.0032500028610229492, - "tpp_threshold_5_total_metric": -0.006750017404556274, - "tpp_threshold_5_intended_diff_only": -0.003000020980834961, - "tpp_threshold_5_unintended_diff_only": 0.0037499964237213135, - "tpp_threshold_10_total_metric": -0.002749994397163391, - "tpp_threshold_10_intended_diff_only": -0.0040000081062316895, - "tpp_threshold_10_unintended_diff_only": -0.0012500137090682983 + "tpp_threshold_2_total_metric": -0.0027499794960021973, + "tpp_threshold_2_intended_diff_only": -0.0009999871253967285, + "tpp_threshold_2_unintended_diff_only": 0.0017499923706054688, + "tpp_threshold_5_total_metric": -0.000250011682510376, + "tpp_threshold_5_intended_diff_only": -0.0040000081062316895, + "tpp_threshold_5_unintended_diff_only": -0.0037499964237213135, + "tpp_threshold_10_total_metric": 0.010749965906143188, + "tpp_threshold_10_intended_diff_only": 0.011999964714050293, + "tpp_threshold_10_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_20_total_metric": 0.0009999871253967285, + "tpp_threshold_20_intended_diff_only": 0.004999995231628418, + "tpp_threshold_20_unintended_diff_only": 0.0040000081062316895, + "tpp_threshold_50_total_metric": 0.004749983549118042, + "tpp_threshold_50_intended_diff_only": 0.004999995231628418, + "tpp_threshold_50_unintended_diff_only": 0.000250011682510376, + "tpp_threshold_100_total_metric": -0.0020000338554382324, + "tpp_threshold_100_intended_diff_only": 0.0029999613761901855, + "tpp_threshold_100_unintended_diff_only": 0.004999995231628418, + "tpp_threshold_500_total_metric": 0.08174997568130493, + "tpp_threshold_500_intended_diff_only": 0.08599996566772461, + "tpp_threshold_500_unintended_diff_only": 0.004249989986419678 } } } diff --git a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-109863-tokens-450000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-109863-tokens-450000000/scr/saebench_scr_custom_sae_eval_results.json index ed35095db401be281b45bbfa493c53ddecde2147..2c046cbab51b38a659e6e2ab01bcaee56c5369eb 100644 --- a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-109863-tokens-450000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-109863-tokens-450000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "03dd6f0c-de7f-4ac7-93c4-b29bd976d00c", - "datetime_epoch_millis": 1776887417282, + "eval_id": "2cb27175-7be1-473c-906f-1e6af6ff38f3", + "datetime_epoch_millis": 1777318570701, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.22673879591497073, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.22831828143528313, "scr_metric_threshold_10": 0.1772069044124458, "scr_dir2_threshold_10": 0.16754154766147558, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.016202965609438895, + "scr_metric_threshold_20": 0.19751609610410986, + "scr_dir2_threshold_20": 0.1912601786878981, + "scr_dir1_threshold_50": 0.1430312256829767, + "scr_metric_threshold_50": 0.275227282170841, + "scr_dir2_threshold_50": 0.268216346570553, + "scr_dir1_threshold_100": -0.04426974601444296, + "scr_metric_threshold_100": 0.27736965412386044, + "scr_dir2_threshold_100": 0.26751488178541943, + "scr_dir1_threshold_500": -0.8083543505356644, + "scr_metric_threshold_500": 0.1330646957177387, + "scr_dir2_threshold_500": 0.12129209467231115 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.47435892537422925, "scr_metric_threshold_10": 0.05440405706850797, "scr_dir2_threshold_10": 0.05440405706850797, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.47435892537422925, + "scr_metric_threshold_20": 0.16839375998107475, + "scr_dir2_threshold_20": 0.16839375998107475, + "scr_dir1_threshold_50": 0.43589788655709094, + "scr_metric_threshold_50": 0.2720207485910297, + "scr_dir2_threshold_50": 0.2720207485910297, + "scr_dir1_threshold_100": 0.43589788655709094, + "scr_metric_threshold_100": 0.3290154456311498, + "scr_dir2_threshold_100": 0.3290154456311498, + "scr_dir1_threshold_500": -2.358974280598767, + "scr_metric_threshold_500": 0.16580312000946262, + "scr_dir2_threshold_500": 0.16580312000946262 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.47727279911026543, "scr_metric_threshold_10": 0.18348623351530674, "scr_dir2_threshold_10": 0.18348623351530674, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.41666662903748003, + "scr_metric_threshold_20": 0.23853212179761762, + "scr_dir2_threshold_20": 0.23853212179761762, + "scr_dir1_threshold_50": -0.4924241158533419, + "scr_metric_threshold_50": 0.4801223079937661, + "scr_dir2_threshold_50": 0.4801223079937661, + "scr_dir1_threshold_100": -1.280302520598593, + "scr_metric_threshold_100": 0.4801223079937661, + "scr_dir2_threshold_100": 0.4801223079937661, + "scr_dir1_threshold_500": -0.06818182844432363, + "scr_metric_threshold_500": 0.036697319613936914, + "scr_dir2_threshold_500": 0.036697319613936914 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.015624796273220196, "scr_metric_threshold_10": 0.15577896145062706, "scr_dir2_threshold_10": 0.15577896145062706, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.8437501746229541, + "scr_metric_threshold_20": 0.16331655168313458, + "scr_dir2_threshold_20": 0.16331655168313458, + "scr_dir1_threshold_50": 0.3437501746229541, + "scr_metric_threshold_50": 0.1507537515352312, + "scr_dir2_threshold_50": 0.1507537515352312, + "scr_dir1_threshold_100": 0.23437473806556883, + "scr_metric_threshold_100": 0.07035174073241517, + "scr_dir2_threshold_100": 0.07035174073241517, + "scr_dir1_threshold_500": -3.4062493597158348, + "scr_metric_threshold_500": -0.16331655168313458, + "scr_dir2_threshold_500": -0.16331655168313458 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": 0.13422806978944357, "scr_metric_threshold_10": 0.19426747602431718, "scr_dir2_threshold_10": 0.19426747602431718, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.1610741637847752, + "scr_metric_threshold_20": -0.04777065348328394, + "scr_dir2_threshold_20": -0.04777065348328394, + "scr_dir1_threshold_50": -0.12751714633741432, + "scr_metric_threshold_50": -0.21337569945289012, + "scr_dir2_threshold_50": -0.21337569945289012, + "scr_dir1_threshold_100": -0.7114094899402154, + "scr_metric_threshold_100": -0.2070063549179335, + "scr_dir2_threshold_100": -0.2070063549179335, + "scr_dir1_threshold_500": -1.7114098899714179, + "scr_metric_threshold_500": -0.48726112110638364, + "scr_dir2_threshold_500": -0.48726112110638364 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.03684225881529179, "scr_metric_threshold_10": 0.20238092140726546, "scr_dir2_threshold_10": 0.20238092140726546, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.057894933322792726, + "scr_metric_threshold_20": 0.3095238038922301, + "scr_dir2_threshold_20": 0.3095238038922301, + "scr_dir1_threshold_50": -0.01052618039942014, + "scr_metric_threshold_50": 0.5476190194611504, + "scr_dir2_threshold_50": 0.5476190194611504, + "scr_dir1_threshold_100": -0.02105267450750093, + "scr_metric_threshold_100": 0.6190476077844602, + "scr_dir2_threshold_100": 0.6190476077844602, + "scr_dir1_threshold_500": -0.05789461961413207, + "scr_metric_threshold_500": 0.6626983066369999, + "scr_dir2_threshold_500": 0.6626983066369999 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.3114753617807135, "scr_metric_threshold_10": 0.14122136275770192, "scr_dir2_threshold_10": 0.14122136275770192, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3224041847027747, + "scr_metric_threshold_20": 0.20229018313866656, + "scr_dir2_threshold_20": 0.20229018313866656, + "scr_dir1_threshold_50": 0.4043716594522235, + "scr_metric_threshold_50": 0.2557252872226991, + "scr_dir2_threshold_50": 0.2557252872226991, + "scr_dir1_threshold_100": 0.3278689218723027, + "scr_metric_threshold_100": 0.38549618928431434, + "scr_dir2_threshold_100": 0.38549618928431434, + "scr_dir1_threshold_500": 0.4754099855711136, + "scr_metric_threshold_500": 0.2557252872226991, + "scr_dir2_threshold_500": 0.2557252872226991 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.19534878692091323, "scr_metric_threshold_10": 0.3047209696576519, "scr_dir2_threshold_10": 0.3047209696576519, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2604650492278843, + "scr_metric_threshold_20": 0.36909885188142205, + "scr_dir2_threshold_20": 0.36909885188142205, + "scr_dir1_threshold_50": 0.3581395813037724, + "scr_metric_threshold_50": 0.4763948958977931, + "scr_dir2_threshold_50": 0.4763948958977931, + "scr_dir1_threshold_100": 0.4232558436107435, + "scr_metric_threshold_100": 0.3047209696576519, + "scr_dir2_threshold_100": 0.3047209696576519, + "scr_dir1_threshold_500": 0.3581395813037724, + "scr_metric_threshold_500": 0.29184559786405584, + "scr_dir2_threshold_500": 0.29184559786405584 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.18139525341818818, "scr_metric_threshold_10": 0.18139525341818818, "scr_dir2_threshold_10": 0.1040723994104265, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.17674414994201715, + "scr_metric_threshold_20": 0.17674414994201715, + "scr_dir2_threshold_20": 0.12669681061232305, + "scr_dir1_threshold_50": 0.232557946117949, + "scr_metric_threshold_50": 0.232557946117949, + "scr_dir2_threshold_50": 0.17647046131564434, + "scr_dir1_threshold_100": 0.2372093268250598, + "scr_metric_threshold_100": 0.2372093268250598, + "scr_dir2_threshold_100": 0.15837114811753172, + "scr_dir1_threshold_500": 0.3023256071842735, + "scr_metric_threshold_500": 0.3023256071842735, + "scr_dir2_threshold_500": 0.208144798820853 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-109863-tokens-450000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-109863-tokens-450000000/tpp/saebench_tpp_custom_sae_eval_results.json index ce7028cebb7f703fc4ea6f494336304ac06bda37..9507df98b3346c29c6c79cab79091b70965926e4 100644 --- a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-109863-tokens-450000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-109863-tokens-450000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "daa16980-fa02-4f79-afab-1051bc4edc8e", - "datetime_epoch_millis": 1776887569568, + "eval_id": "1228efd5-846b-43ba-ba78-cf3056bbb597", + "datetime_epoch_millis": 1777318845804, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.02265000492334366, - "tpp_threshold_2_intended_diff_only": 0.024800008535385134, - "tpp_threshold_2_unintended_diff_only": 0.0021500036120414736, - "tpp_threshold_5_total_metric": 0.06192500591278076, - "tpp_threshold_5_intended_diff_only": 0.07620001435279845, - "tpp_threshold_5_unintended_diff_only": 0.0142750084400177, - "tpp_threshold_10_total_metric": 0.11052500307559968, - "tpp_threshold_10_intended_diff_only": 0.15660001039505003, - "tpp_threshold_10_unintended_diff_only": 0.04607500731945038, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.0202750027179718, + "tpp_threshold_2_intended_diff_only": 0.023600006103515626, + "tpp_threshold_2_unintended_diff_only": 0.0033250033855438234, + "tpp_threshold_5_total_metric": 0.06080000251531601, + "tpp_threshold_5_intended_diff_only": 0.07450000643730163, + "tpp_threshold_5_unintended_diff_only": 0.013700003921985626, + "tpp_threshold_10_total_metric": 0.11032500118017197, + "tpp_threshold_10_intended_diff_only": 0.1568000078201294, + "tpp_threshold_10_unintended_diff_only": 0.046475006639957434, + "tpp_threshold_20_total_metric": 0.12547499537467957, + "tpp_threshold_20_intended_diff_only": 0.1900000035762787, + "tpp_threshold_20_unintended_diff_only": 0.06452500820159912, + "tpp_threshold_50_total_metric": 0.1330750107765198, + "tpp_threshold_50_intended_diff_only": 0.2401000201702118, + "tpp_threshold_50_unintended_diff_only": 0.10702500939369203, + "tpp_threshold_100_total_metric": 0.1345500037074089, + "tpp_threshold_100_intended_diff_only": 0.2809000134468078, + "tpp_threshold_100_unintended_diff_only": 0.14635000973939896, + "tpp_threshold_500_total_metric": 0.20392500907182692, + "tpp_threshold_500_intended_diff_only": 0.40100001692771914, + "tpp_threshold_500_unintended_diff_only": 0.1970750078558922 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.03864999413490296, - "tpp_threshold_2_intended_diff_only": 0.043000006675720216, - "tpp_threshold_2_unintended_diff_only": 0.004350012540817261, - "tpp_threshold_5_total_metric": 0.11275000870227814, - "tpp_threshold_5_intended_diff_only": 0.13960002660751342, - "tpp_threshold_5_unintended_diff_only": 0.02685001790523529, - "tpp_threshold_10_total_metric": 0.2054000049829483, + "tpp_threshold_2_total_metric": 0.036450010538101194, + "tpp_threshold_2_intended_diff_only": 0.04200001955032349, + "tpp_threshold_2_unintended_diff_only": 0.00555000901222229, + "tpp_threshold_5_total_metric": 0.11310000717639923, + "tpp_threshold_5_intended_diff_only": 0.14020001888275146, + "tpp_threshold_5_unintended_diff_only": 0.027100011706352234, + "tpp_threshold_10_total_metric": 0.20415000319480897, "tpp_threshold_10_intended_diff_only": 0.29400001764297484, - "tpp_threshold_10_unintended_diff_only": 0.08860001266002655, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_10_unintended_diff_only": 0.0898500144481659, + "tpp_threshold_20_total_metric": 0.22084999680519105, + "tpp_threshold_20_intended_diff_only": 0.3460000157356262, + "tpp_threshold_20_unintended_diff_only": 0.12515001893043518, + "tpp_threshold_50_total_metric": 0.19165002107620238, + "tpp_threshold_50_intended_diff_only": 0.40220004320144653, + "tpp_threshold_50_unintended_diff_only": 0.21055002212524415, + "tpp_threshold_100_total_metric": 0.14695001244544983, + "tpp_threshold_100_intended_diff_only": 0.42960003614425657, + "tpp_threshold_100_unintended_diff_only": 0.28265002369880676, + "tpp_threshold_500_total_metric": 0.09180000722408295, + "tpp_threshold_500_intended_diff_only": 0.46320003271102905, + "tpp_threshold_500_unintended_diff_only": 0.3714000254869461 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.006650015711784363, - "tpp_threshold_2_intended_diff_only": 0.006600010395050049, - "tpp_threshold_2_unintended_diff_only": -5.000531673431397e-05, - "tpp_threshold_5_total_metric": 0.011100003123283386, - "tpp_threshold_5_intended_diff_only": 0.012800002098083496, - "tpp_threshold_5_unintended_diff_only": 0.0016999989748001099, - "tpp_threshold_10_total_metric": 0.01565000116825104, - "tpp_threshold_10_intended_diff_only": 0.019200003147125243, - "tpp_threshold_10_unintended_diff_only": 0.0035500019788742066, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.004099994897842407, + "tpp_threshold_2_intended_diff_only": 0.005199992656707763, + "tpp_threshold_2_unintended_diff_only": 0.0010999977588653565, + "tpp_threshold_5_total_metric": 0.008499997854232787, + "tpp_threshold_5_intended_diff_only": 0.008799993991851806, + "tpp_threshold_5_unintended_diff_only": 0.0002999961376190186, + "tpp_threshold_10_total_metric": 0.016499999165534972, + "tpp_threshold_10_intended_diff_only": 0.019599997997283937, + "tpp_threshold_10_unintended_diff_only": 0.0030999988317489623, + "tpp_threshold_20_total_metric": 0.03009999394416809, + "tpp_threshold_20_intended_diff_only": 0.03399999141693115, + "tpp_threshold_20_unintended_diff_only": 0.0038999974727630614, + "tpp_threshold_50_total_metric": 0.07450000047683716, + "tpp_threshold_50_intended_diff_only": 0.07799999713897705, + "tpp_threshold_50_unintended_diff_only": 0.0034999966621398924, + "tpp_threshold_100_total_metric": 0.12214999496936799, + "tpp_threshold_100_intended_diff_only": 0.13219999074935912, + "tpp_threshold_100_unintended_diff_only": 0.01004999577999115, + "tpp_threshold_500_total_metric": 0.3160500109195709, + "tpp_threshold_500_intended_diff_only": 0.33880000114440917, + "tpp_threshold_500_unintended_diff_only": 0.022749990224838257 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.0015000104904174805, "tpp_threshold_10_total_metric": 0.2044999599456787, "tpp_threshold_10_intended_diff_only": 0.26499998569488525, - "tpp_threshold_10_unintended_diff_only": 0.06050002574920654 + "tpp_threshold_10_unintended_diff_only": 0.06050002574920654, + "tpp_threshold_20_total_metric": 0.2264999896287918, + "tpp_threshold_20_intended_diff_only": 0.3330000042915344, + "tpp_threshold_20_unintended_diff_only": 0.10650001466274261, + "tpp_threshold_50_total_metric": 0.19074998795986176, + "tpp_threshold_50_intended_diff_only": 0.3960000276565552, + "tpp_threshold_50_unintended_diff_only": 0.20525003969669342, + "tpp_threshold_100_total_metric": 0.1627499908208847, + "tpp_threshold_100_intended_diff_only": 0.42900002002716064, + "tpp_threshold_100_unintended_diff_only": 0.26625002920627594, + "tpp_threshold_500_total_metric": 0.09699995815753937, + "tpp_threshold_500_intended_diff_only": 0.45399999618530273, + "tpp_threshold_500_unintended_diff_only": 0.35700003802776337 }, "1": { - "tpp_threshold_2_total_metric": 0.14574997127056122, - "tpp_threshold_2_intended_diff_only": 0.17100000381469727, - "tpp_threshold_2_unintended_diff_only": 0.025250032544136047, - "tpp_threshold_5_total_metric": 0.15525001287460327, + "tpp_threshold_2_total_metric": 0.14500002562999725, + "tpp_threshold_2_intended_diff_only": 0.16800004243850708, + "tpp_threshold_2_unintended_diff_only": 0.023000016808509827, + "tpp_threshold_5_total_metric": 0.15725001692771912, "tpp_threshold_5_intended_diff_only": 0.18700003623962402, - "tpp_threshold_5_unintended_diff_only": 0.03175002336502075, - "tpp_threshold_10_total_metric": 0.1887499839067459, + "tpp_threshold_5_unintended_diff_only": 0.029750019311904907, + "tpp_threshold_10_total_metric": 0.18524998426437378, "tpp_threshold_10_intended_diff_only": 0.27799999713897705, - "tpp_threshold_10_unintended_diff_only": 0.08925001323223114 + "tpp_threshold_10_unintended_diff_only": 0.09275001287460327, + "tpp_threshold_20_total_metric": 0.21599999070167542, + "tpp_threshold_20_intended_diff_only": 0.3140000104904175, + "tpp_threshold_20_unintended_diff_only": 0.09800001978874207, + "tpp_threshold_50_total_metric": 0.1885000318288803, + "tpp_threshold_50_intended_diff_only": 0.362000048160553, + "tpp_threshold_50_unintended_diff_only": 0.17350001633167267, + "tpp_threshold_100_total_metric": 0.12550002336502075, + "tpp_threshold_100_intended_diff_only": 0.40400004386901855, + "tpp_threshold_100_unintended_diff_only": 0.2785000205039978, + "tpp_threshold_500_total_metric": 0.06850002706050873, + "tpp_threshold_500_intended_diff_only": 0.4540000557899475, + "tpp_threshold_500_unintended_diff_only": 0.3855000287294388 }, "2": { - "tpp_threshold_2_total_metric": 0.027250036597251892, - "tpp_threshold_2_intended_diff_only": 0.024000048637390137, - "tpp_threshold_2_unintended_diff_only": -0.0032499879598617554, - "tpp_threshold_5_total_metric": 0.22075003385543823, - "tpp_threshold_5_intended_diff_only": 0.2500000596046448, - "tpp_threshold_5_unintended_diff_only": 0.029250025749206543, - "tpp_threshold_10_total_metric": 0.20925003290176392, - "tpp_threshold_10_intended_diff_only": 0.3290000557899475, - "tpp_threshold_10_unintended_diff_only": 0.1197500228881836 + "tpp_threshold_2_total_metric": 0.016750037670135498, + "tpp_threshold_2_intended_diff_only": 0.020000040531158447, + "tpp_threshold_2_unintended_diff_only": 0.0032500028610229492, + "tpp_threshold_5_total_metric": 0.21950004994869232, + "tpp_threshold_5_intended_diff_only": 0.2510000467300415, + "tpp_threshold_5_unintended_diff_only": 0.03149999678134918, + "tpp_threshold_10_total_metric": 0.2107500284910202, + "tpp_threshold_10_intended_diff_only": 0.33000004291534424, + "tpp_threshold_10_unintended_diff_only": 0.11925001442432404, + "tpp_threshold_20_total_metric": 0.15299999713897705, + "tpp_threshold_20_intended_diff_only": 0.36400002241134644, + "tpp_threshold_20_unintended_diff_only": 0.21100002527236938, + "tpp_threshold_50_total_metric": 0.11100004613399506, + "tpp_threshold_50_intended_diff_only": 0.3840000629425049, + "tpp_threshold_50_unintended_diff_only": 0.2730000168085098, + "tpp_threshold_100_total_metric": 0.07150000333786011, + "tpp_threshold_100_intended_diff_only": 0.40000003576278687, + "tpp_threshold_100_unintended_diff_only": 0.32850003242492676, + "tpp_threshold_500_total_metric": 0.05250003933906555, + "tpp_threshold_500_intended_diff_only": 0.4490000605583191, + "tpp_threshold_500_unintended_diff_only": 0.39650002121925354 }, "6": { - "tpp_threshold_2_total_metric": 0.008000001311302185, + "tpp_threshold_2_total_metric": 0.0059999823570251465, "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": -0.003000006079673767, - "tpp_threshold_5_total_metric": 0.03500001132488251, - "tpp_threshold_5_intended_diff_only": 0.04500001668930054, - "tpp_threshold_5_unintended_diff_only": 0.01000000536441803, - "tpp_threshold_10_total_metric": 0.26225002110004425, + "tpp_threshold_2_unintended_diff_only": -0.0009999871253967285, + "tpp_threshold_5_total_metric": 0.03899998962879181, + "tpp_threshold_5_intended_diff_only": 0.046000003814697266, + "tpp_threshold_5_unintended_diff_only": 0.0070000141859054565, + "tpp_threshold_10_total_metric": 0.26250001788139343, "tpp_threshold_10_intended_diff_only": 0.36400002241134644, - "tpp_threshold_10_unintended_diff_only": 0.10175000131130219 + "tpp_threshold_10_unintended_diff_only": 0.101500004529953, + "tpp_threshold_20_total_metric": 0.29625000059604645, + "tpp_threshold_20_intended_diff_only": 0.43300002813339233, + "tpp_threshold_20_unintended_diff_only": 0.1367500275373459, + "tpp_threshold_50_total_metric": 0.25349999964237213, + "tpp_threshold_50_intended_diff_only": 0.47600001096725464, + "tpp_threshold_50_unintended_diff_only": 0.2225000113248825, + "tpp_threshold_100_total_metric": 0.1925000548362732, + "tpp_threshold_100_intended_diff_only": 0.48100006580352783, + "tpp_threshold_100_unintended_diff_only": 0.28850001096725464, + "tpp_threshold_500_total_metric": 0.1287500113248825, + "tpp_threshold_500_intended_diff_only": 0.484000027179718, + "tpp_threshold_500_unintended_diff_only": 0.3552500158548355 }, "9": { - "tpp_threshold_2_total_metric": 0.003000006079673767, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_5_total_metric": 0.1392500102519989, - "tpp_threshold_5_intended_diff_only": 0.20100003480911255, - "tpp_threshold_5_unintended_diff_only": 0.06175002455711365, - "tpp_threshold_10_total_metric": 0.16225002706050873, - "tpp_threshold_10_intended_diff_only": 0.23400002717971802, - "tpp_threshold_10_unintended_diff_only": 0.07175000011920929 + "tpp_threshold_2_total_metric": 0.0052500516176223755, + "tpp_threshold_2_intended_diff_only": 0.006000041961669922, + "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.13625000417232513, + "tpp_threshold_5_intended_diff_only": 0.20200002193450928, + "tpp_threshold_5_unintended_diff_only": 0.06575001776218414, + "tpp_threshold_10_total_metric": 0.15775002539157867, + "tpp_threshold_10_intended_diff_only": 0.2330000400543213, + "tpp_threshold_10_unintended_diff_only": 0.07525001466274261, + "tpp_threshold_20_total_metric": 0.21250000596046448, + "tpp_threshold_20_intended_diff_only": 0.28600001335144043, + "tpp_threshold_20_unintended_diff_only": 0.07350000739097595, + "tpp_threshold_50_total_metric": 0.2145000398159027, + "tpp_threshold_50_intended_diff_only": 0.393000066280365, + "tpp_threshold_50_unintended_diff_only": 0.17850002646446228, + "tpp_threshold_100_total_metric": 0.1824999898672104, + "tpp_threshold_100_intended_diff_only": 0.43400001525878906, + "tpp_threshold_100_unintended_diff_only": 0.2515000253915787, + "tpp_threshold_500_total_metric": 0.11225000023841858, + "tpp_threshold_500_intended_diff_only": 0.4750000238418579, + "tpp_threshold_500_unintended_diff_only": 0.36275002360343933 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.00925000011920929, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": -0.0052499920129776, - "tpp_threshold_5_total_metric": 0.010750025510787964, - "tpp_threshold_5_intended_diff_only": 0.008000016212463379, - "tpp_threshold_5_unintended_diff_only": -0.002750009298324585, - "tpp_threshold_10_total_metric": -0.0012499839067459106, - "tpp_threshold_10_intended_diff_only": 0.003000020980834961, - "tpp_threshold_10_unintended_diff_only": 0.004250004887580872 + "tpp_threshold_2_total_metric": 0.002750009298324585, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 0.002249985933303833, + "tpp_threshold_5_total_metric": 0.002500012516975403, + "tpp_threshold_5_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_5_unintended_diff_only": 0.0014999955892562866, + "tpp_threshold_10_total_metric": 0.002749994397163391, + "tpp_threshold_10_intended_diff_only": 0.004999995231628418, + "tpp_threshold_10_unintended_diff_only": 0.002250000834465027, + "tpp_threshold_20_total_metric": 0.00974997878074646, + "tpp_threshold_20_intended_diff_only": 0.010999977588653564, + "tpp_threshold_20_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_50_total_metric": 0.025249972939491272, + "tpp_threshold_50_intended_diff_only": 0.019999980926513672, + "tpp_threshold_50_unintended_diff_only": -0.0052499920129776, + "tpp_threshold_100_total_metric": 0.05000002682209015, + "tpp_threshold_100_intended_diff_only": 0.05500000715255737, + "tpp_threshold_100_unintended_diff_only": 0.004999980330467224, + "tpp_threshold_500_total_metric": 0.2632499933242798, + "tpp_threshold_500_intended_diff_only": 0.27399998903274536, + "tpp_threshold_500_unintended_diff_only": 0.010749995708465576 }, "2": { - "tpp_threshold_2_total_metric": 0.0005000680685043335, - "tpp_threshold_2_intended_diff_only": 0.005000054836273193, - "tpp_threshold_2_unintended_diff_only": 0.00449998676776886, - "tpp_threshold_5_total_metric": 0.01100003719329834, - "tpp_threshold_5_intended_diff_only": 0.016000032424926758, - "tpp_threshold_5_unintended_diff_only": 0.004999995231628418, - "tpp_threshold_10_total_metric": 0.026750028133392334, - "tpp_threshold_10_intended_diff_only": 0.03100001811981201, - "tpp_threshold_10_unintended_diff_only": 0.004249989986419678 + "tpp_threshold_2_total_metric": 0.01000000536441803, + "tpp_threshold_2_intended_diff_only": 0.009000003337860107, + "tpp_threshold_2_unintended_diff_only": -0.0010000020265579224, + "tpp_threshold_5_total_metric": 0.006750017404556274, + "tpp_threshold_5_intended_diff_only": 0.013000011444091797, + "tpp_threshold_5_unintended_diff_only": 0.0062499940395355225, + "tpp_threshold_10_total_metric": 0.027500063180923462, + "tpp_threshold_10_intended_diff_only": 0.033000051975250244, + "tpp_threshold_10_unintended_diff_only": 0.005499988794326782, + "tpp_threshold_20_total_metric": 0.05725003778934479, + "tpp_threshold_20_intended_diff_only": 0.058000028133392334, + "tpp_threshold_20_unintended_diff_only": 0.0007499903440475464, + "tpp_threshold_50_total_metric": 0.12425006926059723, + "tpp_threshold_50_intended_diff_only": 0.1350000500679016, + "tpp_threshold_50_unintended_diff_only": 0.010749980807304382, + "tpp_threshold_100_total_metric": 0.1705000102519989, + "tpp_threshold_100_intended_diff_only": 0.18900001049041748, + "tpp_threshold_100_unintended_diff_only": 0.01850000023841858, + "tpp_threshold_500_total_metric": 0.3680000454187393, + "tpp_threshold_500_intended_diff_only": 0.3960000276565552, + "tpp_threshold_500_unintended_diff_only": 0.027999982237815857 }, "3": { - "tpp_threshold_2_total_metric": 0.010750025510787964, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": -0.006750017404556274, - "tpp_threshold_5_total_metric": 0.015500009059906006, - "tpp_threshold_5_intended_diff_only": 0.013000011444091797, - "tpp_threshold_5_unintended_diff_only": -0.002499997615814209, - "tpp_threshold_10_total_metric": 0.01725001633167267, - "tpp_threshold_10_intended_diff_only": 0.021000027656555176, - "tpp_threshold_10_unintended_diff_only": 0.0037500113248825073 + "tpp_threshold_2_total_metric": 0.000250011682510376, + "tpp_threshold_2_intended_diff_only": 0.0, + "tpp_threshold_2_unintended_diff_only": -0.000250011682510376, + "tpp_threshold_5_total_metric": 0.007500007748603821, + "tpp_threshold_5_intended_diff_only": 0.0009999871253967285, + "tpp_threshold_5_unintended_diff_only": -0.006500020623207092, + "tpp_threshold_10_total_metric": 0.012749969959259033, + "tpp_threshold_10_intended_diff_only": 0.010999977588653564, + "tpp_threshold_10_unintended_diff_only": -0.0017499923706054688, + "tpp_threshold_20_total_metric": 0.008250012993812561, + "tpp_threshold_20_intended_diff_only": 0.013999998569488525, + "tpp_threshold_20_unintended_diff_only": 0.005749985575675964, + "tpp_threshold_50_total_metric": 0.04375000298023224, + "tpp_threshold_50_intended_diff_only": 0.041999995708465576, + "tpp_threshold_50_unintended_diff_only": -0.0017500072717666626, + "tpp_threshold_100_total_metric": 0.07499997317790985, + "tpp_threshold_100_intended_diff_only": 0.08499997854232788, + "tpp_threshold_100_unintended_diff_only": 0.01000000536441803, + "tpp_threshold_500_total_metric": 0.26775000989437103, + "tpp_threshold_500_intended_diff_only": 0.2879999876022339, + "tpp_threshold_500_unintended_diff_only": 0.020249977707862854 }, "5": { - "tpp_threshold_2_total_metric": -0.0047500282526016235, - "tpp_threshold_2_intended_diff_only": 0.001999974250793457, - "tpp_threshold_2_unintended_diff_only": 0.006750002503395081, - "tpp_threshold_5_total_metric": 0.006749957799911499, - "tpp_threshold_5_intended_diff_only": 0.007999956607818604, - "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045, - "tpp_threshold_10_total_metric": 0.014999940991401672, - "tpp_threshold_10_intended_diff_only": 0.01799994707107544, - "tpp_threshold_10_unintended_diff_only": 0.003000006079673767 + "tpp_threshold_2_total_metric": 0.0027499496936798096, + "tpp_threshold_2_intended_diff_only": 0.0029999613761901855, + "tpp_threshold_2_unintended_diff_only": 0.000250011682510376, + "tpp_threshold_5_total_metric": 0.014999985694885254, + "tpp_threshold_5_intended_diff_only": 0.013999998569488525, + "tpp_threshold_5_unintended_diff_only": -0.0009999871253967285, + "tpp_threshold_10_total_metric": 0.007749959826469421, + "tpp_threshold_10_intended_diff_only": 0.012999951839447021, + "tpp_threshold_10_unintended_diff_only": 0.0052499920129776, + "tpp_threshold_20_total_metric": 0.01874995231628418, + "tpp_threshold_20_intended_diff_only": 0.0209999680519104, + "tpp_threshold_20_unintended_diff_only": 0.0022500157356262207, + "tpp_threshold_50_total_metric": 0.046999961137771606, + "tpp_threshold_50_intended_diff_only": 0.05399996042251587, + "tpp_threshold_50_unintended_diff_only": 0.006999999284744263, + "tpp_threshold_100_total_metric": 0.1304999589920044, + "tpp_threshold_100_intended_diff_only": 0.1369999647140503, + "tpp_threshold_100_unintended_diff_only": 0.0065000057220458984, + "tpp_threshold_500_total_metric": 0.3427499979734421, + "tpp_threshold_500_intended_diff_only": 0.3700000047683716, + "tpp_threshold_500_unintended_diff_only": 0.027250006794929504 }, "6": { - "tpp_threshold_2_total_metric": 0.01750001311302185, - "tpp_threshold_2_intended_diff_only": 0.018000006675720215, - "tpp_threshold_2_unintended_diff_only": 0.0004999935626983643, - "tpp_threshold_5_total_metric": 0.011499986052513123, - "tpp_threshold_5_intended_diff_only": 0.018999993801116943, - "tpp_threshold_5_unintended_diff_only": 0.007500007748603821, - "tpp_threshold_10_total_metric": 0.020500004291534424, - "tpp_threshold_10_intended_diff_only": 0.023000001907348633, - "tpp_threshold_10_unintended_diff_only": 0.002499997615814209 + "tpp_threshold_2_total_metric": 0.004749998450279236, + "tpp_threshold_2_intended_diff_only": 0.009000003337860107, + "tpp_threshold_2_unintended_diff_only": 0.004250004887580872, + "tpp_threshold_5_total_metric": 0.010749965906143188, + "tpp_threshold_5_intended_diff_only": 0.011999964714050293, + "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_10_total_metric": 0.03175000846385956, + "tpp_threshold_10_intended_diff_only": 0.03600001335144043, + "tpp_threshold_10_unintended_diff_only": 0.004250004887580872, + "tpp_threshold_20_total_metric": 0.056499987840652466, + "tpp_threshold_20_intended_diff_only": 0.06599998474121094, + "tpp_threshold_20_unintended_diff_only": 0.009499996900558472, + "tpp_threshold_50_total_metric": 0.13224999606609344, + "tpp_threshold_50_intended_diff_only": 0.13899999856948853, + "tpp_threshold_50_unintended_diff_only": 0.006750002503395081, + "tpp_threshold_100_total_metric": 0.1847500056028366, + "tpp_threshold_100_intended_diff_only": 0.19499999284744263, + "tpp_threshold_100_unintended_diff_only": 0.010249987244606018, + "tpp_threshold_500_total_metric": 0.3385000079870224, + "tpp_threshold_500_intended_diff_only": 0.3659999966621399, + "tpp_threshold_500_unintended_diff_only": 0.027499988675117493 } } } diff --git a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-12207-tokens-50000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-12207-tokens-50000000/scr/saebench_scr_custom_sae_eval_results.json index 77a616ecde00cd7e9a79fd8c266e7c8d675daacf..735af14af330e443462775803acd2754d49d5e5e 100644 --- a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-12207-tokens-50000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-12207-tokens-50000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "6f6f535d-66af-426d-be24-909ae593447d", - "datetime_epoch_millis": 1776887789363, + "eval_id": "95f7f2bb-1d91-4e01-b002-7d793f19b2f9", + "datetime_epoch_millis": 1777318602450, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.18370246757936048, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.21218547693883402, "scr_metric_threshold_10": 0.24100529054377431, "scr_dir2_threshold_10": 0.24746115803013294, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.17699851173335074, + "scr_metric_threshold_20": 0.30434652765636866, + "scr_dir2_threshold_20": 0.3134410309554935, + "scr_dir1_threshold_50": 0.16874851796624074, + "scr_metric_threshold_50": 0.37716460703126997, + "scr_dir2_threshold_50": 0.38724825594988266, + "scr_dir1_threshold_100": 0.04353196451639832, + "scr_metric_threshold_100": 0.3519082414514541, + "scr_dir2_threshold_100": 0.3701156410178882, + "scr_dir1_threshold_500": 0.003937524428453562, + "scr_metric_threshold_500": 0.3082370445595443, + "scr_dir2_threshold_500": 0.32867535690764565 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.5000003820810118, "scr_metric_threshold_10": 0.09585491427895527, "scr_dir2_threshold_10": 0.09585491427895527, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.5000003820810118, + "scr_metric_threshold_20": 0.2124352571631342, + "scr_dir2_threshold_20": 0.2124352571631342, + "scr_dir1_threshold_50": 0.39743608357792887, + "scr_metric_threshold_50": 0.25129531998580606, + "scr_dir2_threshold_50": 0.25129531998580606, + "scr_dir1_threshold_100": -0.1282049910478419, + "scr_metric_threshold_100": 0.2694299542032543, + "scr_dir2_threshold_100": 0.2694299542032543, + "scr_dir1_threshold_500": -0.46153781493982604, + "scr_metric_threshold_500": 0.2720207485910297, + "scr_dir2_threshold_500": 0.2720207485910297 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.16666674192503994, "scr_metric_threshold_10": 0.3883792823747072, "scr_dir2_threshold_10": 0.3883792823747072, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.24242422874090183, + "scr_metric_threshold_20": 0.4617737393253921, + "scr_dir2_threshold_20": 0.4617737393253921, + "scr_dir1_threshold_50": -0.28030297214883276, + "scr_metric_threshold_50": 0.49235474786507843, + "scr_dir2_threshold_50": 0.49235474786507843, + "scr_dir1_threshold_100": -0.42424228740901826, + "scr_metric_threshold_100": 0.10091740109184036, + "scr_dir2_threshold_100": 0.10091740109184036, + "scr_dir1_threshold_500": 0.07575748681586188, + "scr_metric_threshold_500": 0.19266060898808818, + "scr_dir2_threshold_500": 0.19266060898808818 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.31249965075409175, "scr_metric_threshold_10": 0.10804029093651613, "scr_dir2_threshold_10": 0.10804029093651613, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.32812537834973393, + "scr_metric_threshold_20": 0.13316589123232284, + "scr_dir2_threshold_20": 0.13316589123232284, + "scr_dir1_threshold_50": 0.45312467985791743, + "scr_metric_threshold_50": 0.2211054922676464, + "scr_dir2_threshold_50": 0.2211054922676464, + "scr_dir1_threshold_100": 0.4062502910382569, + "scr_metric_threshold_100": 0.27386937269715317, + "scr_dir2_threshold_100": 0.27386937269715317, + "scr_dir1_threshold_500": 0.17187462165026607, + "scr_metric_threshold_500": 0.13065336115482035, + "scr_dir2_threshold_500": 0.13065336115482035 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": 0.24832216912919208, "scr_metric_threshold_10": 0.15286635689969297, "scr_dir2_threshold_10": 0.15286635689969297, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.19463078120093366, + "scr_metric_threshold_20": 0.20382168265045522, + "scr_dir2_threshold_20": 0.20382168265045522, + "scr_dir1_threshold_50": 0.26845613957888714, + "scr_metric_threshold_50": 0.3630573840851048, + "scr_dir2_threshold_50": 0.3630573840851048, + "scr_dir1_threshold_100": 0.3087248805406822, + "scr_metric_threshold_100": 0.45859869105167267, + "scr_dir2_threshold_100": 0.45859869105167267, + "scr_dir1_threshold_500": -0.02013437048089751, + "scr_metric_threshold_500": 0.12101906475380056, + "scr_dir2_threshold_500": 0.12101906475380056 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.0631580235225028, "scr_metric_threshold_10": 0.46428558757517696, "scr_dir2_threshold_10": 0.46428558757517696, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.057894933322792726, + "scr_metric_threshold_20": 0.49603167939138937, + "scr_dir2_threshold_20": 0.49603167939138937, + "scr_dir1_threshold_50": -0.1947368470585578, + "scr_metric_threshold_50": 0.567460267714699, + "scr_dir2_threshold_50": 0.567460267714699, + "scr_dir1_threshold_100": -0.16842108235134678, + "scr_metric_threshold_100": 0.6150794054390177, + "scr_dir2_threshold_100": 0.6150794054390177, + "scr_dir1_threshold_500": -0.2210526117657688, + "scr_metric_threshold_500": 0.654761901946115, + "scr_dir2_threshold_500": 0.654761901946115 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.20218582972611174, "scr_metric_threshold_10": 0.11450392446499717, "scr_dir2_threshold_10": 0.11450392446499717, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2568305957534126, + "scr_metric_threshold_20": 0.19465646684173443, + "scr_dir2_threshold_20": 0.19465646684173443, + "scr_dir1_threshold_50": 0.3060109503196829, + "scr_metric_threshold_50": 0.2786259811162493, + "scr_dir2_threshold_50": 0.2786259811162493, + "scr_dir1_threshold_100": 0.3169397732417441, + "scr_metric_threshold_100": 0.33587782959943635, + "scr_dir2_threshold_100": 0.33587782959943635, + "scr_dir1_threshold_500": 0.3005465388586523, + "scr_metric_threshold_500": 0.39694664998040097, + "scr_dir2_threshold_500": 0.39694664998040097 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.10697673644937127, "scr_metric_threshold_10": 0.506437685896798, "scr_dir2_threshold_10": 0.506437685896798, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.16744189656960076, + "scr_metric_threshold_20": 0.5793991493162989, + "scr_dir2_threshold_20": 0.5793991493162989, + "scr_dir1_threshold_50": 0.20465126852525933, + "scr_metric_threshold_50": 0.6480688221379344, + "scr_dir2_threshold_50": 0.6480688221379344, + "scr_dir1_threshold_100": -0.09767453207588807, + "scr_metric_threshold_100": 0.6266096133346601, + "scr_dir2_threshold_100": 0.6266096133346601, + "scr_dir1_threshold_500": 0.04186047414240019, + "scr_metric_threshold_500": 0.5536481499151593, + "scr_dir2_threshold_500": 0.5536481499151593 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.09767428192335063, "scr_metric_threshold_10": 0.09767428192335063, "scr_dir2_threshold_10": 0.1493212218142196, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.15348835533022226, + "scr_metric_threshold_20": 0.15348835533022226, + "scr_dir2_threshold_20": 0.2262443817232214, + "scr_dir1_threshold_50": 0.19534884107764103, + "scr_metric_threshold_50": 0.19534884107764103, + "scr_dir2_threshold_50": 0.27601803242654266, + "scr_dir1_threshold_100": 0.13488366419459838, + "scr_metric_threshold_100": 0.13488366419459838, + "scr_dir2_threshold_100": 0.2805428607260708, + "scr_dir1_threshold_500": 0.14418587114694043, + "scr_metric_threshold_500": 0.14418587114694043, + "scr_dir2_threshold_500": 0.30769236993175136 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-12207-tokens-50000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-12207-tokens-50000000/tpp/saebench_tpp_custom_sae_eval_results.json index b4aeb0900fdd07e7c3061c3ac6ba3c87d4492325..c5f59bd02e68549fe728b96a44b7a3892b12a28e 100644 --- a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-12207-tokens-50000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-12207-tokens-50000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "d8b0d4b2-519d-470b-aa91-d07175f168e1", - "datetime_epoch_millis": 1776887942966, + "eval_id": "e65bf5d3-3269-4794-96ab-6e88c68e5f54", + "datetime_epoch_millis": 1777318875537, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.006724984943866729, - "tpp_threshold_2_intended_diff_only": 0.006599992513656616, - "tpp_threshold_2_unintended_diff_only": -0.00012499243021011353, - "tpp_threshold_5_total_metric": 0.009149989485740662, - "tpp_threshold_5_intended_diff_only": 0.011299997568130493, - "tpp_threshold_5_unintended_diff_only": 0.0021500080823898316, - "tpp_threshold_10_total_metric": 0.013549995422363282, - "tpp_threshold_10_intended_diff_only": 0.016299998760223387, - "tpp_threshold_10_unintended_diff_only": 0.0027500033378601075, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.005474998056888581, + "tpp_threshold_2_intended_diff_only": 0.006300002336502075, + "tpp_threshold_2_unintended_diff_only": 0.0008250042796134949, + "tpp_threshold_5_total_metric": 0.009174996614456176, + "tpp_threshold_5_intended_diff_only": 0.009899997711181642, + "tpp_threshold_5_unintended_diff_only": 0.0007250010967254639, + "tpp_threshold_10_total_metric": 0.015074992179870607, + "tpp_threshold_10_intended_diff_only": 0.017699998617172242, + "tpp_threshold_10_unintended_diff_only": 0.002625006437301636, + "tpp_threshold_20_total_metric": 0.028049996495246886, + "tpp_threshold_20_intended_diff_only": 0.03190000057220459, + "tpp_threshold_20_unintended_diff_only": 0.003850004076957703, + "tpp_threshold_50_total_metric": 0.06009999960660934, + "tpp_threshold_50_intended_diff_only": 0.06399999856948853, + "tpp_threshold_50_unintended_diff_only": 0.003899998962879181, + "tpp_threshold_100_total_metric": 0.11219999939203262, + "tpp_threshold_100_intended_diff_only": 0.11870000362396241, + "tpp_threshold_100_unintended_diff_only": 0.006500004231929779, + "tpp_threshold_500_total_metric": 0.30187501907348635, + "tpp_threshold_500_intended_diff_only": 0.3114000201225281, + "tpp_threshold_500_unintended_diff_only": 0.009525001049041748 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.007049983739852906, - "tpp_threshold_2_intended_diff_only": 0.006599998474121094, - "tpp_threshold_2_unintended_diff_only": -0.0004499852657318115, - "tpp_threshold_5_total_metric": 0.007549989223480225, - "tpp_threshold_5_intended_diff_only": 0.009400010108947754, - "tpp_threshold_5_unintended_diff_only": 0.0018500208854675293, - "tpp_threshold_10_total_metric": 0.012199997901916504, - "tpp_threshold_10_intended_diff_only": 0.014400005340576172, - "tpp_threshold_10_unintended_diff_only": 0.002200007438659668, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.006249997019767761, + "tpp_threshold_2_intended_diff_only": 0.007000005245208741, + "tpp_threshold_2_unintended_diff_only": 0.000750008225440979, + "tpp_threshold_5_total_metric": 0.00769999623298645, + "tpp_threshold_5_intended_diff_only": 0.008600008487701417, + "tpp_threshold_5_unintended_diff_only": 0.0009000122547149659, + "tpp_threshold_10_total_metric": 0.012249985337257385, + "tpp_threshold_10_intended_diff_only": 0.014600002765655517, + "tpp_threshold_10_unintended_diff_only": 0.0023500174283981323, + "tpp_threshold_20_total_metric": 0.030449992418289183, + "tpp_threshold_20_intended_diff_only": 0.03480000495910644, + "tpp_threshold_20_unintended_diff_only": 0.004350012540817261, + "tpp_threshold_50_total_metric": 0.06339999437332153, + "tpp_threshold_50_intended_diff_only": 0.0690000057220459, + "tpp_threshold_50_unintended_diff_only": 0.005600011348724366, + "tpp_threshold_100_total_metric": 0.1287500023841858, + "tpp_threshold_100_intended_diff_only": 0.1342000126838684, + "tpp_threshold_100_unintended_diff_only": 0.005450010299682617, + "tpp_threshold_500_total_metric": 0.34585002064704895, + "tpp_threshold_500_intended_diff_only": 0.3552000284194946, + "tpp_threshold_500_unintended_diff_only": 0.009350007772445679 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.006399986147880554, - "tpp_threshold_2_intended_diff_only": 0.006599986553192138, - "tpp_threshold_2_unintended_diff_only": 0.00020000040531158446, - "tpp_threshold_5_total_metric": 0.010749989748001098, - "tpp_threshold_5_intended_diff_only": 0.013199985027313232, - "tpp_threshold_5_unintended_diff_only": 0.0024499952793121337, - "tpp_threshold_10_total_metric": 0.014899992942810058, - "tpp_threshold_10_intended_diff_only": 0.018199992179870606, - "tpp_threshold_10_unintended_diff_only": 0.003299999237060547, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.004699999094009399, + "tpp_threshold_2_intended_diff_only": 0.00559999942779541, + "tpp_threshold_2_unintended_diff_only": 0.0009000003337860107, + "tpp_threshold_5_total_metric": 0.010649996995925903, + "tpp_threshold_5_intended_diff_only": 0.011199986934661866, + "tpp_threshold_5_unintended_diff_only": 0.0005499899387359619, + "tpp_threshold_10_total_metric": 0.017899999022483827, + "tpp_threshold_10_intended_diff_only": 0.020799994468688965, + "tpp_threshold_10_unintended_diff_only": 0.002899995446205139, + "tpp_threshold_20_total_metric": 0.02565000057220459, + "tpp_threshold_20_intended_diff_only": 0.028999996185302735, + "tpp_threshold_20_unintended_diff_only": 0.0033499956130981444, + "tpp_threshold_50_total_metric": 0.05680000483989715, + "tpp_threshold_50_intended_diff_only": 0.05899999141693115, + "tpp_threshold_50_unintended_diff_only": 0.0021999865770339967, + "tpp_threshold_100_total_metric": 0.09564999639987945, + "tpp_threshold_100_intended_diff_only": 0.1031999945640564, + "tpp_threshold_100_unintended_diff_only": 0.007549998164176941, + "tpp_threshold_500_total_metric": 0.2579000174999237, + "tpp_threshold_500_intended_diff_only": 0.26760001182556153, + "tpp_threshold_500_unintended_diff_only": 0.009699994325637817 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.0015000104904174805, "tpp_threshold_10_total_metric": 0.005749940872192383, "tpp_threshold_10_intended_diff_only": 0.007999956607818604, - "tpp_threshold_10_unintended_diff_only": 0.0022500157356262207 + "tpp_threshold_10_unintended_diff_only": 0.0022500157356262207, + "tpp_threshold_20_total_metric": 0.04074995219707489, + "tpp_threshold_20_intended_diff_only": 0.05299997329711914, + "tpp_threshold_20_unintended_diff_only": 0.01225002110004425, + "tpp_threshold_50_total_metric": 0.07924994826316833, + "tpp_threshold_50_intended_diff_only": 0.09499996900558472, + "tpp_threshold_50_unintended_diff_only": 0.015750020742416382, + "tpp_threshold_100_total_metric": 0.15999995172023773, + "tpp_threshold_100_intended_diff_only": 0.17299997806549072, + "tpp_threshold_100_unintended_diff_only": 0.01300002634525299, + "tpp_threshold_500_total_metric": 0.3642500042915344, + "tpp_threshold_500_intended_diff_only": 0.3830000162124634, + "tpp_threshold_500_unintended_diff_only": 0.018750011920928955 }, "1": { - "tpp_threshold_2_total_metric": 0.00024996697902679443, - "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 0.0007500201463699341, - "tpp_threshold_5_total_metric": 0.00424996018409729, - "tpp_threshold_5_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_5_unintended_diff_only": 0.0017500221729278564, - "tpp_threshold_10_total_metric": 0.00925000011920929, - "tpp_threshold_10_intended_diff_only": 0.013999998569488525, - "tpp_threshold_10_unintended_diff_only": 0.004749998450279236 + "tpp_threshold_2_total_metric": 0.0052499920129776, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": -0.00024999678134918213, + "tpp_threshold_5_total_metric": -0.00025004148483276367, + "tpp_threshold_5_intended_diff_only": 0.0009999871253967285, + "tpp_threshold_5_unintended_diff_only": 0.0012500286102294922, + "tpp_threshold_10_total_metric": 0.01400001347064972, + "tpp_threshold_10_intended_diff_only": 0.016000032424926758, + "tpp_threshold_10_unintended_diff_only": 0.0020000189542770386, + "tpp_threshold_20_total_metric": 0.018749967217445374, + "tpp_threshold_20_intended_diff_only": 0.02399998903274536, + "tpp_threshold_20_unintended_diff_only": 0.005250021815299988, + "tpp_threshold_50_total_metric": 0.06650000810623169, + "tpp_threshold_50_intended_diff_only": 0.07200002670288086, + "tpp_threshold_50_unintended_diff_only": 0.00550001859664917, + "tpp_threshold_100_total_metric": 0.13349996507167816, + "tpp_threshold_100_intended_diff_only": 0.13999998569488525, + "tpp_threshold_100_unintended_diff_only": 0.006500020623207092, + "tpp_threshold_500_total_metric": 0.3892500251531601, + "tpp_threshold_500_intended_diff_only": 0.40000003576278687, + "tpp_threshold_500_unintended_diff_only": 0.01075001060962677 }, "2": { - "tpp_threshold_2_total_metric": 0.016499996185302734, - "tpp_threshold_2_intended_diff_only": 0.013000011444091797, - "tpp_threshold_2_unintended_diff_only": -0.0034999847412109375, - "tpp_threshold_5_total_metric": 0.021000012755393982, - "tpp_threshold_5_intended_diff_only": 0.021000027656555176, - "tpp_threshold_5_unintended_diff_only": 1.4901161193847656e-08, - "tpp_threshold_10_total_metric": 0.02575002610683441, - "tpp_threshold_10_intended_diff_only": 0.025000035762786865, - "tpp_threshold_10_unintended_diff_only": -0.0007499903440475464 + "tpp_threshold_2_total_metric": 0.008000046014785767, + "tpp_threshold_2_intended_diff_only": 0.010000050067901611, + "tpp_threshold_2_unintended_diff_only": 0.0020000040531158447, + "tpp_threshold_5_total_metric": 0.016500011086463928, + "tpp_threshold_5_intended_diff_only": 0.017000019550323486, + "tpp_threshold_5_unintended_diff_only": 0.0005000084638595581, + "tpp_threshold_10_total_metric": 0.024750009179115295, + "tpp_threshold_10_intended_diff_only": 0.027000010013580322, + "tpp_threshold_10_unintended_diff_only": 0.002250000834465027, + "tpp_threshold_20_total_metric": 0.045750007033348083, + "tpp_threshold_20_intended_diff_only": 0.046000003814697266, + "tpp_threshold_20_unintended_diff_only": 0.00024999678134918213, + "tpp_threshold_50_total_metric": 0.06850001215934753, + "tpp_threshold_50_intended_diff_only": 0.06800001859664917, + "tpp_threshold_50_unintended_diff_only": -0.0004999935626983643, + "tpp_threshold_100_total_metric": 0.09050002694129944, + "tpp_threshold_100_intended_diff_only": 0.09000003337860107, + "tpp_threshold_100_unintended_diff_only": -0.0004999935626983643, + "tpp_threshold_500_total_metric": 0.278750017285347, + "tpp_threshold_500_intended_diff_only": 0.281000018119812, + "tpp_threshold_500_unintended_diff_only": 0.002250000834465027 }, "6": { - "tpp_threshold_2_total_metric": 0.007249996066093445, + "tpp_threshold_2_total_metric": 0.005249977111816406, "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": -0.002250000834465027, - "tpp_threshold_5_total_metric": -0.0014999955892562866, - "tpp_threshold_5_intended_diff_only": 0.0020000338554382324, - "tpp_threshold_5_unintended_diff_only": 0.003500029444694519, - "tpp_threshold_10_total_metric": 0.00599999725818634, - "tpp_threshold_10_intended_diff_only": 0.009000003337860107, - "tpp_threshold_10_unintended_diff_only": 0.003000006079673767 + "tpp_threshold_2_unintended_diff_only": -0.0002499818801879883, + "tpp_threshold_5_total_metric": 0.005000010132789612, + "tpp_threshold_5_intended_diff_only": 0.003000020980834961, + "tpp_threshold_5_unintended_diff_only": -0.001999989151954651, + "tpp_threshold_10_total_metric": 0.008499965071678162, + "tpp_threshold_10_intended_diff_only": 0.009999990463256836, + "tpp_threshold_10_unintended_diff_only": 0.0015000253915786743, + "tpp_threshold_20_total_metric": 0.01225002110004425, + "tpp_threshold_20_intended_diff_only": 0.012000024318695068, + "tpp_threshold_20_unintended_diff_only": -0.00024999678134918213, + "tpp_threshold_50_total_metric": 0.021499991416931152, + "tpp_threshold_50_intended_diff_only": 0.02399998903274536, + "tpp_threshold_50_unintended_diff_only": 0.002499997615814209, + "tpp_threshold_100_total_metric": 0.04125003516674042, + "tpp_threshold_100_intended_diff_only": 0.04400002956390381, + "tpp_threshold_100_unintended_diff_only": 0.002749994397163391, + "tpp_threshold_500_total_metric": 0.2500000447034836, + "tpp_threshold_500_intended_diff_only": 0.2510000467300415, + "tpp_threshold_500_unintended_diff_only": 0.0010000020265579224 }, "9": { - "tpp_threshold_2_total_metric": 0.0015000104904174805, - "tpp_threshold_2_intended_diff_only": 0.003000020980834961, - "tpp_threshold_2_unintended_diff_only": 0.0015000104904174805, - "tpp_threshold_5_total_metric": 0.0004999935626983643, - "tpp_threshold_5_intended_diff_only": 0.003000020980834961, - "tpp_threshold_5_unintended_diff_only": 0.0025000274181365967, - "tpp_threshold_10_total_metric": 0.014250025153160095, - "tpp_threshold_10_intended_diff_only": 0.016000032424926758, - "tpp_threshold_10_unintended_diff_only": 0.0017500072717666626 + "tpp_threshold_2_total_metric": 0.003000020980834961, + "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_2_unintended_diff_only": 0.0009999871253967285, + "tpp_threshold_5_total_metric": 0.003750026226043701, + "tpp_threshold_5_intended_diff_only": 0.00700002908706665, + "tpp_threshold_5_unintended_diff_only": 0.0032500028610229492, + "tpp_threshold_10_total_metric": 0.008249998092651367, + "tpp_threshold_10_intended_diff_only": 0.012000024318695068, + "tpp_threshold_10_unintended_diff_only": 0.003750026226043701, + "tpp_threshold_20_total_metric": 0.034750014543533325, + "tpp_threshold_20_intended_diff_only": 0.03900003433227539, + "tpp_threshold_20_unintended_diff_only": 0.004250019788742065, + "tpp_threshold_50_total_metric": 0.08125001192092896, + "tpp_threshold_50_intended_diff_only": 0.08600002527236938, + "tpp_threshold_50_unintended_diff_only": 0.00475001335144043, + "tpp_threshold_100_total_metric": 0.2185000330209732, + "tpp_threshold_100_intended_diff_only": 0.22400003671646118, + "tpp_threshold_100_unintended_diff_only": 0.005500003695487976, + "tpp_threshold_500_total_metric": 0.44700001180171967, + "tpp_threshold_500_intended_diff_only": 0.4610000252723694, + "tpp_threshold_500_unintended_diff_only": 0.01400001347064972 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.011499971151351929, - "tpp_threshold_2_intended_diff_only": 0.006999969482421875, - "tpp_threshold_2_unintended_diff_only": -0.004500001668930054, - "tpp_threshold_5_total_metric": 0.012999966740608215, - "tpp_threshold_5_intended_diff_only": 0.010999977588653564, - "tpp_threshold_5_unintended_diff_only": -0.001999989151954651, - "tpp_threshold_10_total_metric": 0.001749962568283081, - "tpp_threshold_10_intended_diff_only": 0.006999969482421875, - "tpp_threshold_10_unintended_diff_only": 0.005250006914138794 + "tpp_threshold_2_total_metric": 0.005250006914138794, + "tpp_threshold_2_intended_diff_only": 0.009000003337860107, + "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_5_total_metric": 0.002750009298324585, + "tpp_threshold_5_intended_diff_only": 0.004999995231628418, + "tpp_threshold_5_unintended_diff_only": 0.002249985933303833, + "tpp_threshold_10_total_metric": 0.002499997615814209, + "tpp_threshold_10_intended_diff_only": 0.004999995231628418, + "tpp_threshold_10_unintended_diff_only": 0.002499997615814209, + "tpp_threshold_20_total_metric": 0.012249976396560669, + "tpp_threshold_20_intended_diff_only": 0.015999972820281982, + "tpp_threshold_20_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_50_total_metric": 0.027500003576278687, + "tpp_threshold_50_intended_diff_only": 0.02399998903274536, + "tpp_threshold_50_unintended_diff_only": -0.003500014543533325, + "tpp_threshold_100_total_metric": 0.0587499737739563, + "tpp_threshold_100_intended_diff_only": 0.06199997663497925, + "tpp_threshold_100_unintended_diff_only": 0.0032500028610229492, + "tpp_threshold_500_total_metric": 0.24174998700618744, + "tpp_threshold_500_intended_diff_only": 0.2459999918937683, + "tpp_threshold_500_unintended_diff_only": 0.004250004887580872 }, "2": { - "tpp_threshold_2_total_metric": -0.006749987602233887, - "tpp_threshold_2_intended_diff_only": -0.0009999871253967285, - "tpp_threshold_2_unintended_diff_only": 0.005750000476837158, - "tpp_threshold_5_total_metric": 0.0002500563859939575, - "tpp_threshold_5_intended_diff_only": 0.00700002908706665, - "tpp_threshold_5_unintended_diff_only": 0.006749972701072693, - "tpp_threshold_10_total_metric": 0.009000048041343689, - "tpp_threshold_10_intended_diff_only": 0.01100003719329834, - "tpp_threshold_10_unintended_diff_only": 0.001999989151954651 + "tpp_threshold_2_total_metric": 0.004250034689903259, + "tpp_threshold_2_intended_diff_only": 0.0020000338554382324, + "tpp_threshold_2_unintended_diff_only": -0.002250000834465027, + "tpp_threshold_5_total_metric": -0.001999974250793457, + "tpp_threshold_5_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_5_unintended_diff_only": 0.0059999823570251465, + "tpp_threshold_10_total_metric": 0.01400001347064972, + "tpp_threshold_10_intended_diff_only": 0.018000006675720215, + "tpp_threshold_10_unintended_diff_only": 0.003999993205070496, + "tpp_threshold_20_total_metric": 0.019500061869621277, + "tpp_threshold_20_intended_diff_only": 0.01900005340576172, + "tpp_threshold_20_unintended_diff_only": -0.0005000084638595581, + "tpp_threshold_50_total_metric": 0.05075004696846008, + "tpp_threshold_50_intended_diff_only": 0.058000028133392334, + "tpp_threshold_50_unintended_diff_only": 0.007249981164932251, + "tpp_threshold_100_total_metric": 0.0897500216960907, + "tpp_threshold_100_intended_diff_only": 0.10100001096725464, + "tpp_threshold_100_unintended_diff_only": 0.01124998927116394, + "tpp_threshold_500_total_metric": 0.2235000729560852, + "tpp_threshold_500_intended_diff_only": 0.23200005292892456, + "tpp_threshold_500_unintended_diff_only": 0.008499979972839355 }, "3": { - "tpp_threshold_2_total_metric": 0.016749992966651917, - "tpp_threshold_2_intended_diff_only": 0.009999990463256836, - "tpp_threshold_2_unintended_diff_only": -0.006750002503395081, - "tpp_threshold_5_total_metric": 0.0195000022649765, - "tpp_threshold_5_intended_diff_only": 0.018000006675720215, - "tpp_threshold_5_unintended_diff_only": -0.0014999955892562866, - "tpp_threshold_10_total_metric": 0.021499991416931152, + "tpp_threshold_2_total_metric": 0.007249981164932251, + "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_2_unintended_diff_only": -0.0012499988079071045, + "tpp_threshold_5_total_metric": 0.013750016689300537, + "tpp_threshold_5_intended_diff_only": 0.009000003337860107, + "tpp_threshold_5_unintended_diff_only": -0.00475001335144043, + "tpp_threshold_10_total_metric": 0.026249989867210388, "tpp_threshold_10_intended_diff_only": 0.02399998903274536, - "tpp_threshold_10_unintended_diff_only": 0.002499997615814209 + "tpp_threshold_10_unintended_diff_only": -0.002250000834465027, + "tpp_threshold_20_total_metric": 0.017750009894371033, + "tpp_threshold_20_intended_diff_only": 0.018999993801116943, + "tpp_threshold_20_unintended_diff_only": 0.0012499839067459106, + "tpp_threshold_50_total_metric": 0.0495000034570694, + "tpp_threshold_50_intended_diff_only": 0.046999990940093994, + "tpp_threshold_50_unintended_diff_only": -0.002500012516975403, + "tpp_threshold_100_total_metric": 0.08449998497962952, + "tpp_threshold_100_intended_diff_only": 0.08899998664855957, + "tpp_threshold_100_unintended_diff_only": 0.004500001668930054, + "tpp_threshold_500_total_metric": 0.24825002253055573, + "tpp_threshold_500_intended_diff_only": 0.25300002098083496, + "tpp_threshold_500_unintended_diff_only": 0.004749998450279236 }, "5": { - "tpp_threshold_2_total_metric": -0.004000052809715271, - "tpp_threshold_2_intended_diff_only": 0.003999948501586914, - "tpp_threshold_2_unintended_diff_only": 0.008000001311302185, - "tpp_threshold_5_total_metric": 0.0024999529123306274, - "tpp_threshold_5_intended_diff_only": 0.003999948501586914, - "tpp_threshold_5_unintended_diff_only": 0.0014999955892562866, - "tpp_threshold_10_total_metric": 0.005499988794326782, - "tpp_threshold_10_intended_diff_only": 0.009999990463256836, - "tpp_threshold_10_unintended_diff_only": 0.004500001668930054 + "tpp_threshold_2_total_metric": 0.005249977111816406, + "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_2_unintended_diff_only": 0.0007500052452087402, + "tpp_threshold_5_total_metric": 0.012499958276748657, + "tpp_threshold_5_intended_diff_only": 0.011999964714050293, + "tpp_threshold_5_unintended_diff_only": -0.0004999935626983643, + "tpp_threshold_10_total_metric": -0.0005000084638595581, + "tpp_threshold_10_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_10_unintended_diff_only": 0.006499990820884705, + "tpp_threshold_20_total_metric": 0.005249932408332825, + "tpp_threshold_20_intended_diff_only": 0.008999943733215332, + "tpp_threshold_20_unintended_diff_only": 0.0037500113248825073, + "tpp_threshold_50_total_metric": 0.030249983072280884, + "tpp_threshold_50_intended_diff_only": 0.034999966621398926, + "tpp_threshold_50_unintended_diff_only": 0.004749983549118042, + "tpp_threshold_100_total_metric": 0.07499998807907104, + "tpp_threshold_100_intended_diff_only": 0.08399999141693115, + "tpp_threshold_100_unintended_diff_only": 0.009000003337860107, + "tpp_threshold_500_total_metric": 0.24050000309944153, + "tpp_threshold_500_intended_diff_only": 0.2549999952316284, + "tpp_threshold_500_unintended_diff_only": 0.01449999213218689 }, "6": { - "tpp_threshold_2_total_metric": 0.014500007033348083, - "tpp_threshold_2_intended_diff_only": 0.013000011444091797, - "tpp_threshold_2_unintended_diff_only": -0.0014999955892562866, - "tpp_threshold_5_total_metric": 0.01849997043609619, + "tpp_threshold_2_total_metric": 0.0014999955892562866, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 0.0034999996423721313, + "tpp_threshold_5_total_metric": 0.026249974966049194, "tpp_threshold_5_intended_diff_only": 0.02599996328353882, - "tpp_threshold_5_unintended_diff_only": 0.007499992847442627, - "tpp_threshold_10_total_metric": 0.03674997389316559, - "tpp_threshold_10_intended_diff_only": 0.038999974727630615, - "tpp_threshold_10_unintended_diff_only": 0.002250000834465027 + "tpp_threshold_5_unintended_diff_only": -0.000250011682510376, + "tpp_threshold_10_total_metric": 0.04725000262260437, + "tpp_threshold_10_intended_diff_only": 0.050999999046325684, + "tpp_threshold_10_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_20_total_metric": 0.07350002229213715, + "tpp_threshold_20_intended_diff_only": 0.0820000171661377, + "tpp_threshold_20_unintended_diff_only": 0.00849999487400055, + "tpp_threshold_50_total_metric": 0.12599998712539673, + "tpp_threshold_50_intended_diff_only": 0.13099998235702515, + "tpp_threshold_50_unintended_diff_only": 0.004999995231628418, + "tpp_threshold_100_total_metric": 0.17025001347064972, + "tpp_threshold_100_intended_diff_only": 0.18000000715255737, + "tpp_threshold_100_unintended_diff_only": 0.009749993681907654, + "tpp_threshold_500_total_metric": 0.33550000190734863, + "tpp_threshold_500_intended_diff_only": 0.35199999809265137, + "tpp_threshold_500_unintended_diff_only": 0.016499996185302734 } } } diff --git a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-122070-tokens-500000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-122070-tokens-500000000/scr/saebench_scr_custom_sae_eval_results.json index 329f13a6371530f42f582659e9ecd77f069b6e82..67854ae743b321382ef0423ab1dd543c23378717 100644 --- a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-122070-tokens-500000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-122070-tokens-500000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "566cd8bd-6784-48c6-b29c-a758303bf9ed", - "datetime_epoch_millis": 1776888071796, + "eval_id": "7f1d4b1d-0240-4dbe-844b-e44144d34ac6", + "datetime_epoch_millis": 1777318664947, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.23480957659350482, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.0822807968187218, "scr_metric_threshold_10": 0.1570888667485258, "scr_dir2_threshold_10": 0.1514459407148636, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.04465508084325942, + "scr_metric_threshold_20": 0.19616574407524248, + "scr_dir2_threshold_20": 0.19278523219134436, + "scr_dir1_threshold_50": 0.25410788345281526, + "scr_metric_threshold_50": 0.25097501257259874, + "scr_dir2_threshold_50": 0.2520089037530592, + "scr_dir1_threshold_100": -0.06088992275699545, + "scr_metric_threshold_100": 0.2688017896546623, + "scr_dir2_threshold_100": 0.2686097664369231, + "scr_dir1_threshold_500": -0.3623963777063147, + "scr_metric_threshold_500": 0.13050927956428743, + "scr_dir2_threshold_500": 0.1221145490594576 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.3333335880540079, "scr_metric_threshold_10": 0.007772074330999693, "scr_dir2_threshold_10": 0.007772074330999693, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2820514388024664, + "scr_metric_threshold_20": 0.14766833137585111, + "scr_dir2_threshold_20": 0.14766833137585111, + "scr_dir1_threshold_50": 0.4487182328294704, + "scr_metric_threshold_50": 0.1813471142552987, + "scr_dir2_threshold_50": 0.1813471142552987, + "scr_dir1_threshold_100": 0.29487178507484585, + "scr_metric_threshold_100": 0.28497410286525365, + "scr_dir2_threshold_100": 0.28497410286525365, + "scr_dir1_threshold_500": -3.230768525388901, + "scr_metric_threshold_500": 0.0492227771252837, + "scr_dir2_threshold_500": 0.0492227771252837 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.48484845748180366, "scr_metric_threshold_10": 0.11314984096315267, "scr_dir2_threshold_10": 0.11314984096315267, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.4924241158533419, + "scr_metric_threshold_20": 0.2293577463248362, + "scr_dir2_threshold_20": 0.2293577463248362, + "scr_dir1_threshold_50": 0.42424228740901826, + "scr_metric_threshold_50": 0.48623861906801674, + "scr_dir2_threshold_50": 0.48623861906801674, + "scr_dir1_threshold_100": -0.287878630520371, + "scr_metric_threshold_100": 0.4892966834665476, + "scr_dir2_threshold_100": 0.4892966834665476, + "scr_dir1_threshold_500": -0.16666629037480024, + "scr_metric_threshold_500": 0.2048930488594005, + "scr_dir2_threshold_500": 0.2048930488594005 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": -0.7656252619344311, "scr_metric_threshold_10": 0.16331655168313458, "scr_dir2_threshold_10": 0.16331655168313458, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -1.343750174622954, + "scr_metric_threshold_20": 0.1783920316689313, + "scr_dir2_threshold_20": 0.1783920316689313, + "scr_dir1_threshold_50": 0.20312514551912844, + "scr_metric_threshold_50": 0.21859296219014387, + "scr_dir2_threshold_50": 0.21859296219014387, + "scr_dir1_threshold_100": -1.656249825377046, + "scr_metric_threshold_100": 0.07286442057030854, + "scr_dir2_threshold_100": 0.07286442057030854, + "scr_dir1_threshold_500": -0.6406250291038257, + "scr_metric_threshold_500": -0.13819095138732784, + "scr_dir2_threshold_500": -0.13819095138732784 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": -0.1208054228229802, "scr_metric_threshold_10": 0.11783439248632226, "scr_dir2_threshold_10": 0.11783439248632226, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.42952990333245994, + "scr_metric_threshold_20": -0.08917196243161127, + "scr_dir2_threshold_20": -0.08917196243161127, + "scr_dir1_threshold_50": 0.10738237582531437, + "scr_metric_threshold_50": -0.2324841127051662, + "scr_dir2_threshold_50": -0.2324841127051662, + "scr_dir1_threshold_100": 0.187919457717702, + "scr_metric_threshold_100": -0.2675158872948338, + "scr_dir2_threshold_100": -0.2675158872948338, + "scr_dir1_threshold_500": 0.18120773420326786, + "scr_metric_threshold_500": -0.5159235511610947, + "scr_dir2_threshold_500": -0.5159235511610947 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.0631580235225028, "scr_metric_threshold_10": 0.2182539673153717, "scr_dir2_threshold_10": 0.2182539673153717, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.03157916861558172, + "scr_metric_threshold_20": 0.35317450274476986, + "scr_dir2_threshold_20": 0.35317450274476986, + "scr_dir1_threshold_50": -0.03684194510663114, + "scr_metric_threshold_50": 0.5396826147702655, + "scr_dir2_threshold_50": 0.5396826147702655, + "scr_dir1_threshold_100": -0.026315764707211, + "scr_metric_threshold_100": 0.6111109665672388, + "scr_dir2_threshold_100": 0.6111109665672388, + "scr_dir1_threshold_500": -0.06842111372221286, + "scr_metric_threshold_500": 0.5833333136228053, + "scr_dir2_threshold_500": 0.5833333136228053 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.3005465388586523, "scr_metric_threshold_10": 0.12595415766246074, "scr_dir2_threshold_10": 0.12595415766246074, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3497268934249226, + "scr_metric_threshold_20": 0.17938926174649328, + "scr_dir2_threshold_20": 0.17938926174649328, + "scr_dir1_threshold_50": 0.3606557163469838, + "scr_metric_threshold_50": 0.19847321124088896, + "scr_dir2_threshold_50": 0.19847321124088896, + "scr_dir1_threshold_100": 0.4098360709132541, + "scr_metric_threshold_100": 0.29007644181233594, + "scr_dir2_threshold_100": 0.29007644181233594, + "scr_dir1_threshold_500": 0.5191256029678558, + "scr_metric_threshold_500": 0.24045808212745792, + "scr_dir2_threshold_500": 0.24045808212745792 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.19999988910765484, "scr_metric_threshold_10": 0.3476393872642003, "scr_dir2_threshold_10": 0.3476393872642003, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.22325567727222573, + "scr_metric_threshold_20": 0.40772547889010513, + "scr_dir2_threshold_20": 0.40772547889010513, + "scr_dir1_threshold_50": 0.330232413721597, + "scr_metric_threshold_50": 0.4206008506837011, + "scr_dir2_threshold_50": 0.4206008506837011, + "scr_dir1_threshold_100": 0.3674417856772556, + "scr_metric_threshold_100": 0.4463518500848407, + "scr_dir2_threshold_100": 0.4463518500848407, + "scr_dir1_threshold_500": 0.19999988910765484, + "scr_metric_threshold_500": 0.31330480666733007, + "scr_dir2_threshold_500": 0.31330480666733007 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.1627905622825643, "scr_metric_threshold_10": 0.1627905622825643, "scr_dir2_threshold_10": 0.11764715401326675, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.1627905622825643, + "scr_metric_threshold_20": 0.1627905622825643, + "scr_dir2_threshold_20": 0.13574646721137937, + "scr_dir1_threshold_50": 0.19534884107764103, + "scr_metric_threshold_50": 0.19534884107764103, + "scr_dir2_threshold_50": 0.20361997052132483, + "scr_dir1_threshold_100": 0.22325573916560695, + "scr_metric_threshold_100": 0.22325573916560695, + "scr_dir2_threshold_100": 0.22171955342369326, + "scr_dir1_threshold_500": 0.3069767106604445, + "scr_metric_threshold_500": 0.3069767106604445, + "scr_dir2_threshold_500": 0.23981886662180585 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-122070-tokens-500000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-122070-tokens-500000000/tpp/saebench_tpp_custom_sae_eval_results.json index e94cc69236a6578f65e719906c176658c2c11e06..757b0f4995c8687d73fcbb95cd4a32fb60c2df59 100644 --- a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-122070-tokens-500000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-122070-tokens-500000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "32a1c55c-68f2-459c-9557-30102cee6666", - "datetime_epoch_millis": 1776888231817, + "eval_id": "95058832-1f37-44b0-b9a4-cd28e9bf03f4", + "datetime_epoch_millis": 1777318956480, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.03662500232458114, - "tpp_threshold_2_intended_diff_only": 0.046100002527236936, - "tpp_threshold_2_unintended_diff_only": 0.009475000202655792, - "tpp_threshold_5_total_metric": 0.0736000031232834, - "tpp_threshold_5_intended_diff_only": 0.10090001225471497, - "tpp_threshold_5_unintended_diff_only": 0.02730000913143158, - "tpp_threshold_10_total_metric": 0.10670000314712524, - "tpp_threshold_10_intended_diff_only": 0.1630000114440918, - "tpp_threshold_10_unintended_diff_only": 0.056300008296966554, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.034475003182888035, + "tpp_threshold_2_intended_diff_only": 0.04510000944137573, + "tpp_threshold_2_unintended_diff_only": 0.010625006258487701, + "tpp_threshold_5_total_metric": 0.07185000777244568, + "tpp_threshold_5_intended_diff_only": 0.09880001544952394, + "tpp_threshold_5_unintended_diff_only": 0.026950007677078246, + "tpp_threshold_10_total_metric": 0.10792500227689743, + "tpp_threshold_10_intended_diff_only": 0.165500009059906, + "tpp_threshold_10_unintended_diff_only": 0.05757500678300857, + "tpp_threshold_20_total_metric": 0.12540001422166824, + "tpp_threshold_20_intended_diff_only": 0.20110002160072327, + "tpp_threshold_20_unintended_diff_only": 0.07570000737905502, + "tpp_threshold_50_total_metric": 0.13055001348257064, + "tpp_threshold_50_intended_diff_only": 0.2476000189781189, + "tpp_threshold_50_unintended_diff_only": 0.11705000549554825, + "tpp_threshold_100_total_metric": 0.13352500945329665, + "tpp_threshold_100_intended_diff_only": 0.2854000210762024, + "tpp_threshold_100_unintended_diff_only": 0.15187501162290573, + "tpp_threshold_500_total_metric": 0.2041750058531761, + "tpp_threshold_500_intended_diff_only": 0.403900021314621, + "tpp_threshold_500_unintended_diff_only": 0.19972501546144486 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.0668500006198883, - "tpp_threshold_2_intended_diff_only": 0.08540000915527343, - "tpp_threshold_2_unintended_diff_only": 0.018550008535385132, - "tpp_threshold_5_total_metric": 0.1350000023841858, - "tpp_threshold_5_intended_diff_only": 0.18740001916885377, - "tpp_threshold_5_unintended_diff_only": 0.05240001678466797, - "tpp_threshold_10_total_metric": 0.19629999995231628, - "tpp_threshold_10_intended_diff_only": 0.30680001974105836, - "tpp_threshold_10_unintended_diff_only": 0.11050001978874206, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.06495001018047333, + "tpp_threshold_2_intended_diff_only": 0.08520002365112304, + "tpp_threshold_2_unintended_diff_only": 0.020250013470649718, + "tpp_threshold_5_total_metric": 0.13385001420974732, + "tpp_threshold_5_intended_diff_only": 0.18760002851486207, + "tpp_threshold_5_unintended_diff_only": 0.053750014305114745, + "tpp_threshold_10_total_metric": 0.19385000169277192, + "tpp_threshold_10_intended_diff_only": 0.306600022315979, + "tpp_threshold_10_unintended_diff_only": 0.11275002062320709, + "tpp_threshold_20_total_metric": 0.21070002615451813, + "tpp_threshold_20_intended_diff_only": 0.3588000416755676, + "tpp_threshold_20_unintended_diff_only": 0.1481000155210495, + "tpp_threshold_50_total_metric": 0.17495002150535582, + "tpp_threshold_50_intended_diff_only": 0.40600004196166994, + "tpp_threshold_50_unintended_diff_only": 0.2310500204563141, + "tpp_threshold_100_total_metric": 0.13630000948905946, + "tpp_threshold_100_intended_diff_only": 0.4312000393867493, + "tpp_threshold_100_unintended_diff_only": 0.2949000298976898, + "tpp_threshold_500_total_metric": 0.08520000278949738, + "tpp_threshold_500_intended_diff_only": 0.46380003690719607, + "tpp_threshold_500_unintended_diff_only": 0.37860003411769866 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.006400004029273987, - "tpp_threshold_2_intended_diff_only": 0.0067999958992004395, - "tpp_threshold_2_unintended_diff_only": 0.00039999186992645264, - "tpp_threshold_5_total_metric": 0.012200003862380982, - "tpp_threshold_5_intended_diff_only": 0.014400005340576172, - "tpp_threshold_5_unintended_diff_only": 0.0022000014781951905, - "tpp_threshold_10_total_metric": 0.017100006341934204, - "tpp_threshold_10_intended_diff_only": 0.019200003147125243, - "tpp_threshold_10_unintended_diff_only": 0.00209999680519104, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.0039999961853027345, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 0.0009999990463256836, + "tpp_threshold_5_total_metric": 0.009850001335144043, + "tpp_threshold_5_intended_diff_only": 0.010000002384185792, + "tpp_threshold_5_unintended_diff_only": 0.00015000104904174804, + "tpp_threshold_10_total_metric": 0.02200000286102295, + "tpp_threshold_10_intended_diff_only": 0.024399995803833008, + "tpp_threshold_10_unintended_diff_only": 0.0023999929428100584, + "tpp_threshold_20_total_metric": 0.04010000228881836, + "tpp_threshold_20_intended_diff_only": 0.043400001525878903, + "tpp_threshold_20_unintended_diff_only": 0.003299999237060547, + "tpp_threshold_50_total_metric": 0.08615000545978546, + "tpp_threshold_50_intended_diff_only": 0.08919999599456788, + "tpp_threshold_50_unintended_diff_only": 0.0030499905347824096, + "tpp_threshold_100_total_metric": 0.13075000941753387, + "tpp_threshold_100_intended_diff_only": 0.1396000027656555, + "tpp_threshold_100_unintended_diff_only": 0.008849993348121643, + "tpp_threshold_500_total_metric": 0.32315000891685486, + "tpp_threshold_500_intended_diff_only": 0.3440000057220459, + "tpp_threshold_500_unintended_diff_only": 0.02084999680519104 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.009000018239021301, "tpp_threshold_10_total_metric": 0.1964999884366989, "tpp_threshold_10_intended_diff_only": 0.22600001096725464, - "tpp_threshold_10_unintended_diff_only": 0.029500022530555725 + "tpp_threshold_10_unintended_diff_only": 0.029500022530555725, + "tpp_threshold_20_total_metric": 0.2407499998807907, + "tpp_threshold_20_intended_diff_only": 0.35500001907348633, + "tpp_threshold_20_unintended_diff_only": 0.11425001919269562, + "tpp_threshold_50_total_metric": 0.19324995577335358, + "tpp_threshold_50_intended_diff_only": 0.4129999876022339, + "tpp_threshold_50_unintended_diff_only": 0.2197500318288803, + "tpp_threshold_100_total_metric": 0.16849997639656067, + "tpp_threshold_100_intended_diff_only": 0.4399999976158142, + "tpp_threshold_100_unintended_diff_only": 0.27150002121925354, + "tpp_threshold_500_total_metric": 0.08399999141693115, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.3720000386238098 }, "1": { - "tpp_threshold_2_total_metric": 0.16649998724460602, - "tpp_threshold_2_intended_diff_only": 0.24500000476837158, - "tpp_threshold_2_unintended_diff_only": 0.07850001752376556, - "tpp_threshold_5_total_metric": 0.16624999046325684, + "tpp_threshold_2_total_metric": 0.1652500182390213, + "tpp_threshold_2_intended_diff_only": 0.2420000433921814, + "tpp_threshold_2_unintended_diff_only": 0.0767500251531601, + "tpp_threshold_5_total_metric": 0.16249999403953552, "tpp_threshold_5_intended_diff_only": 0.2540000081062317, - "tpp_threshold_5_unintended_diff_only": 0.08775001764297485, - "tpp_threshold_10_total_metric": 0.17374996840953827, + "tpp_threshold_5_unintended_diff_only": 0.09150001406669617, + "tpp_threshold_10_total_metric": 0.16599996387958527, "tpp_threshold_10_intended_diff_only": 0.3199999928474426, - "tpp_threshold_10_unintended_diff_only": 0.14625002443790436 + "tpp_threshold_10_unintended_diff_only": 0.15400002896785736, + "tpp_threshold_20_total_metric": 0.17550000548362732, + "tpp_threshold_20_intended_diff_only": 0.3370000123977661, + "tpp_threshold_20_unintended_diff_only": 0.1615000069141388, + "tpp_threshold_50_total_metric": 0.1327500343322754, + "tpp_threshold_50_intended_diff_only": 0.3710000514984131, + "tpp_threshold_50_unintended_diff_only": 0.2382500171661377, + "tpp_threshold_100_total_metric": 0.10324999690055847, + "tpp_threshold_100_intended_diff_only": 0.409000039100647, + "tpp_threshold_100_unintended_diff_only": 0.3057500422000885, + "tpp_threshold_500_total_metric": 0.06324999034404755, + "tpp_threshold_500_intended_diff_only": 0.4520000219345093, + "tpp_threshold_500_unintended_diff_only": 0.38875003159046173 }, "2": { - "tpp_threshold_2_total_metric": 0.1417500078678131, - "tpp_threshold_2_intended_diff_only": 0.156000018119812, - "tpp_threshold_2_unintended_diff_only": 0.014250010251998901, - "tpp_threshold_5_total_metric": 0.22175002098083496, - "tpp_threshold_5_intended_diff_only": 0.27900004386901855, - "tpp_threshold_5_unintended_diff_only": 0.057250022888183594, - "tpp_threshold_10_total_metric": 0.18575000762939453, - "tpp_threshold_10_intended_diff_only": 0.3360000252723694, - "tpp_threshold_10_unintended_diff_only": 0.15025001764297485 + "tpp_threshold_2_total_metric": 0.13250002264976501, + "tpp_threshold_2_intended_diff_only": 0.15500003099441528, + "tpp_threshold_2_unintended_diff_only": 0.02250000834465027, + "tpp_threshold_5_total_metric": 0.22425004839897156, + "tpp_threshold_5_intended_diff_only": 0.2780000567436218, + "tpp_threshold_5_unintended_diff_only": 0.05375000834465027, + "tpp_threshold_10_total_metric": 0.18274998664855957, + "tpp_threshold_10_intended_diff_only": 0.3370000123977661, + "tpp_threshold_10_unintended_diff_only": 0.15425002574920654, + "tpp_threshold_20_total_metric": 0.1337500512599945, + "tpp_threshold_20_intended_diff_only": 0.36500006914138794, + "tpp_threshold_20_unintended_diff_only": 0.23125001788139343, + "tpp_threshold_50_total_metric": 0.09500004351139069, + "tpp_threshold_50_intended_diff_only": 0.3850000500679016, + "tpp_threshold_50_unintended_diff_only": 0.2900000065565109, + "tpp_threshold_100_total_metric": 0.06025002896785736, + "tpp_threshold_100_intended_diff_only": 0.3980000615119934, + "tpp_threshold_100_unintended_diff_only": 0.33775003254413605, + "tpp_threshold_500_total_metric": 0.05175001919269562, + "tpp_threshold_500_intended_diff_only": 0.4490000605583191, + "tpp_threshold_500_unintended_diff_only": 0.3972500413656235 }, "6": { - "tpp_threshold_2_total_metric": 0.008750051259994507, + "tpp_threshold_2_total_metric": 0.00700002908706665, "tpp_threshold_2_intended_diff_only": 0.006000041961669922, - "tpp_threshold_2_unintended_diff_only": -0.002750009298324585, - "tpp_threshold_5_total_metric": 0.0847499817609787, - "tpp_threshold_5_intended_diff_only": 0.11599999666213989, - "tpp_threshold_5_unintended_diff_only": 0.031250014901161194, - "tpp_threshold_10_total_metric": 0.24425002932548523, + "tpp_threshold_2_unintended_diff_only": -0.0009999871253967285, + "tpp_threshold_5_total_metric": 0.08775003254413605, + "tpp_threshold_5_intended_diff_only": 0.1170000433921814, + "tpp_threshold_5_unintended_diff_only": 0.02925001084804535, + "tpp_threshold_10_total_metric": 0.24700003862380981, "tpp_threshold_10_intended_diff_only": 0.3810000419616699, - "tpp_threshold_10_unintended_diff_only": 0.1367500126361847 + "tpp_threshold_10_unintended_diff_only": 0.1340000033378601, + "tpp_threshold_20_total_metric": 0.2835000455379486, + "tpp_threshold_20_intended_diff_only": 0.4310000538825989, + "tpp_threshold_20_unintended_diff_only": 0.14750000834465027, + "tpp_threshold_50_total_metric": 0.2382500320672989, + "tpp_threshold_50_intended_diff_only": 0.47700005769729614, + "tpp_threshold_50_unintended_diff_only": 0.23875002562999725, + "tpp_threshold_100_total_metric": 0.18175001442432404, + "tpp_threshold_100_intended_diff_only": 0.48200005292892456, + "tpp_threshold_100_unintended_diff_only": 0.3002500385046005, + "tpp_threshold_500_total_metric": 0.11525002121925354, + "tpp_threshold_500_intended_diff_only": 0.487000048160553, + "tpp_threshold_500_unintended_diff_only": 0.37175002694129944 }, "9": { - "tpp_threshold_2_total_metric": 0.0070000141859054565, - "tpp_threshold_2_intended_diff_only": 0.008000016212463379, - "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_5_total_metric": 0.15925005078315735, - "tpp_threshold_5_intended_diff_only": 0.23600006103515625, - "tpp_threshold_5_unintended_diff_only": 0.0767500102519989, - "tpp_threshold_10_total_metric": 0.18125000596046448, - "tpp_threshold_10_intended_diff_only": 0.2710000276565552, - "tpp_threshold_10_unintended_diff_only": 0.0897500216960907 + "tpp_threshold_2_total_metric": 0.009750038385391235, + "tpp_threshold_2_intended_diff_only": 0.01100003719329834, + "tpp_threshold_2_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_5_total_metric": 0.15175002813339233, + "tpp_threshold_5_intended_diff_only": 0.23700004816055298, + "tpp_threshold_5_unintended_diff_only": 0.08525002002716064, + "tpp_threshold_10_total_metric": 0.177000030875206, + "tpp_threshold_10_intended_diff_only": 0.2690000534057617, + "tpp_threshold_10_unintended_diff_only": 0.09200002253055573, + "tpp_threshold_20_total_metric": 0.2200000286102295, + "tpp_threshold_20_intended_diff_only": 0.3060000538825989, + "tpp_threshold_20_unintended_diff_only": 0.08600002527236938, + "tpp_threshold_50_total_metric": 0.21550004184246063, + "tpp_threshold_50_intended_diff_only": 0.3840000629425049, + "tpp_threshold_50_unintended_diff_only": 0.16850002110004425, + "tpp_threshold_100_total_metric": 0.1677500307559967, + "tpp_threshold_100_intended_diff_only": 0.4270000457763672, + "tpp_threshold_100_unintended_diff_only": 0.2592500150203705, + "tpp_threshold_500_total_metric": 0.11174999177455902, + "tpp_threshold_500_intended_diff_only": 0.4750000238418579, + "tpp_threshold_500_unintended_diff_only": 0.3632500320672989 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.007500007748603821, - "tpp_threshold_2_intended_diff_only": 0.003000020980834961, - "tpp_threshold_2_unintended_diff_only": -0.00449998676776886, - "tpp_threshold_5_total_metric": 0.010000020265579224, - "tpp_threshold_5_intended_diff_only": 0.008000016212463379, - "tpp_threshold_5_unintended_diff_only": -0.0020000040531158447, - "tpp_threshold_10_total_metric": 0.000250011682510376, - "tpp_threshold_10_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_10_unintended_diff_only": 0.0037499964237213135 + "tpp_threshold_2_total_metric": 0.0005000084638595581, + "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_2_unintended_diff_only": 0.0034999996423721313, + "tpp_threshold_5_total_metric": 0.002750009298324585, + "tpp_threshold_5_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_10_total_metric": 0.0037499815225601196, + "tpp_threshold_10_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_10_unintended_diff_only": 0.002250000834465027, + "tpp_threshold_20_total_metric": 0.010500013828277588, + "tpp_threshold_20_intended_diff_only": 0.013000011444091797, + "tpp_threshold_20_unintended_diff_only": 0.002499997615814209, + "tpp_threshold_50_total_metric": 0.026000022888183594, + "tpp_threshold_50_intended_diff_only": 0.022000014781951904, + "tpp_threshold_50_unintended_diff_only": -0.0040000081062316895, + "tpp_threshold_100_total_metric": 0.04749999940395355, + "tpp_threshold_100_intended_diff_only": 0.05199998617172241, + "tpp_threshold_100_unintended_diff_only": 0.00449998676776886, + "tpp_threshold_500_total_metric": 0.2799999862909317, + "tpp_threshold_500_intended_diff_only": 0.2929999828338623, + "tpp_threshold_500_unintended_diff_only": 0.012999996542930603 }, "2": { - "tpp_threshold_2_total_metric": -0.0009999722242355347, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": 0.004999980330467224, - "tpp_threshold_5_total_metric": 0.003000006079673767, - "tpp_threshold_5_intended_diff_only": 0.009000003337860107, - "tpp_threshold_5_unintended_diff_only": 0.00599999725818634, - "tpp_threshold_10_total_metric": 0.014250040054321289, - "tpp_threshold_10_intended_diff_only": 0.017000019550323486, - "tpp_threshold_10_unintended_diff_only": 0.0027499794960021973 + "tpp_threshold_2_total_metric": 0.00850003957748413, + "tpp_threshold_2_intended_diff_only": 0.006000041961669922, + "tpp_threshold_2_unintended_diff_only": -0.002499997615814209, + "tpp_threshold_5_total_metric": -0.0014999955892562866, + "tpp_threshold_5_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_5_unintended_diff_only": 0.005500003695487976, + "tpp_threshold_10_total_metric": 0.019750013947486877, + "tpp_threshold_10_intended_diff_only": 0.023000001907348633, + "tpp_threshold_10_unintended_diff_only": 0.0032499879598617554, + "tpp_threshold_20_total_metric": 0.04075005650520325, + "tpp_threshold_20_intended_diff_only": 0.04300004243850708, + "tpp_threshold_20_unintended_diff_only": 0.002249985933303833, + "tpp_threshold_50_total_metric": 0.10075005888938904, + "tpp_threshold_50_intended_diff_only": 0.10900002717971802, + "tpp_threshold_50_unintended_diff_only": 0.00824996829032898, + "tpp_threshold_100_total_metric": 0.14075005054473877, + "tpp_threshold_100_intended_diff_only": 0.15500003099441528, + "tpp_threshold_100_unintended_diff_only": 0.014249980449676514, + "tpp_threshold_500_total_metric": 0.3342500478029251, + "tpp_threshold_500_intended_diff_only": 0.3580000400543213, + "tpp_threshold_500_unintended_diff_only": 0.02374999225139618 }, "3": { - "tpp_threshold_2_total_metric": 0.01150001585483551, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": -0.006500020623207092, - "tpp_threshold_5_total_metric": 0.015750005841255188, - "tpp_threshold_5_intended_diff_only": 0.013000011444091797, - "tpp_threshold_5_unintended_diff_only": -0.002749994397163391, - "tpp_threshold_10_total_metric": 0.013999998569488525, + "tpp_threshold_2_total_metric": 0.0012499988079071045, + "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, + "tpp_threshold_2_unintended_diff_only": -0.000250011682510376, + "tpp_threshold_5_total_metric": 0.007499992847442627, + "tpp_threshold_5_intended_diff_only": 0.0009999871253967285, + "tpp_threshold_5_unintended_diff_only": -0.0065000057220458984, + "tpp_threshold_10_total_metric": 0.017000004649162292, "tpp_threshold_10_intended_diff_only": 0.013999998569488525, - "tpp_threshold_10_unintended_diff_only": 0.0 + "tpp_threshold_10_unintended_diff_only": -0.003000006079673767, + "tpp_threshold_20_total_metric": 0.008249998092651367, + "tpp_threshold_20_intended_diff_only": 0.009000003337860107, + "tpp_threshold_20_unintended_diff_only": 0.0007500052452087402, + "tpp_threshold_50_total_metric": 0.031999990344047546, + "tpp_threshold_50_intended_diff_only": 0.02899998426437378, + "tpp_threshold_50_unintended_diff_only": -0.003000006079673767, + "tpp_threshold_100_total_metric": 0.08600002527236938, + "tpp_threshold_100_intended_diff_only": 0.0910000205039978, + "tpp_threshold_100_unintended_diff_only": 0.004999995231628418, + "tpp_threshold_500_total_metric": 0.27925001084804535, + "tpp_threshold_500_intended_diff_only": 0.29600000381469727, + "tpp_threshold_500_unintended_diff_only": 0.016749992966651917 }, "5": { - "tpp_threshold_2_total_metric": 0.0002499520778656006, + "tpp_threshold_2_total_metric": 0.008499935269355774, "tpp_threshold_2_intended_diff_only": 0.008999943733215332, - "tpp_threshold_2_unintended_diff_only": 0.008749991655349731, - "tpp_threshold_5_total_metric": 0.016749978065490723, - "tpp_threshold_5_intended_diff_only": 0.018999993801116943, - "tpp_threshold_5_unintended_diff_only": 0.0022500157356262207, - "tpp_threshold_10_total_metric": 0.035749971866607666, - "tpp_threshold_10_intended_diff_only": 0.038999974727630615, - "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492 + "tpp_threshold_2_unintended_diff_only": 0.0005000084638595581, + "tpp_threshold_5_total_metric": 0.02799999713897705, + "tpp_threshold_5_intended_diff_only": 0.02799999713897705, + "tpp_threshold_5_unintended_diff_only": 0.0, + "tpp_threshold_10_total_metric": 0.037749990820884705, + "tpp_threshold_10_intended_diff_only": 0.042999982833862305, + "tpp_threshold_10_unintended_diff_only": 0.0052499920129776, + "tpp_threshold_20_total_metric": 0.08799993991851807, + "tpp_threshold_20_intended_diff_only": 0.09199994802474976, + "tpp_threshold_20_unintended_diff_only": 0.0040000081062316895, + "tpp_threshold_50_total_metric": 0.14149998128414154, + "tpp_threshold_50_intended_diff_only": 0.1499999761581421, + "tpp_threshold_50_unintended_diff_only": 0.00849999487400055, + "tpp_threshold_100_total_metric": 0.1925000101327896, + "tpp_threshold_100_intended_diff_only": 0.203000009059906, + "tpp_threshold_100_unintended_diff_only": 0.010499998927116394, + "tpp_threshold_500_total_metric": 0.37975001335144043, + "tpp_threshold_500_intended_diff_only": 0.40700000524520874, + "tpp_threshold_500_unintended_diff_only": 0.02724999189376831 }, "6": { - "tpp_threshold_2_total_metric": 0.013750016689300537, - "tpp_threshold_2_intended_diff_only": 0.013000011444091797, - "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402, - "tpp_threshold_5_total_metric": 0.015500009059906006, - "tpp_threshold_5_intended_diff_only": 0.023000001907348633, - "tpp_threshold_5_unintended_diff_only": 0.007499992847442627, - "tpp_threshold_10_total_metric": 0.021250009536743164, - "tpp_threshold_10_intended_diff_only": 0.022000014781951904, - "tpp_threshold_10_unintended_diff_only": 0.0007500052452087402 + "tpp_threshold_2_total_metric": 0.0012499988079071045, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_5_total_metric": 0.012500002980232239, + "tpp_threshold_5_intended_diff_only": 0.013000011444091797, + "tpp_threshold_5_unintended_diff_only": 0.0005000084638595581, + "tpp_threshold_10_total_metric": 0.03175002336502075, + "tpp_threshold_10_intended_diff_only": 0.03600001335144043, + "tpp_threshold_10_unintended_diff_only": 0.004249989986419678, + "tpp_threshold_20_total_metric": 0.05300000309944153, + "tpp_threshold_20_intended_diff_only": 0.06000000238418579, + "tpp_threshold_20_unintended_diff_only": 0.006999999284744263, + "tpp_threshold_50_total_metric": 0.1304999738931656, + "tpp_threshold_50_intended_diff_only": 0.13599997758865356, + "tpp_threshold_50_unintended_diff_only": 0.005500003695487976, + "tpp_threshold_100_total_metric": 0.18699996173381805, + "tpp_threshold_100_intended_diff_only": 0.19699996709823608, + "tpp_threshold_100_unintended_diff_only": 0.01000000536441803, + "tpp_threshold_500_total_metric": 0.3424999862909317, + "tpp_threshold_500_intended_diff_only": 0.3659999966621399, + "tpp_threshold_500_unintended_diff_only": 0.02350001037120819 } } } diff --git a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-134277-tokens-550000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-134277-tokens-550000000/scr/saebench_scr_custom_sae_eval_results.json index 301073b3e0a283e27f626f2909fe0a208895af05..7390241409e776cf4395c239136c07839c6e7e7a 100644 --- a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-134277-tokens-550000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-134277-tokens-550000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "c8067843-b33f-4214-b553-551933f461a0", - "datetime_epoch_millis": 1776888090814, + "eval_id": "e9d9b2f8-4383-4e98-8885-f8622072f01e", + "datetime_epoch_millis": 1777318777425, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.21787181216399415, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.15927973238923598, "scr_metric_threshold_10": 0.16492815410899112, "scr_dir2_threshold_10": 0.15926944367824855, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.06403741995516163, + "scr_metric_threshold_20": 0.17821840542037454, + "scr_dir2_threshold_20": 0.1695895821340007, + "scr_dir1_threshold_50": -0.06384944761201751, + "scr_metric_threshold_50": 0.26323810882868226, + "scr_dir2_threshold_50": 0.25845805135619404, + "scr_dir1_threshold_100": -0.04951360075581539, + "scr_metric_threshold_100": 0.28090672126254324, + "scr_dir2_threshold_100": 0.2916822844899428, + "scr_dir1_threshold_500": -0.8534963557937664, + "scr_metric_threshold_500": 0.20603594348642645, + "scr_dir2_threshold_500": 0.20145844371435345 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.14102610148224506, "scr_metric_threshold_10": 0.07512948567373162, "scr_dir2_threshold_10": 0.07512948567373162, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.43589788655709094, + "scr_metric_threshold_20": 0.12435226279901532, + "scr_dir2_threshold_20": 0.12435226279901532, + "scr_dir1_threshold_50": 0.0, + "scr_metric_threshold_50": 0.2616580342884179, + "scr_dir2_threshold_50": 0.2616580342884179, + "scr_dir1_threshold_100": 0.269231092530087, + "scr_metric_threshold_100": 0.3212435257163134, + "scr_dir2_threshold_100": 0.3212435257163134, + "scr_dir1_threshold_500": -2.8717942447901343, + "scr_metric_threshold_500": 0.34196895432153707, + "scr_dir2_threshold_500": 0.34196895432153707 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.3560604589646946, "scr_metric_threshold_10": 0.10397564776756013, "scr_dir2_threshold_10": 0.10397564776756013, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.06818182844432363, + "scr_metric_threshold_20": 0.015290504269843151, + "scr_dir2_threshold_20": 0.015290504269843151, + "scr_dir1_threshold_50": -1.2272724604475855, + "scr_metric_threshold_50": 0.33027532969386547, + "scr_dir2_threshold_50": 0.33027532969386547, + "scr_dir1_threshold_100": -0.5606059442976655, + "scr_metric_threshold_100": 0.446483235055549, + "scr_dir2_threshold_100": 0.446483235055549, + "scr_dir1_threshold_500": -1.4545449208951713, + "scr_metric_threshold_500": 0.28440363460714707, + "scr_dir2_threshold_500": 0.28440363460714707 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.45312467985791743, "scr_metric_threshold_10": 0.17085429167603294, "scr_dir2_threshold_10": 0.17085429167603294, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -1.8906245634426146, + "scr_metric_threshold_20": 0.2110552221972455, + "scr_dir2_threshold_20": 0.2110552221972455, + "scr_dir1_threshold_50": -0.1250002328306055, + "scr_metric_threshold_50": 0.23115576233804724, + "scr_dir2_threshold_50": 0.23115576233804724, + "scr_dir1_threshold_100": -0.28125005820765137, + "scr_metric_threshold_100": 0.2964824429154574, + "scr_dir2_threshold_100": 0.2964824429154574, + "scr_dir1_threshold_500": -2.2968748544808717, + "scr_metric_threshold_500": -0.13316574147193197, + "scr_dir2_threshold_500": -0.13316574147193197 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": -0.4362420268780965, "scr_metric_threshold_10": 0.09872616905774931, "scr_dir2_threshold_10": 0.09872616905774931, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.17449641072003616, + "scr_metric_threshold_20": -0.035031774589667594, + "scr_dir2_threshold_20": -0.035031774589667594, + "scr_dir1_threshold_50": -0.1946311812321361, + "scr_metric_threshold_50": -0.21019102718541183, + "scr_dir2_threshold_50": -0.21019102718541183, + "scr_dir1_threshold_100": -0.6711411490096227, + "scr_metric_threshold_100": -0.28025476618845013, + "scr_dir2_threshold_100": -0.28025476618845013, + "scr_dir1_threshold_500": -1.3758393154666064, + "scr_metric_threshold_500": -0.509554016802435, + "scr_dir2_threshold_500": -0.509554016802435 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.052631529414422, "scr_metric_threshold_10": 0.2182539673153717, "scr_dir2_threshold_10": 0.2182539673153717, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.057894933322792726, + "scr_metric_threshold_20": 0.3055556015467876, + "scr_dir2_threshold_20": 0.3055556015467876, + "scr_dir1_threshold_50": -0.02105267450750093, + "scr_metric_threshold_50": 0.5238095688621593, + "scr_dir2_threshold_50": 0.5238095688621593, + "scr_dir1_threshold_100": 0.00526309019971007, + "scr_metric_threshold_100": 0.46428558757517696, + "scr_dir2_threshold_100": 0.46428558757517696, + "scr_dir1_threshold_500": 0.01578958430779086, + "scr_metric_threshold_500": 0.567460267714699, + "scr_dir2_threshold_500": 0.567460267714699 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.3169397732417441, "scr_metric_threshold_10": 0.1374046183585474, "scr_dir2_threshold_10": 0.1374046183585474, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3224041847027747, + "scr_metric_threshold_20": 0.1832060061456478, + "scr_dir2_threshold_20": 0.1832060061456478, + "scr_dir1_threshold_50": 0.4153004823742847, + "scr_metric_threshold_50": 0.22519087703221677, + "scr_dir2_threshold_50": 0.22519087703221677, + "scr_dir1_threshold_100": 0.4098360709132541, + "scr_metric_threshold_100": 0.31297713570588614, + "scr_dir2_threshold_100": 0.31297713570588614, + "scr_dir1_threshold_500": 0.4808743970321442, + "scr_metric_threshold_500": 0.26335877602100816, + "scr_dir2_threshold_500": 0.26335877602100816 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.22325567727222573, "scr_metric_threshold_10": 0.3476393872642003, "scr_dir2_threshold_10": 0.3476393872642003, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.24651146543679664, + "scr_metric_threshold_20": 0.41201726948797046, + "scr_dir2_threshold_20": 0.41201726948797046, + "scr_dir1_threshold_50": 0.40000005544617256, + "scr_metric_threshold_50": 0.5021458952989327, + "scr_dir2_threshold_50": 0.5021458952989327, + "scr_dir1_threshold_100": 0.274418633018972, + "scr_metric_threshold_100": 0.5278971505140198, + "scr_dir2_threshold_100": 0.5278971505140198, + "scr_dir1_threshold_500": 0.3255813115348554, + "scr_metric_threshold_500": 0.48497847709352376, + "scr_dir2_threshold_500": 0.48497847709352376 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.16744166575873534, "scr_metric_threshold_10": 0.16744166575873534, "scr_dir2_threshold_10": 0.1221719823127949, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2093021515061541, + "scr_metric_threshold_20": 0.2093021515061541, + "scr_dir2_threshold_20": 0.14027156521516332, + "scr_dir1_threshold_50": 0.24186043030123083, + "scr_metric_threshold_50": 0.24186043030123083, + "scr_dir2_threshold_50": 0.20361997052132483, + "scr_dir1_threshold_100": 0.15813945880639327, + "scr_metric_threshold_100": 0.15813945880639327, + "scr_dir2_threshold_100": 0.2443439646255898, + "scr_dir1_threshold_500": 0.34883719640786326, + "scr_metric_threshold_500": 0.34883719640786326, + "scr_dir2_threshold_500": 0.31221719823127947 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-134277-tokens-550000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-134277-tokens-550000000/tpp/saebench_tpp_custom_sae_eval_results.json index 042ba803348cba65e5054b2ad9a9e4e5fc1cb6dd..de6f00aef70917a7919b1d5952bc9762438d24a1 100644 --- a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-134277-tokens-550000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-134277-tokens-550000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "560ab405-89c6-49de-acf7-6ad007a780bc", - "datetime_epoch_millis": 1776888254545, + "eval_id": "1670f09d-ac07-4696-ae2c-68d2f8bbd927", + "datetime_epoch_millis": 1777319057915, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.03337500691413879, - "tpp_threshold_2_intended_diff_only": 0.038200008869171145, - "tpp_threshold_2_unintended_diff_only": 0.004825001955032349, - "tpp_threshold_5_total_metric": 0.0745999976992607, - "tpp_threshold_5_intended_diff_only": 0.10279999971389771, - "tpp_threshold_5_unintended_diff_only": 0.02820000201463699, - "tpp_threshold_10_total_metric": 0.11652500182390213, - "tpp_threshold_10_intended_diff_only": 0.17850000858306886, - "tpp_threshold_10_unintended_diff_only": 0.06197500675916672, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.031624998152256015, + "tpp_threshold_2_intended_diff_only": 0.03760000467300415, + "tpp_threshold_2_unintended_diff_only": 0.005975006520748139, + "tpp_threshold_5_total_metric": 0.0729999989271164, + "tpp_threshold_5_intended_diff_only": 0.10130000710487365, + "tpp_threshold_5_unintended_diff_only": 0.028300008177757262, + "tpp_threshold_10_total_metric": 0.11737499684095383, + "tpp_threshold_10_intended_diff_only": 0.18040000796318056, + "tpp_threshold_10_unintended_diff_only": 0.06302501112222672, + "tpp_threshold_20_total_metric": 0.11775000840425491, + "tpp_threshold_20_intended_diff_only": 0.20590001344680786, + "tpp_threshold_20_unintended_diff_only": 0.08815000504255295, + "tpp_threshold_50_total_metric": 0.12320000827312469, + "tpp_threshold_50_intended_diff_only": 0.24470001459121704, + "tpp_threshold_50_unintended_diff_only": 0.12150000631809234, + "tpp_threshold_100_total_metric": 0.12010001093149186, + "tpp_threshold_100_intended_diff_only": 0.27970002293586727, + "tpp_threshold_100_unintended_diff_only": 0.15960001200437546, + "tpp_threshold_500_total_metric": 0.20087501555681228, + "tpp_threshold_500_intended_diff_only": 0.40230002999305725, + "tpp_threshold_500_unintended_diff_only": 0.20142501443624497 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.05740000903606415, + "tpp_threshold_2_total_metric": 0.05615000724792481, "tpp_threshold_2_intended_diff_only": 0.06700001955032349, - "tpp_threshold_2_unintended_diff_only": 0.009600010514259339, - "tpp_threshold_5_total_metric": 0.13780000805854797, - "tpp_threshold_5_intended_diff_only": 0.19280002117156983, - "tpp_threshold_5_unintended_diff_only": 0.05500001311302185, - "tpp_threshold_10_total_metric": 0.21645000874996184, + "tpp_threshold_2_unintended_diff_only": 0.010850012302398682, + "tpp_threshold_5_total_metric": 0.13659999370574952, + "tpp_threshold_5_intended_diff_only": 0.19340001344680785, + "tpp_threshold_5_unintended_diff_only": 0.05680001974105835, + "tpp_threshold_10_total_metric": 0.21470000445842743, "tpp_threshold_10_intended_diff_only": 0.3380000233650208, - "tpp_threshold_10_unintended_diff_only": 0.1215500146150589, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_10_unintended_diff_only": 0.12330001890659333, + "tpp_threshold_20_total_metric": 0.1996000111103058, + "tpp_threshold_20_intended_diff_only": 0.3728000283241272, + "tpp_threshold_20_unintended_diff_only": 0.17320001721382142, + "tpp_threshold_50_total_metric": 0.17205002009868622, + "tpp_threshold_50_intended_diff_only": 0.41100003719329836, + "tpp_threshold_50_unintended_diff_only": 0.2389500170946121, + "tpp_threshold_100_total_metric": 0.1223000168800354, + "tpp_threshold_100_intended_diff_only": 0.43180004358291624, + "tpp_threshold_100_unintended_diff_only": 0.30950002670288085, + "tpp_threshold_500_total_metric": 0.08255000412464142, + "tpp_threshold_500_intended_diff_only": 0.46420003175735475, + "tpp_threshold_500_unintended_diff_only": 0.38165002763271333 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.00935000479221344, - "tpp_threshold_2_intended_diff_only": 0.009399998188018798, - "tpp_threshold_2_unintended_diff_only": 4.999339580535889e-05, - "tpp_threshold_5_total_metric": 0.011399987339973449, - "tpp_threshold_5_intended_diff_only": 0.012799978256225586, - "tpp_threshold_5_unintended_diff_only": 0.0013999909162521363, - "tpp_threshold_10_total_metric": 0.016599994897842408, - "tpp_threshold_10_intended_diff_only": 0.018999993801116943, - "tpp_threshold_10_unintended_diff_only": 0.0023999989032745363, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.007099989056587219, + "tpp_threshold_2_intended_diff_only": 0.008199989795684814, + "tpp_threshold_2_unintended_diff_only": 0.0011000007390975953, + "tpp_threshold_5_total_metric": 0.009400004148483276, + "tpp_threshold_5_intended_diff_only": 0.009200000762939453, + "tpp_threshold_5_unintended_diff_only": -0.00020000338554382324, + "tpp_threshold_10_total_metric": 0.020049989223480225, + "tpp_threshold_10_intended_diff_only": 0.022799992561340333, + "tpp_threshold_10_unintended_diff_only": 0.0027500033378601075, + "tpp_threshold_20_total_metric": 0.03590000569820404, + "tpp_threshold_20_intended_diff_only": 0.03899999856948853, + "tpp_threshold_20_unintended_diff_only": 0.003099992871284485, + "tpp_threshold_50_total_metric": 0.07434999644756317, + "tpp_threshold_50_intended_diff_only": 0.07839999198913575, + "tpp_threshold_50_unintended_diff_only": 0.0040499955415725705, + "tpp_threshold_100_total_metric": 0.11790000498294831, + "tpp_threshold_100_intended_diff_only": 0.12760000228881835, + "tpp_threshold_100_unintended_diff_only": 0.009699997305870057, + "tpp_threshold_500_total_metric": 0.31920002698898314, + "tpp_threshold_500_intended_diff_only": 0.34040002822875975, + "tpp_threshold_500_unintended_diff_only": 0.02120000123977661 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.003500029444694519, "tpp_threshold_10_total_metric": 0.24050000309944153, "tpp_threshold_10_intended_diff_only": 0.30400002002716064, - "tpp_threshold_10_unintended_diff_only": 0.06350001692771912 + "tpp_threshold_10_unintended_diff_only": 0.06350001692771912, + "tpp_threshold_20_total_metric": 0.22374999523162842, + "tpp_threshold_20_intended_diff_only": 0.3830000162124634, + "tpp_threshold_20_unintended_diff_only": 0.15925002098083496, + "tpp_threshold_50_total_metric": 0.19749997556209564, + "tpp_threshold_50_intended_diff_only": 0.421999990940094, + "tpp_threshold_50_unintended_diff_only": 0.22450001537799835, + "tpp_threshold_100_total_metric": 0.14274999499320984, + "tpp_threshold_100_intended_diff_only": 0.44300001859664917, + "tpp_threshold_100_unintended_diff_only": 0.30025002360343933, + "tpp_threshold_500_total_metric": 0.07899999618530273, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.37700003385543823 }, "1": { - "tpp_threshold_2_total_metric": 0.10599999129772186, - "tpp_threshold_2_intended_diff_only": 0.1420000195503235, - "tpp_threshold_2_unintended_diff_only": 0.036000028252601624, - "tpp_threshold_5_total_metric": 0.16175001859664917, - "tpp_threshold_5_intended_diff_only": 0.26600003242492676, - "tpp_threshold_5_unintended_diff_only": 0.10425001382827759, - "tpp_threshold_10_total_metric": 0.1784999966621399, + "tpp_threshold_2_total_metric": 0.1069999486207962, + "tpp_threshold_2_intended_diff_only": 0.13999998569488525, + "tpp_threshold_2_unintended_diff_only": 0.03300003707408905, + "tpp_threshold_5_total_metric": 0.15724998712539673, + "tpp_threshold_5_intended_diff_only": 0.2670000195503235, + "tpp_threshold_5_unintended_diff_only": 0.10975003242492676, + "tpp_threshold_10_total_metric": 0.17274998128414154, "tpp_threshold_10_intended_diff_only": 0.3190000057220459, - "tpp_threshold_10_unintended_diff_only": 0.140500009059906 + "tpp_threshold_10_unintended_diff_only": 0.14625002443790436, + "tpp_threshold_20_total_metric": 0.1822499930858612, + "tpp_threshold_20_intended_diff_only": 0.3370000123977661, + "tpp_threshold_20_unintended_diff_only": 0.1547500193119049, + "tpp_threshold_50_total_metric": 0.14125002920627594, + "tpp_threshold_50_intended_diff_only": 0.3810000419616699, + "tpp_threshold_50_unintended_diff_only": 0.23975001275539398, + "tpp_threshold_100_total_metric": 0.09400001168251038, + "tpp_threshold_100_intended_diff_only": 0.409000039100647, + "tpp_threshold_100_unintended_diff_only": 0.3150000274181366, + "tpp_threshold_500_total_metric": 0.06299997866153717, + "tpp_threshold_500_intended_diff_only": 0.453000009059906, + "tpp_threshold_500_unintended_diff_only": 0.39000003039836884 }, "2": { - "tpp_threshold_2_total_metric": 0.14824999868869781, - "tpp_threshold_2_intended_diff_only": 0.16100001335144043, - "tpp_threshold_2_unintended_diff_only": 0.012750014662742615, - "tpp_threshold_5_total_metric": 0.22350001335144043, + "tpp_threshold_2_total_metric": 0.13950002193450928, + "tpp_threshold_2_intended_diff_only": 0.1600000262260437, + "tpp_threshold_2_unintended_diff_only": 0.020500004291534424, + "tpp_threshold_5_total_metric": 0.22574998438358307, "tpp_threshold_5_intended_diff_only": 0.2770000100135803, - "tpp_threshold_5_unintended_diff_only": 0.05349999666213989, - "tpp_threshold_10_total_metric": 0.19275005161762238, - "tpp_threshold_10_intended_diff_only": 0.3420000672340393, - "tpp_threshold_10_unintended_diff_only": 0.14925001561641693 + "tpp_threshold_5_unintended_diff_only": 0.05125002562999725, + "tpp_threshold_10_total_metric": 0.19150003790855408, + "tpp_threshold_10_intended_diff_only": 0.34300005435943604, + "tpp_threshold_10_unintended_diff_only": 0.15150001645088196, + "tpp_threshold_20_total_metric": 0.11675000190734863, + "tpp_threshold_20_intended_diff_only": 0.36400002241134644, + "tpp_threshold_20_unintended_diff_only": 0.2472500205039978, + "tpp_threshold_50_total_metric": 0.114500030875206, + "tpp_threshold_50_intended_diff_only": 0.3810000419616699, + "tpp_threshold_50_unintended_diff_only": 0.26650001108646393, + "tpp_threshold_100_total_metric": 0.058750003576278687, + "tpp_threshold_100_intended_diff_only": 0.3960000276565552, + "tpp_threshold_100_unintended_diff_only": 0.3372500240802765, + "tpp_threshold_500_total_metric": 0.05050000548362732, + "tpp_threshold_500_intended_diff_only": 0.45100003480911255, + "tpp_threshold_500_unintended_diff_only": 0.40050002932548523 }, "6": { - "tpp_threshold_2_total_metric": 0.009000048041343689, + "tpp_threshold_2_total_metric": 0.00700002908706665, "tpp_threshold_2_intended_diff_only": 0.006000041961669922, - "tpp_threshold_2_unintended_diff_only": -0.003000006079673767, - "tpp_threshold_5_total_metric": 0.12200000882148743, - "tpp_threshold_5_intended_diff_only": 0.16100001335144043, - "tpp_threshold_5_unintended_diff_only": 0.039000004529953, - "tpp_threshold_10_total_metric": 0.28049999475479126, + "tpp_threshold_2_unintended_diff_only": -0.0009999871253967285, + "tpp_threshold_5_total_metric": 0.12475000321865082, + "tpp_threshold_5_intended_diff_only": 0.16200000047683716, + "tpp_threshold_5_unintended_diff_only": 0.03724999725818634, + "tpp_threshold_10_total_metric": 0.28324998915195465, "tpp_threshold_10_intended_diff_only": 0.4480000138282776, - "tpp_threshold_10_unintended_diff_only": 0.16750001907348633 + "tpp_threshold_10_unintended_diff_only": 0.16475002467632294, + "tpp_threshold_20_total_metric": 0.2600000202655792, + "tpp_threshold_20_intended_diff_only": 0.4650000333786011, + "tpp_threshold_20_unintended_diff_only": 0.20500001311302185, + "tpp_threshold_50_total_metric": 0.21500001847743988, + "tpp_threshold_50_intended_diff_only": 0.47800004482269287, + "tpp_threshold_50_unintended_diff_only": 0.263000026345253, + "tpp_threshold_100_total_metric": 0.1730000376701355, + "tpp_threshold_100_intended_diff_only": 0.48100006580352783, + "tpp_threshold_100_unintended_diff_only": 0.30800002813339233, + "tpp_threshold_500_total_metric": 0.11599999666213989, + "tpp_threshold_500_intended_diff_only": 0.48500001430511475, + "tpp_threshold_500_unintended_diff_only": 0.36900001764297485 }, "9": { - "tpp_threshold_2_total_metric": 0.003000006079673767, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_5_total_metric": 0.1522500365972519, - "tpp_threshold_5_intended_diff_only": 0.22700005769729614, - "tpp_threshold_5_unintended_diff_only": 0.07475002110004425, - "tpp_threshold_10_total_metric": 0.1899999976158142, - "tpp_threshold_10_intended_diff_only": 0.2770000100135803, - "tpp_threshold_10_unintended_diff_only": 0.08700001239776611 + "tpp_threshold_2_total_metric": 0.006500035524368286, + "tpp_threshold_2_intended_diff_only": 0.00700002908706665, + "tpp_threshold_2_unintended_diff_only": 0.0004999935626983643, + "tpp_threshold_5_total_metric": 0.145750030875206, + "tpp_threshold_5_intended_diff_only": 0.22800004482269287, + "tpp_threshold_5_unintended_diff_only": 0.08225001394748688, + "tpp_threshold_10_total_metric": 0.18550001084804535, + "tpp_threshold_10_intended_diff_only": 0.2760000228881836, + "tpp_threshold_10_unintended_diff_only": 0.09050001204013824, + "tpp_threshold_20_total_metric": 0.21525004506111145, + "tpp_threshold_20_intended_diff_only": 0.315000057220459, + "tpp_threshold_20_unintended_diff_only": 0.09975001215934753, + "tpp_threshold_50_total_metric": 0.19200004637241364, + "tpp_threshold_50_intended_diff_only": 0.393000066280365, + "tpp_threshold_50_unintended_diff_only": 0.20100001990795135, + "tpp_threshold_100_total_metric": 0.1430000364780426, + "tpp_threshold_100_intended_diff_only": 0.43000006675720215, + "tpp_threshold_100_unintended_diff_only": 0.28700003027915955, + "tpp_threshold_500_total_metric": 0.10425004363059998, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.37175002694129944 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.008000016212463379, - "tpp_threshold_2_intended_diff_only": 0.003000020980834961, - "tpp_threshold_2_unintended_diff_only": -0.004999995231628418, - "tpp_threshold_5_total_metric": 0.009499981999397278, - "tpp_threshold_5_intended_diff_only": 0.006999969482421875, - "tpp_threshold_5_unintended_diff_only": -0.002500012516975403, - "tpp_threshold_10_total_metric": 0.0014999806880950928, - "tpp_threshold_10_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_10_unintended_diff_only": 0.004500001668930054 + "tpp_threshold_2_total_metric": 0.0020000040531158447, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 0.0029999911785125732, + "tpp_threshold_5_total_metric": 0.0017500221729278564, + "tpp_threshold_5_intended_diff_only": 0.003000020980834961, + "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_10_total_metric": 0.004499971866607666, + "tpp_threshold_10_intended_diff_only": 0.006999969482421875, + "tpp_threshold_10_unintended_diff_only": 0.002499997615814209, + "tpp_threshold_20_total_metric": 0.011000022292137146, + "tpp_threshold_20_intended_diff_only": 0.013000011444091797, + "tpp_threshold_20_unintended_diff_only": 0.001999989151954651, + "tpp_threshold_50_total_metric": 0.030500024557113647, + "tpp_threshold_50_intended_diff_only": 0.027000010013580322, + "tpp_threshold_50_unintended_diff_only": -0.003500014543533325, + "tpp_threshold_100_total_metric": 0.05900001525878906, + "tpp_threshold_100_intended_diff_only": 0.06400001049041748, + "tpp_threshold_100_unintended_diff_only": 0.004999995231628418, + "tpp_threshold_500_total_metric": 0.29250001907348633, + "tpp_threshold_500_intended_diff_only": 0.30400002002716064, + "tpp_threshold_500_unintended_diff_only": 0.011500000953674316 }, "2": { - "tpp_threshold_2_total_metric": 0.0020000338554382324, - "tpp_threshold_2_intended_diff_only": 0.00700002908706665, - "tpp_threshold_2_unintended_diff_only": 0.004999995231628418, - "tpp_threshold_5_total_metric": 0.0025000274181365967, - "tpp_threshold_5_intended_diff_only": 0.009000003337860107, - "tpp_threshold_5_unintended_diff_only": 0.006499975919723511, - "tpp_threshold_10_total_metric": 0.018500030040740967, - "tpp_threshold_10_intended_diff_only": 0.021000027656555176, - "tpp_threshold_10_unintended_diff_only": 0.002499997615814209 + "tpp_threshold_2_total_metric": 0.009250015020370483, + "tpp_threshold_2_intended_diff_only": 0.008000016212463379, + "tpp_threshold_2_unintended_diff_only": -0.0012499988079071045, + "tpp_threshold_5_total_metric": -0.002499982714653015, + "tpp_threshold_5_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_5_unintended_diff_only": 0.006499990820884705, + "tpp_threshold_10_total_metric": 0.02275000512599945, + "tpp_threshold_10_intended_diff_only": 0.027000010013580322, + "tpp_threshold_10_unintended_diff_only": 0.004250004887580872, + "tpp_threshold_20_total_metric": 0.0402500182390213, + "tpp_threshold_20_intended_diff_only": 0.04100000858306885, + "tpp_threshold_20_unintended_diff_only": 0.0007499903440475464, + "tpp_threshold_50_total_metric": 0.08250004053115845, + "tpp_threshold_50_intended_diff_only": 0.09000003337860107, + "tpp_threshold_50_unintended_diff_only": 0.007499992847442627, + "tpp_threshold_100_total_metric": 0.11925001442432404, + "tpp_threshold_100_intended_diff_only": 0.13300001621246338, + "tpp_threshold_100_unintended_diff_only": 0.013750001788139343, + "tpp_threshold_500_total_metric": 0.3252500742673874, + "tpp_threshold_500_intended_diff_only": 0.3420000672340393, + "tpp_threshold_500_unintended_diff_only": 0.016749992966651917 }, "3": { - "tpp_threshold_2_total_metric": 0.01100003719329834, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": -0.00700002908706665, - "tpp_threshold_5_total_metric": 0.018249988555908203, - "tpp_threshold_5_intended_diff_only": 0.014999985694885254, - "tpp_threshold_5_unintended_diff_only": -0.0032500028610229492, - "tpp_threshold_10_total_metric": 0.01150001585483551, - "tpp_threshold_10_intended_diff_only": 0.013000011444091797, - "tpp_threshold_10_unintended_diff_only": 0.0014999955892562866 + "tpp_threshold_2_total_metric": 0.0017499923706054688, + "tpp_threshold_2_intended_diff_only": 0.0009999871253967285, + "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402, + "tpp_threshold_5_total_metric": 0.008999988436698914, + "tpp_threshold_5_intended_diff_only": 0.001999974250793457, + "tpp_threshold_5_unintended_diff_only": -0.0070000141859054565, + "tpp_threshold_10_total_metric": 0.013250023126602173, + "tpp_threshold_10_intended_diff_only": 0.012000024318695068, + "tpp_threshold_10_unintended_diff_only": -0.0012499988079071045, + "tpp_threshold_20_total_metric": 0.014000028371810913, + "tpp_threshold_20_intended_diff_only": 0.017000019550323486, + "tpp_threshold_20_unintended_diff_only": 0.0029999911785125732, + "tpp_threshold_50_total_metric": 0.04274998605251312, + "tpp_threshold_50_intended_diff_only": 0.046999990940093994, + "tpp_threshold_50_unintended_diff_only": 0.004250004887580872, + "tpp_threshold_100_total_metric": 0.08675001561641693, + "tpp_threshold_100_intended_diff_only": 0.09600001573562622, + "tpp_threshold_100_unintended_diff_only": 0.00925000011920929, + "tpp_threshold_500_total_metric": 0.2850000113248825, + "tpp_threshold_500_intended_diff_only": 0.30900001525878906, + "tpp_threshold_500_unintended_diff_only": 0.024000003933906555 }, "5": { - "tpp_threshold_2_total_metric": -0.0012500286102294922, - "tpp_threshold_2_intended_diff_only": 0.006999969482421875, - "tpp_threshold_2_unintended_diff_only": 0.008249998092651367, - "tpp_threshold_5_total_metric": 0.006249964237213135, - "tpp_threshold_5_intended_diff_only": 0.006999969482421875, - "tpp_threshold_5_unintended_diff_only": 0.0007500052452087402, - "tpp_threshold_10_total_metric": 0.012749984860420227, - "tpp_threshold_10_intended_diff_only": 0.015999972820281982, - "tpp_threshold_10_unintended_diff_only": 0.0032499879598617554 + "tpp_threshold_2_total_metric": 0.007499948143959045, + "tpp_threshold_2_intended_diff_only": 0.007999956607818604, + "tpp_threshold_2_unintended_diff_only": 0.0005000084638595581, + "tpp_threshold_5_total_metric": 0.014999985694885254, + "tpp_threshold_5_intended_diff_only": 0.013999998569488525, + "tpp_threshold_5_unintended_diff_only": -0.0009999871253967285, + "tpp_threshold_10_total_metric": 0.008249998092651367, + "tpp_threshold_10_intended_diff_only": 0.013999998569488525, + "tpp_threshold_10_unintended_diff_only": 0.005750000476837158, + "tpp_threshold_20_total_metric": 0.0417499840259552, + "tpp_threshold_20_intended_diff_only": 0.042999982833862305, + "tpp_threshold_20_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_50_total_metric": 0.06074996292591095, + "tpp_threshold_50_intended_diff_only": 0.0679999589920044, + "tpp_threshold_50_unintended_diff_only": 0.007249996066093445, + "tpp_threshold_100_total_metric": 0.11324997246265411, + "tpp_threshold_100_intended_diff_only": 0.12299996614456177, + "tpp_threshold_100_unintended_diff_only": 0.009749993681907654, + "tpp_threshold_500_total_metric": 0.3490000069141388, + "tpp_threshold_500_intended_diff_only": 0.3790000081062317, + "tpp_threshold_500_unintended_diff_only": 0.030000001192092896 }, "6": { - "tpp_threshold_2_total_metric": 0.02699996531009674, - "tpp_threshold_2_intended_diff_only": 0.02599996328353882, - "tpp_threshold_2_unintended_diff_only": -0.0010000020265579224, - "tpp_threshold_5_total_metric": 0.020499974489212036, - "tpp_threshold_5_intended_diff_only": 0.02599996328353882, - "tpp_threshold_5_unintended_diff_only": 0.005499988794326782, - "tpp_threshold_10_total_metric": 0.03874996304512024, - "tpp_threshold_10_intended_diff_only": 0.038999974727630615, - "tpp_threshold_10_unintended_diff_only": 0.000250011682510376 + "tpp_threshold_2_total_metric": 0.014999985694885254, + "tpp_threshold_2_intended_diff_only": 0.018999993801116943, + "tpp_threshold_2_unintended_diff_only": 0.0040000081062316895, + "tpp_threshold_5_total_metric": 0.023750007152557373, + "tpp_threshold_5_intended_diff_only": 0.023000001907348633, + "tpp_threshold_5_unintended_diff_only": -0.0007500052452087402, + "tpp_threshold_10_total_metric": 0.051499947905540466, + "tpp_threshold_10_intended_diff_only": 0.05399996042251587, + "tpp_threshold_10_unintended_diff_only": 0.002500012516975403, + "tpp_threshold_20_total_metric": 0.07249997556209564, + "tpp_threshold_20_intended_diff_only": 0.08099997043609619, + "tpp_threshold_20_unintended_diff_only": 0.00849999487400055, + "tpp_threshold_50_total_metric": 0.1552499681711197, + "tpp_threshold_50_intended_diff_only": 0.15999996662139893, + "tpp_threshold_50_unintended_diff_only": 0.004749998450279236, + "tpp_threshold_100_total_metric": 0.21125000715255737, + "tpp_threshold_100_intended_diff_only": 0.22200000286102295, + "tpp_threshold_100_unintended_diff_only": 0.010749995708465576, + "tpp_threshold_500_total_metric": 0.34425002336502075, + "tpp_threshold_500_intended_diff_only": 0.3680000305175781, + "tpp_threshold_500_unintended_diff_only": 0.023750007152557373 } } } diff --git a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-146484-tokens-600000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-146484-tokens-600000000/scr/saebench_scr_custom_sae_eval_results.json index abaf39b58eb5131330de17fde6c160cd980cf85d..55cfd7c9a4716be45ec0e8ea3b41bd2068dc530c 100644 --- a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-146484-tokens-600000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-146484-tokens-600000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "48b8ae56-ff81-495d-b652-74fe984fbb21", - "datetime_epoch_millis": 1776888369842, + "eval_id": "19c63205-96a4-4aa5-b8f3-5d02880331d0", + "datetime_epoch_millis": 1777319059965, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.2269996703253795, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.11536790303130313, "scr_metric_threshold_10": 0.15406420537849302, "scr_dir2_threshold_10": 0.1461114436369262, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.10054578090880584, + "scr_metric_threshold_20": 0.16612647234739203, + "scr_dir2_threshold_20": 0.15804746820137885, + "scr_dir1_threshold_50": 0.038031759196331824, + "scr_metric_threshold_50": 0.24049722759877468, + "scr_dir2_threshold_50": 0.23975535152764282, + "scr_dir1_threshold_100": -0.11050132059069222, + "scr_metric_threshold_100": 0.2991254303694807, + "scr_dir2_threshold_100": 0.29563442582434907, + "scr_dir1_threshold_500": -0.4047838013012987, + "scr_metric_threshold_500": 0.09110661570025388, + "scr_dir2_threshold_500": 0.0894519093056031 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.14102610148224506, "scr_metric_threshold_10": 0.010362714302611827, "scr_dir2_threshold_10": 0.010362714302611827, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.5000003820810118, + "scr_metric_threshold_20": 0.08808283994795557, + "scr_dir2_threshold_20": 0.08808283994795557, + "scr_dir1_threshold_50": 0.39743608357792887, + "scr_metric_threshold_50": 0.2694299542032543, + "scr_dir2_threshold_50": 0.2694299542032543, + "scr_dir1_threshold_100": 0.025641456706782625, + "scr_metric_threshold_100": 0.3264248056595377, + "scr_dir2_threshold_100": 0.3264248056595377, + "scr_dir1_threshold_500": -2.730768907469913, + "scr_metric_threshold_500": 0.16062168565007506, + "scr_dir2_threshold_500": 0.16062168565007506 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": -0.18939371703965469, "scr_metric_threshold_10": 0.11314984096315267, "scr_dir2_threshold_10": 0.11314984096315267, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.13636365688864727, + "scr_metric_threshold_20": 0.10397564776756013, + "scr_dir2_threshold_20": 0.10397564776756013, + "scr_dir1_threshold_50": -0.613636004448673, + "scr_metric_threshold_50": 0.3608563382335518, + "scr_dir2_threshold_50": 0.3608563382335518, + "scr_dir1_threshold_100": -0.6060603460771347, + "scr_metric_threshold_100": 0.5443425717488585, + "scr_dir2_threshold_100": 0.5443425717488585, + "scr_dir1_threshold_500": -0.007575658371538246, + "scr_metric_threshold_500": -0.2599387548645225, + "scr_dir2_threshold_500": -0.2599387548645225 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.3749997671693945, "scr_metric_threshold_10": 0.1457286913802262, "scr_dir2_threshold_10": 0.1457286913802262, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -1.171874621650266, + "scr_metric_threshold_20": 0.16834176159853043, + "scr_dir2_threshold_20": 0.16834176159853043, + "scr_dir1_threshold_50": -0.20312514551912844, + "scr_metric_threshold_50": 0.17336682175353543, + "scr_dir2_threshold_50": 0.17336682175353543, + "scr_dir1_threshold_100": 0.3593749708961743, + "scr_metric_threshold_100": 0.21859296219014387, + "scr_dir2_threshold_100": 0.21859296219014387, + "scr_dir1_threshold_500": -1.5, + "scr_metric_threshold_500": -0.13316574147193197, + "scr_dir2_threshold_500": -0.13316574147193197 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": -0.2147651516818312, "scr_metric_threshold_10": 0.15286635689969297, "scr_dir2_threshold_10": 0.15286635689969297, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.38255023891863565, + "scr_metric_threshold_20": -0.06369420464437858, + "scr_dir2_threshold_20": -0.06369420464437858, + "scr_dir1_threshold_50": -0.228188198679497, + "scr_metric_threshold_50": -0.25796168066869574, + "scr_dir2_threshold_50": -0.25796168066869574, + "scr_dir1_threshold_100": -1.583893143665206, + "scr_metric_threshold_100": -0.28025476618845013, + "scr_dir2_threshold_100": -0.28025476618845013, + "scr_dir1_threshold_500": -0.093959728858851, + "scr_metric_threshold_500": -0.4840764488389054, + "scr_dir2_threshold_500": -0.4840764488389054 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.052631529414422, "scr_metric_threshold_10": 0.2182539673153717, "scr_dir2_threshold_10": 0.2182539673153717, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.06842111372221286, + "scr_metric_threshold_20": 0.3571429416165487, + "scr_dir2_threshold_20": 0.3571429416165487, + "scr_dir1_threshold_50": -0.047368439214711934, + "scr_metric_threshold_50": 0.5634920653692567, + "scr_dir2_threshold_50": 0.5634920653692567, + "scr_dir1_threshold_100": -0.01052618039942014, + "scr_metric_threshold_100": 0.6349206536925663, + "scr_dir2_threshold_100": 0.6349206536925663, + "scr_dir1_threshold_500": -0.015789270599130212, + "scr_metric_threshold_500": 0.6150794054390177, + "scr_dir2_threshold_500": 0.6150794054390177 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.377049276438573, "scr_metric_threshold_10": 0.11068695256721958, "scr_dir2_threshold_10": 0.11068695256721958, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.4098360709132541, + "scr_metric_threshold_20": 0.15648856785294307, + "scr_dir2_threshold_20": 0.15648856785294307, + "scr_dir1_threshold_50": 0.4316940424658739, + "scr_metric_threshold_50": 0.20992367193697561, + "scr_dir2_threshold_50": 0.20992367193697561, + "scr_dir1_threshold_100": 0.5081967800457946, + "scr_metric_threshold_100": 0.35877852349298656, + "scr_dir2_threshold_100": 0.35877852349298656, + "scr_dir1_threshold_500": 0.5191256029678558, + "scr_metric_threshold_500": 0.21755716073528464, + "scr_dir2_threshold_500": 0.21755716073528464 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.20465126852525933, "scr_metric_threshold_10": 0.3047209696576519, "scr_dir2_threshold_10": 0.3047209696576519, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.2651161514146259, + "scr_metric_threshold_20": 0.3047209696576519, + "scr_dir2_threshold_20": 0.3047209696576519, + "scr_dir1_threshold_50": 0.3488370996994263, + "scr_metric_threshold_50": 0.3862660142728834, + "scr_dir2_threshold_50": 0.3862660142728834, + "scr_dir1_threshold_100": 0.22790705668983022, + "scr_metric_threshold_100": 0.3948498512825616, + "scr_dir2_threshold_100": 0.3948498512825616, + "scr_dir1_threshold_500": 0.26976753083223043, + "scr_metric_threshold_500": 0.29184559786405584, + "scr_dir2_threshold_500": 0.29184559786405584 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.17674414994201715, "scr_metric_threshold_10": 0.17674414994201715, "scr_dir2_threshold_10": 0.1131220560094828, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.21395325498232515, + "scr_metric_threshold_20": 0.21395325498232515, + "scr_dir2_threshold_20": 0.1493212218142196, + "scr_dir1_threshold_50": 0.21860463568943592, + "scr_metric_threshold_50": 0.21860463568943592, + "scr_dir2_threshold_50": 0.21266962712038115, + "scr_dir1_threshold_100": 0.19534884107764103, + "scr_metric_threshold_100": 0.19534884107764103, + "scr_dir2_threshold_100": 0.16742080471658802, + "scr_dir1_threshold_500": 0.3209300210889576, + "scr_metric_threshold_500": 0.3209300210889576, + "scr_dir2_threshold_500": 0.30769236993175136 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-146484-tokens-600000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-146484-tokens-600000000/tpp/saebench_tpp_custom_sae_eval_results.json index dc170a4ace04a00e2073e15015193485ce6b8ff7..24903f31ff7e9e4a99af5b49a92ac54ef432c769 100644 --- a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-146484-tokens-600000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-146484-tokens-600000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "8fbbf8f9-b010-445b-8d99-533f3d56cc59", - "datetime_epoch_millis": 1776888528227, + "eval_id": "4f6a4dc9-ca6c-4041-9d69-6d115f94e0cf", + "datetime_epoch_millis": 1777319372447, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.04437500685453415, - "tpp_threshold_2_intended_diff_only": 0.05000000596046448, - "tpp_threshold_2_unintended_diff_only": 0.005624999105930328, - "tpp_threshold_5_total_metric": 0.08774999082088471, - "tpp_threshold_5_intended_diff_only": 0.12080000042915344, - "tpp_threshold_5_unintended_diff_only": 0.03305000960826874, - "tpp_threshold_10_total_metric": 0.11149999648332595, - "tpp_threshold_10_intended_diff_only": 0.1837000012397766, - "tpp_threshold_10_unintended_diff_only": 0.07220000475645066, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.04277501255273819, + "tpp_threshold_2_intended_diff_only": 0.04930001497268677, + "tpp_threshold_2_unintended_diff_only": 0.006525002419948578, + "tpp_threshold_5_total_metric": 0.08607500344514847, + "tpp_threshold_5_intended_diff_only": 0.11950001120567322, + "tpp_threshold_5_unintended_diff_only": 0.03342500776052475, + "tpp_threshold_10_total_metric": 0.11242499649524688, + "tpp_threshold_10_intended_diff_only": 0.1856000006198883, + "tpp_threshold_10_unintended_diff_only": 0.07317500412464142, + "tpp_threshold_20_total_metric": 0.10940001308917999, + "tpp_threshold_20_intended_diff_only": 0.2115000247955322, + "tpp_threshold_20_unintended_diff_only": 0.10210001170635224, + "tpp_threshold_50_total_metric": 0.11167500615119934, + "tpp_threshold_50_intended_diff_only": 0.25110002160072326, + "tpp_threshold_50_unintended_diff_only": 0.13942501544952393, + "tpp_threshold_100_total_metric": 0.12027499824762344, + "tpp_threshold_100_intended_diff_only": 0.2863000094890594, + "tpp_threshold_100_unintended_diff_only": 0.16602501124143598, + "tpp_threshold_500_total_metric": 0.20412500500679015, + "tpp_threshold_500_intended_diff_only": 0.40910001993179324, + "tpp_threshold_500_unintended_diff_only": 0.20497501492500306 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.08115001320838929, - "tpp_threshold_2_intended_diff_only": 0.09200001955032348, - "tpp_threshold_2_unintended_diff_only": 0.010850006341934204, - "tpp_threshold_5_total_metric": 0.16499999761581421, - "tpp_threshold_5_intended_diff_only": 0.22900002002716063, - "tpp_threshold_5_unintended_diff_only": 0.06400002241134643, - "tpp_threshold_10_total_metric": 0.20719999969005584, + "tpp_threshold_2_total_metric": 0.08020002543926238, + "tpp_threshold_2_intended_diff_only": 0.09200003147125244, + "tpp_threshold_2_unintended_diff_only": 0.011800006031990051, + "tpp_threshold_5_total_metric": 0.1632000118494034, + "tpp_threshold_5_intended_diff_only": 0.22940002679824828, + "tpp_threshold_5_unintended_diff_only": 0.0662000149488449, + "tpp_threshold_10_total_metric": 0.2054999977350235, "tpp_threshold_10_intended_diff_only": 0.34900001287460325, - "tpp_threshold_10_unintended_diff_only": 0.14180001318454744, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_10_unintended_diff_only": 0.14350001513957977, + "tpp_threshold_20_total_metric": 0.1825000286102295, + "tpp_threshold_20_intended_diff_only": 0.3830000519752502, + "tpp_threshold_20_unintended_diff_only": 0.20050002336502076, + "tpp_threshold_50_total_metric": 0.13855000734329223, + "tpp_threshold_50_intended_diff_only": 0.4130000352859497, + "tpp_threshold_50_unintended_diff_only": 0.2744500279426575, + "tpp_threshold_100_total_metric": 0.10954999923706055, + "tpp_threshold_100_intended_diff_only": 0.4306000232696533, + "tpp_threshold_100_unintended_diff_only": 0.32105002403259275, + "tpp_threshold_500_total_metric": 0.07929999232292176, + "tpp_threshold_500_intended_diff_only": 0.4648000240325928, + "tpp_threshold_500_unintended_diff_only": 0.385500031709671 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.0076000005006790165, - "tpp_threshold_2_intended_diff_only": 0.007999992370605469, - "tpp_threshold_2_unintended_diff_only": 0.00039999186992645264, - "tpp_threshold_5_total_metric": 0.0104999840259552, - "tpp_threshold_5_intended_diff_only": 0.01259998083114624, - "tpp_threshold_5_unintended_diff_only": 0.00209999680519104, - "tpp_threshold_10_total_metric": 0.01579999327659607, - "tpp_threshold_10_intended_diff_only": 0.01839998960494995, - "tpp_threshold_10_unintended_diff_only": 0.0025999963283538817, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.00534999966621399, + "tpp_threshold_2_intended_diff_only": 0.006599998474121094, + "tpp_threshold_2_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_5_total_metric": 0.008949995040893555, + "tpp_threshold_5_intended_diff_only": 0.009599995613098145, + "tpp_threshold_5_unintended_diff_only": 0.0006500005722045899, + "tpp_threshold_10_total_metric": 0.019349995255470275, + "tpp_threshold_10_intended_diff_only": 0.02219998836517334, + "tpp_threshold_10_unintended_diff_only": 0.002849993109703064, + "tpp_threshold_20_total_metric": 0.036299997568130495, + "tpp_threshold_20_intended_diff_only": 0.03999999761581421, + "tpp_threshold_20_unintended_diff_only": 0.0037000000476837156, + "tpp_threshold_50_total_metric": 0.08480000495910645, + "tpp_threshold_50_intended_diff_only": 0.08920000791549683, + "tpp_threshold_50_unintended_diff_only": 0.004400002956390381, + "tpp_threshold_100_total_metric": 0.13099999725818634, + "tpp_threshold_100_intended_diff_only": 0.14199999570846558, + "tpp_threshold_100_unintended_diff_only": 0.010999998450279236, + "tpp_threshold_500_total_metric": 0.32895001769065857, + "tpp_threshold_500_intended_diff_only": 0.3534000158309937, + "tpp_threshold_500_unintended_diff_only": 0.024449998140335084 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.022500023245811462, "tpp_threshold_10_total_metric": 0.23749996721744537, "tpp_threshold_10_intended_diff_only": 0.32099997997283936, - "tpp_threshold_10_unintended_diff_only": 0.08350001275539398 + "tpp_threshold_10_unintended_diff_only": 0.08350001275539398, + "tpp_threshold_20_total_metric": 0.18799999356269836, + "tpp_threshold_20_intended_diff_only": 0.39100003242492676, + "tpp_threshold_20_unintended_diff_only": 0.2030000388622284, + "tpp_threshold_50_total_metric": 0.1614999920129776, + "tpp_threshold_50_intended_diff_only": 0.4240000247955322, + "tpp_threshold_50_unintended_diff_only": 0.2625000327825546, + "tpp_threshold_100_total_metric": 0.1199999451637268, + "tpp_threshold_100_intended_diff_only": 0.43699997663497925, + "tpp_threshold_100_unintended_diff_only": 0.31700003147125244, + "tpp_threshold_500_total_metric": 0.0717499852180481, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.38425004482269287 }, "1": { - "tpp_threshold_2_total_metric": 0.06974998116493225, - "tpp_threshold_2_intended_diff_only": 0.08799999952316284, - "tpp_threshold_2_unintended_diff_only": 0.01825001835823059, - "tpp_threshold_5_total_metric": 0.17274999618530273, - "tpp_threshold_5_intended_diff_only": 0.2990000247955322, - "tpp_threshold_5_unintended_diff_only": 0.1262500286102295, - "tpp_threshold_10_total_metric": 0.1849999725818634, + "tpp_threshold_2_total_metric": 0.07199999690055847, + "tpp_threshold_2_intended_diff_only": 0.08700001239776611, + "tpp_threshold_2_unintended_diff_only": 0.015000015497207642, + "tpp_threshold_5_total_metric": 0.16649998724460602, + "tpp_threshold_5_intended_diff_only": 0.30000001192092896, + "tpp_threshold_5_unintended_diff_only": 0.13350002467632294, + "tpp_threshold_10_total_metric": 0.1784999668598175, "tpp_threshold_10_intended_diff_only": 0.3199999928474426, - "tpp_threshold_10_unintended_diff_only": 0.13500002026557922 + "tpp_threshold_10_unintended_diff_only": 0.14150002598762512, + "tpp_threshold_20_total_metric": 0.1847500205039978, + "tpp_threshold_20_intended_diff_only": 0.34800004959106445, + "tpp_threshold_20_unintended_diff_only": 0.16325002908706665, + "tpp_threshold_50_total_metric": 0.12850001454353333, + "tpp_threshold_50_intended_diff_only": 0.3720000386238098, + "tpp_threshold_50_unintended_diff_only": 0.2435000240802765, + "tpp_threshold_100_total_metric": 0.1054999828338623, + "tpp_threshold_100_intended_diff_only": 0.406000018119812, + "tpp_threshold_100_unintended_diff_only": 0.3005000352859497, + "tpp_threshold_500_total_metric": 0.0612499862909317, + "tpp_threshold_500_intended_diff_only": 0.453000009059906, + "tpp_threshold_500_unintended_diff_only": 0.3917500227689743 }, "2": { - "tpp_threshold_2_total_metric": 0.1835000365972519, - "tpp_threshold_2_intended_diff_only": 0.20500004291534424, - "tpp_threshold_2_unintended_diff_only": 0.021500006318092346, - "tpp_threshold_5_total_metric": 0.23925001919269562, - "tpp_threshold_5_intended_diff_only": 0.3030000329017639, - "tpp_threshold_5_unintended_diff_only": 0.0637500137090683, - "tpp_threshold_10_total_metric": 0.18550004065036774, - "tpp_threshold_10_intended_diff_only": 0.35700005292892456, - "tpp_threshold_10_unintended_diff_only": 0.17150001227855682 + "tpp_threshold_2_total_metric": 0.17625005543231964, + "tpp_threshold_2_intended_diff_only": 0.2040000557899475, + "tpp_threshold_2_unintended_diff_only": 0.02775000035762787, + "tpp_threshold_5_total_metric": 0.23850004374980927, + "tpp_threshold_5_intended_diff_only": 0.3020000457763672, + "tpp_threshold_5_unintended_diff_only": 0.06350000202655792, + "tpp_threshold_10_total_metric": 0.1832500398159027, + "tpp_threshold_10_intended_diff_only": 0.3580000400543213, + "tpp_threshold_10_unintended_diff_only": 0.17475000023841858, + "tpp_threshold_20_total_metric": 0.12050004303455353, + "tpp_threshold_20_intended_diff_only": 0.3750000596046448, + "tpp_threshold_20_unintended_diff_only": 0.25450001657009125, + "tpp_threshold_50_total_metric": 0.08000002801418304, + "tpp_threshold_50_intended_diff_only": 0.3940000534057617, + "tpp_threshold_50_unintended_diff_only": 0.3140000253915787, + "tpp_threshold_100_total_metric": 0.05925002694129944, + "tpp_threshold_100_intended_diff_only": 0.40800005197525024, + "tpp_threshold_100_unintended_diff_only": 0.3487500250339508, + "tpp_threshold_500_total_metric": 0.04975000023841858, + "tpp_threshold_500_intended_diff_only": 0.45100003480911255, + "tpp_threshold_500_unintended_diff_only": 0.40125003457069397 }, "6": { - "tpp_threshold_2_total_metric": 0.010000035166740417, + "tpp_threshold_2_total_metric": 0.008000016212463379, "tpp_threshold_2_intended_diff_only": 0.00700002908706665, - "tpp_threshold_2_unintended_diff_only": -0.003000006079673767, - "tpp_threshold_5_total_metric": 0.13824999332427979, - "tpp_threshold_5_intended_diff_only": 0.17900002002716064, - "tpp_threshold_5_unintended_diff_only": 0.04075002670288086, - "tpp_threshold_10_total_metric": 0.265500009059906, + "tpp_threshold_2_unintended_diff_only": -0.0009999871253967285, + "tpp_threshold_5_total_metric": 0.13999998569488525, + "tpp_threshold_5_intended_diff_only": 0.18000000715255737, + "tpp_threshold_5_unintended_diff_only": 0.04000002145767212, + "tpp_threshold_10_total_metric": 0.2670000046491623, "tpp_threshold_10_intended_diff_only": 0.43800002336502075, - "tpp_threshold_10_unintended_diff_only": 0.17250001430511475 + "tpp_threshold_10_unintended_diff_only": 0.17100001871585846, + "tpp_threshold_20_total_metric": 0.25300003588199615, + "tpp_threshold_20_intended_diff_only": 0.4590000510215759, + "tpp_threshold_20_unintended_diff_only": 0.20600001513957977, + "tpp_threshold_50_total_metric": 0.19499999284744263, + "tpp_threshold_50_intended_diff_only": 0.4790000319480896, + "tpp_threshold_50_unintended_diff_only": 0.284000039100647, + "tpp_threshold_100_total_metric": 0.15800000727176666, + "tpp_threshold_100_intended_diff_only": 0.4830000400543213, + "tpp_threshold_100_unintended_diff_only": 0.3250000327825546, + "tpp_threshold_500_total_metric": 0.11225000023841858, + "tpp_threshold_500_intended_diff_only": 0.48900002241134644, + "tpp_threshold_500_unintended_diff_only": 0.37675002217292786 }, "9": { - "tpp_threshold_2_total_metric": 0.003000006079673767, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, - "tpp_threshold_5_total_metric": 0.12224999070167542, - "tpp_threshold_5_intended_diff_only": 0.18900001049041748, - "tpp_threshold_5_unintended_diff_only": 0.06675001978874207, - "tpp_threshold_10_total_metric": 0.16250000894069672, - "tpp_threshold_10_intended_diff_only": 0.30900001525878906, - "tpp_threshold_10_unintended_diff_only": 0.14650000631809235 + "tpp_threshold_2_total_metric": 0.0052500516176223755, + "tpp_threshold_2_intended_diff_only": 0.006000041961669922, + "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.11850005388259888, + "tpp_threshold_5_intended_diff_only": 0.19000005722045898, + "tpp_threshold_5_unintended_diff_only": 0.07150000333786011, + "tpp_threshold_10_total_metric": 0.1612500101327896, + "tpp_threshold_10_intended_diff_only": 0.30800002813339233, + "tpp_threshold_10_unintended_diff_only": 0.14675001800060272, + "tpp_threshold_20_total_metric": 0.1662500500679016, + "tpp_threshold_20_intended_diff_only": 0.3420000672340393, + "tpp_threshold_20_unintended_diff_only": 0.1757500171661377, + "tpp_threshold_50_total_metric": 0.12775000929832458, + "tpp_threshold_50_intended_diff_only": 0.3960000276565552, + "tpp_threshold_50_unintended_diff_only": 0.2682500183582306, + "tpp_threshold_100_total_metric": 0.10500003397464752, + "tpp_threshold_100_intended_diff_only": 0.4190000295639038, + "tpp_threshold_100_unintended_diff_only": 0.3139999955892563, + "tpp_threshold_500_total_metric": 0.10149998962879181, + "tpp_threshold_500_intended_diff_only": 0.4750000238418579, + "tpp_threshold_500_unintended_diff_only": 0.3735000342130661 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.009749993681907654, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": -0.004749998450279236, - "tpp_threshold_5_total_metric": 0.00849999487400055, - "tpp_threshold_5_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_5_unintended_diff_only": -0.002500012516975403, - "tpp_threshold_10_total_metric": 0.000250011682510376, + "tpp_threshold_2_total_metric": 0.0029999911785125732, + "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_2_unintended_diff_only": 0.0029999911785125732, + "tpp_threshold_5_total_metric": 0.0015000253915786743, + "tpp_threshold_5_intended_diff_only": 0.003000020980834961, + "tpp_threshold_5_unintended_diff_only": 0.0014999955892562866, + "tpp_threshold_10_total_metric": 0.0020000040531158447, "tpp_threshold_10_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_10_unintended_diff_only": 0.0037499964237213135 + "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447, + "tpp_threshold_20_total_metric": 0.016499996185302734, + "tpp_threshold_20_intended_diff_only": 0.018000006675720215, + "tpp_threshold_20_unintended_diff_only": 0.0015000104904174805, + "tpp_threshold_50_total_metric": 0.03650002181529999, + "tpp_threshold_50_intended_diff_only": 0.03100001811981201, + "tpp_threshold_50_unintended_diff_only": -0.005500003695487976, + "tpp_threshold_100_total_metric": 0.06350001692771912, + "tpp_threshold_100_intended_diff_only": 0.06800001859664917, + "tpp_threshold_100_unintended_diff_only": 0.004500001668930054, + "tpp_threshold_500_total_metric": 0.30675002932548523, + "tpp_threshold_500_intended_diff_only": 0.31700003147125244, + "tpp_threshold_500_unintended_diff_only": 0.010250002145767212 }, "2": { - "tpp_threshold_2_total_metric": 0.003250032663345337, - "tpp_threshold_2_intended_diff_only": 0.008000016212463379, - "tpp_threshold_2_unintended_diff_only": 0.004749983549118042, - "tpp_threshold_5_total_metric": 0.0012500286102294922, - "tpp_threshold_5_intended_diff_only": 0.008000016212463379, - "tpp_threshold_5_unintended_diff_only": 0.006749987602233887, - "tpp_threshold_10_total_metric": 0.02475002408027649, - "tpp_threshold_10_intended_diff_only": 0.027000010013580322, - "tpp_threshold_10_unintended_diff_only": 0.002249985933303833 + "tpp_threshold_2_total_metric": 0.009750008583068848, + "tpp_threshold_2_intended_diff_only": 0.009000003337860107, + "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402, + "tpp_threshold_5_total_metric": -0.0032499879598617554, + "tpp_threshold_5_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_5_unintended_diff_only": 0.007249996066093445, + "tpp_threshold_10_total_metric": 0.027250036597251892, + "tpp_threshold_10_intended_diff_only": 0.03100001811981201, + "tpp_threshold_10_unintended_diff_only": 0.0037499815225601196, + "tpp_threshold_20_total_metric": 0.04325002431869507, + "tpp_threshold_20_intended_diff_only": 0.04500001668930054, + "tpp_threshold_20_unintended_diff_only": 0.0017499923706054688, + "tpp_threshold_50_total_metric": 0.1132500171661377, + "tpp_threshold_50_intended_diff_only": 0.12400001287460327, + "tpp_threshold_50_unintended_diff_only": 0.010749995708465576, + "tpp_threshold_100_total_metric": 0.1597500592470169, + "tpp_threshold_100_intended_diff_only": 0.1770000457763672, + "tpp_threshold_100_unintended_diff_only": 0.01724998652935028, + "tpp_threshold_500_total_metric": 0.35225003957748413, + "tpp_threshold_500_intended_diff_only": 0.37700003385543823, + "tpp_threshold_500_unintended_diff_only": 0.0247499942779541 }, "3": { - "tpp_threshold_2_total_metric": 0.010500013828277588, - "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_2_unintended_diff_only": -0.0065000057220458984, - "tpp_threshold_5_total_metric": 0.01774996519088745, - "tpp_threshold_5_intended_diff_only": 0.015999972820281982, - "tpp_threshold_5_unintended_diff_only": -0.0017499923706054688, - "tpp_threshold_10_total_metric": 0.010500028729438782, - "tpp_threshold_10_intended_diff_only": 0.012000024318695068, - "tpp_threshold_10_unintended_diff_only": 0.0014999955892562866 + "tpp_threshold_2_total_metric": 0.0005000084638595581, + "tpp_threshold_2_intended_diff_only": 0.0, + "tpp_threshold_2_unintended_diff_only": -0.0005000084638595581, + "tpp_threshold_5_total_metric": 0.010749965906143188, + "tpp_threshold_5_intended_diff_only": 0.0059999823570251465, + "tpp_threshold_5_unintended_diff_only": -0.004749983549118042, + "tpp_threshold_10_total_metric": 0.011500000953674316, + "tpp_threshold_10_intended_diff_only": 0.009999990463256836, + "tpp_threshold_10_unintended_diff_only": -0.0015000104904174805, + "tpp_threshold_20_total_metric": 0.015750005841255188, + "tpp_threshold_20_intended_diff_only": 0.018999993801116943, + "tpp_threshold_20_unintended_diff_only": 0.0032499879598617554, + "tpp_threshold_50_total_metric": 0.04525002837181091, + "tpp_threshold_50_intended_diff_only": 0.04400002956390381, + "tpp_threshold_50_unintended_diff_only": -0.0012499988079071045, + "tpp_threshold_100_total_metric": 0.09024997055530548, + "tpp_threshold_100_intended_diff_only": 0.0989999771118164, + "tpp_threshold_100_unintended_diff_only": 0.008750006556510925, + "tpp_threshold_500_total_metric": 0.304500013589859, + "tpp_threshold_500_intended_diff_only": 0.328000009059906, + "tpp_threshold_500_unintended_diff_only": 0.023499995470046997 }, "5": { - "tpp_threshold_2_total_metric": -0.004500046372413635, - "tpp_threshold_2_intended_diff_only": 0.003999948501586914, - "tpp_threshold_2_unintended_diff_only": 0.00849999487400055, - "tpp_threshold_5_total_metric": 0.006749957799911499, - "tpp_threshold_5_intended_diff_only": 0.007999956607818604, - "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045, - "tpp_threshold_10_total_metric": 0.01474994421005249, - "tpp_threshold_10_intended_diff_only": 0.01799994707107544, - "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492 + "tpp_threshold_2_total_metric": 0.004249975085258484, + "tpp_threshold_2_intended_diff_only": 0.004999995231628418, + "tpp_threshold_2_unintended_diff_only": 0.0007500201463699341, + "tpp_threshold_5_total_metric": 0.0157499760389328, + "tpp_threshold_5_intended_diff_only": 0.014999985694885254, + "tpp_threshold_5_unintended_diff_only": -0.0007499903440475464, + "tpp_threshold_10_total_metric": 0.006999954581260681, + "tpp_threshold_10_intended_diff_only": 0.012999951839447021, + "tpp_threshold_10_unintended_diff_only": 0.00599999725818634, + "tpp_threshold_20_total_metric": 0.03999999165534973, + "tpp_threshold_20_intended_diff_only": 0.041999995708465576, + "tpp_threshold_20_unintended_diff_only": 0.0020000040531158447, + "tpp_threshold_50_total_metric": 0.06499995291233063, + "tpp_threshold_50_intended_diff_only": 0.07199996709823608, + "tpp_threshold_50_unintended_diff_only": 0.0070000141859054565, + "tpp_threshold_100_total_metric": 0.11949996650218964, + "tpp_threshold_100_intended_diff_only": 0.12799996137619019, + "tpp_threshold_100_unintended_diff_only": 0.00849999487400055, + "tpp_threshold_500_total_metric": 0.3447500020265579, + "tpp_threshold_500_intended_diff_only": 0.375, + "tpp_threshold_500_unintended_diff_only": 0.030249997973442078 }, "6": { - "tpp_threshold_2_total_metric": 0.019000008702278137, - "tpp_threshold_2_intended_diff_only": 0.018999993801116943, - "tpp_threshold_2_unintended_diff_only": -1.4901161193847656e-08, - "tpp_threshold_5_total_metric": 0.01824997365474701, - "tpp_threshold_5_intended_diff_only": 0.02499997615814209, - "tpp_threshold_5_unintended_diff_only": 0.006750002503395081, - "tpp_threshold_10_total_metric": 0.02874995768070221, - "tpp_threshold_10_intended_diff_only": 0.030999958515167236, - "tpp_threshold_10_unintended_diff_only": 0.002250000834465027 + "tpp_threshold_2_total_metric": 0.009250015020370483, + "tpp_threshold_2_intended_diff_only": 0.013000011444091797, + "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135, + "tpp_threshold_5_total_metric": 0.019999995827674866, + "tpp_threshold_5_intended_diff_only": 0.019999980926513672, + "tpp_threshold_5_unintended_diff_only": -1.4901161193847656e-08, + "tpp_threshold_10_total_metric": 0.048999980092048645, + "tpp_threshold_10_intended_diff_only": 0.05299997329711914, + "tpp_threshold_10_unintended_diff_only": 0.003999993205070496, + "tpp_threshold_20_total_metric": 0.06599996984004974, + "tpp_threshold_20_intended_diff_only": 0.07599997520446777, + "tpp_threshold_20_unintended_diff_only": 0.01000000536441803, + "tpp_threshold_50_total_metric": 0.164000004529953, + "tpp_threshold_50_intended_diff_only": 0.17500001192092896, + "tpp_threshold_50_unintended_diff_only": 0.011000007390975952, + "tpp_threshold_100_total_metric": 0.22199997305870056, + "tpp_threshold_100_intended_diff_only": 0.23799997568130493, + "tpp_threshold_100_unintended_diff_only": 0.01600000262260437, + "tpp_threshold_500_total_metric": 0.33650000393390656, + "tpp_threshold_500_intended_diff_only": 0.3700000047683716, + "tpp_threshold_500_unintended_diff_only": 0.03350000083446503 } } } diff --git a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-158691-tokens-650000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-158691-tokens-650000000/scr/saebench_scr_custom_sae_eval_results.json index a11e47ae14d2bb6f68263d8c685462d20ebcefc7..03e799f76b840d609f3108952dfbd68bc5ce8a40 100644 --- a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-158691-tokens-650000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-158691-tokens-650000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "2f75ab13-09ac-478d-8fa2-69199499964b", - "datetime_epoch_millis": 1776888334482, + "eval_id": "e6406764-1ed4-4cda-a900-1c88ce223ce1", + "datetime_epoch_millis": 1777319142134, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.22364544466588532, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": -0.111770642278261, "scr_metric_threshold_10": 0.1744125918751408, "scr_dir2_threshold_10": 0.1681724588560094, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.08817796219582252, + "scr_metric_threshold_20": 0.18381757334274101, + "scr_dir2_threshold_20": 0.1798241037893253, + "scr_dir1_threshold_50": 0.022856066870728303, + "scr_metric_threshold_50": 0.2596957648258872, + "scr_dir2_threshold_50": 0.2514430957764516, + "scr_dir1_threshold_100": -0.2609792460766421, + "scr_metric_threshold_100": 0.26471930044768543, + "scr_dir2_threshold_100": 0.25355965707177547, + "scr_dir1_threshold_500": -0.8296813798712246, + "scr_metric_threshold_500": 0.15681855274671544, + "scr_dir2_threshold_500": 0.14204427005149586 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": -0.3205124776196047, "scr_metric_threshold_10": 0.07253884570211949, "scr_dir2_threshold_10": 0.07253884570211949, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.4615385791018498, + "scr_metric_threshold_20": 0.17875647428368657, + "scr_dir2_threshold_20": 0.17875647428368657, + "scr_dir1_threshold_50": 0.29487178507484585, + "scr_metric_threshold_50": 0.24352324565480638, + "scr_dir2_threshold_50": 0.24352324565480638, + "scr_dir1_threshold_100": 0.2820514388024664, + "scr_metric_threshold_100": 0.33160624001892525, + "scr_dir2_threshold_100": 0.33160624001892525, + "scr_dir1_threshold_500": -2.7179485611975336, + "scr_metric_threshold_500": 0.2046631828321345, + "scr_dir2_threshold_500": 0.2046631828321345 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.34090914222161817, "scr_metric_threshold_10": 0.12844034523299583, "scr_dir2_threshold_10": 0.12844034523299583, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.41666662903748003, + "scr_metric_threshold_20": 0.03363925521540607, + "scr_dir2_threshold_20": 0.03363925521540607, + "scr_dir1_threshold_50": -0.5681816026692038, + "scr_metric_threshold_50": 0.3241590186196148, + "scr_dir2_threshold_50": 0.3241590186196148, + "scr_dir1_threshold_100": -1.6439390894850656, + "scr_metric_threshold_100": 0.10091740109184036, + "scr_dir2_threshold_100": 0.10091740109184036, + "scr_dir1_threshold_500": -0.7348483445942438, + "scr_metric_threshold_500": 0.02446487974262461, + "scr_dir2_threshold_500": 0.02446487974262461 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": -0.8437501746229541, "scr_metric_threshold_10": 0.13316589123232284, "scr_dir2_threshold_10": 0.13316589123232284, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -1.2812500582076514, + "scr_metric_threshold_20": 0.1783920316689313, + "scr_dir2_threshold_20": 0.1783920316689313, + "scr_dir1_threshold_50": -0.5312495925464404, + "scr_metric_threshold_50": 0.2386935023309456, + "scr_dir2_threshold_50": 0.2386935023309456, + "scr_dir1_threshold_100": -0.031249592546440393, + "scr_metric_threshold_100": 0.2412060324084481, + "scr_dir2_threshold_100": 0.2412060324084481, + "scr_dir1_threshold_500": -2.578124912688523, + "scr_metric_threshold_500": -0.16331655168313458, + "scr_dir2_threshold_500": -0.16331655168313458 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": -0.7718122013517055, "scr_metric_threshold_10": 0.11146504795136565, "scr_dir2_threshold_10": 0.11146504795136565, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.24832216912919208, + "scr_metric_threshold_20": -0.11783439248632226, + "scr_dir2_threshold_20": -0.11783439248632226, + "scr_dir1_threshold_50": 0.08053668186118518, + "scr_metric_threshold_50": -0.28343943845592845, + "scr_dir2_threshold_50": -0.28343943845592845, + "scr_dir1_threshold_100": -1.604027114114901, + "scr_metric_threshold_100": -0.2547770084012175, + "scr_dir2_threshold_100": -0.2547770084012175, + "scr_dir1_threshold_500": -1.6845641960072886, + "scr_metric_threshold_500": -0.5382164468571459, + "scr_dir2_threshold_500": -0.5382164468571459 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.057894933322792726, "scr_metric_threshold_10": 0.27777771207601765, "scr_dir2_threshold_10": 0.27777771207601765, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.04210534901500186, + "scr_metric_threshold_20": 0.37698418987009735, + "scr_dir2_threshold_20": 0.37698418987009735, + "scr_dir1_threshold_50": -0.01052618039942014, + "scr_metric_threshold_50": 0.5912697183136902, + "scr_dir2_threshold_50": 0.5912697183136902, + "scr_dir1_threshold_100": -0.026315764707211, + "scr_metric_threshold_100": 0.6468252607288938, + "scr_dir2_threshold_100": 0.6468252607288938, + "scr_dir1_threshold_500": -0.047368439214711934, + "scr_metric_threshold_500": 0.7023808031440972, + "scr_dir2_threshold_500": 0.7023808031440972 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.2896173902280937, "scr_metric_threshold_10": 0.12213741326330622, "scr_dir2_threshold_10": 0.12213741326330622, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3278689218723027, + "scr_metric_threshold_20": 0.17175577294818423, + "scr_dir2_threshold_20": 0.17175577294818423, + "scr_dir1_threshold_50": 0.3825136878996036, + "scr_metric_threshold_50": 0.21755716073528464, + "scr_dir2_threshold_50": 0.21755716073528464, + "scr_dir1_threshold_100": 0.4426228653879351, + "scr_metric_threshold_100": 0.2786259811162493, + "scr_dir2_threshold_100": 0.2786259811162493, + "scr_dir1_threshold_500": 0.5300547515984144, + "scr_metric_threshold_500": 0.26335877602100816, + "scr_dir2_threshold_500": 0.26335877602100816 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.18139520312982554, "scr_metric_threshold_10": 0.3776824330771527, "scr_dir2_threshold_10": 0.3776824330771527, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.23255815887657183, + "scr_metric_threshold_20": 0.47210310529992777, + "scr_dir2_threshold_20": 0.47210310529992777, + "scr_dir1_threshold_50": 0.26976753083223043, + "scr_metric_threshold_50": 0.48068668649565843, + "scr_dir2_threshold_50": 0.48068668649565843, + "scr_dir1_threshold_100": 0.20465126852525933, + "scr_metric_threshold_100": 0.48497847709352376, + "scr_dir2_threshold_100": 0.48497847709352376, + "scr_dir1_threshold_500": 0.24186036325005503, + "scr_metric_threshold_500": 0.40772547889010513, + "scr_dir2_threshold_500": 0.40772547889010513 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.1720930464658461, "scr_metric_threshold_10": 0.1720930464658461, "scr_dir2_threshold_10": 0.1221719823127949, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.17674414994201715, + "scr_metric_threshold_20": 0.17674414994201715, + "scr_dir2_threshold_20": 0.14479639351469145, + "scr_dir1_threshold_50": 0.2651162249130257, + "scr_metric_threshold_50": 0.2651162249130257, + "scr_dir2_threshold_50": 0.19909487251754088, + "scr_dir1_threshold_100": 0.2883720195248206, + "scr_metric_threshold_100": 0.2883720195248206, + "scr_dir2_threshold_100": 0.19909487251754088, + "scr_dir1_threshold_500": 0.3534882998840343, + "scr_metric_threshold_500": 0.3534882998840343, + "scr_dir2_threshold_500": 0.23529403832227772 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-170898-tokens-700000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-170898-tokens-700000000/scr/saebench_scr_custom_sae_eval_results.json index 9191f2a346cc4b4c8880eb6182096fb68df2306c..e4fc17c0c69beb53bd3a2898c63b9a7f7c06476f 100644 --- a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-170898-tokens-700000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-170898-tokens-700000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "5d66de21-b065-4b14-9484-7869fcd1a90b", - "datetime_epoch_millis": 1776888454499, + "eval_id": "cab18dce-1750-4ead-a182-b721231bb3ae", + "datetime_epoch_millis": 1777319248590, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.2132455396307936, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.09799222102907341, "scr_metric_threshold_10": 0.154227161519067, "scr_dir2_threshold_10": 0.15383254688788053, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.04419470865508533, + "scr_metric_threshold_20": 0.1727258273620318, + "scr_dir2_threshold_20": 0.17338353059252204, + "scr_dir1_threshold_50": 0.0776959875515677, + "scr_metric_threshold_50": 0.22899071545179572, + "scr_dir2_threshold_50": 0.2323501930242092, + "scr_dir1_threshold_100": -0.23128291944532411, + "scr_metric_threshold_100": 0.2897144847232666, + "scr_dir2_threshold_100": 0.28545271131001027, + "scr_dir1_threshold_500": -1.0169561817110646, + "scr_metric_threshold_500": 0.10361698093196486, + "scr_dir2_threshold_500": 0.12439991243587573 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": -0.08974318806867983, "scr_metric_threshold_10": 0.0492227771252837, "scr_dir2_threshold_10": 0.0492227771252837, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.4615385791018498, + "scr_metric_threshold_20": 0.12435226279901532, + "scr_dir2_threshold_20": 0.12435226279901532, + "scr_dir1_threshold_50": -0.06410249552392094, + "scr_metric_threshold_50": 0.1968911085011348, + "scr_dir2_threshold_50": 0.1968911085011348, + "scr_dir1_threshold_100": -1.0128203462723795, + "scr_metric_threshold_100": 0.2823834628936415, + "scr_dir2_threshold_100": 0.2823834628936415, + "scr_dir1_threshold_500": -2.730768907469913, + "scr_metric_threshold_500": 0.046632137153671566, + "scr_dir2_threshold_500": 0.046632137153671566 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": 0.42424228740901826, "scr_metric_threshold_10": 0.021406815344093763, "scr_dir2_threshold_10": 0.021406815344093763, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.4924241158533419, + "scr_metric_threshold_20": -0.012232439871312305, + "scr_dir2_threshold_20": -0.012232439871312305, + "scr_dir1_threshold_50": -0.2954542888919093, + "scr_metric_threshold_50": 0.25688069046599166, + "scr_dir2_threshold_50": 0.25688069046599166, + "scr_dir1_threshold_100": -0.7954540631167895, + "scr_metric_threshold_100": 0.5137615632091722, + "scr_dir2_threshold_100": 0.5137615632091722, + "scr_dir1_threshold_500": -0.8106058314101056, + "scr_metric_threshold_500": -0.13455647403005752, + "scr_dir2_threshold_500": -0.13455647403005752 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.45312467985791743, "scr_metric_threshold_10": 0.1532662816127337, "scr_dir2_threshold_10": 0.1532662816127337, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -1.343750174622954, + "scr_metric_threshold_20": 0.16834176159853043, + "scr_dir2_threshold_20": 0.16834176159853043, + "scr_dir1_threshold_50": 0.31249965075409175, + "scr_metric_threshold_50": 0.2386935023309456, + "scr_dir2_threshold_50": 0.2386935023309456, + "scr_dir1_threshold_100": 0.3593749708961743, + "scr_metric_threshold_100": 0.04522614043660845, + "scr_dir2_threshold_100": 0.04522614043660845, + "scr_dir1_threshold_500": -3.9374998835846973, + "scr_metric_threshold_500": -0.1582913417677387, + "scr_dir2_threshold_500": -0.1582913417677387 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": -0.6711411490096227, "scr_metric_threshold_10": 0.10828037568388735, "scr_dir2_threshold_10": 0.10828037568388735, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.22818779864829455, + "scr_metric_threshold_20": -0.10509551359270591, + "scr_dir2_threshold_20": -0.10509551359270591, + "scr_dir1_threshold_50": -0.1812081342344703, + "scr_metric_threshold_50": -0.2707005595623121, + "scr_dir2_threshold_50": -0.2707005595623121, + "scr_dir1_threshold_100": -1.2147655517130336, + "scr_metric_threshold_100": -0.28343943845592845, + "scr_dir2_threshold_100": -0.28343943845592845, + "scr_dir1_threshold_500": -1.5906044671484376, + "scr_metric_threshold_500": -0.5382164468571459, + "scr_dir2_threshold_500": -0.5382164468571459 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.06842111372221286, "scr_metric_threshold_10": 0.2936507579841239, "scr_dir2_threshold_10": 0.2936507579841239, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.08421069803000372, + "scr_metric_threshold_20": 0.4087300451599734, + "scr_dir2_threshold_20": 0.4087300451599734, + "scr_dir1_threshold_50": -0.02105267450750093, + "scr_metric_threshold_50": 0.5833333136228053, + "scr_dir2_threshold_50": 0.5833333136228053, + "scr_dir1_threshold_100": -0.047368439214711934, + "scr_metric_threshold_100": 0.6349206536925663, + "scr_dir2_threshold_100": 0.6349206536925663, + "scr_dir1_threshold_500": -0.03157885490692107, + "scr_metric_threshold_500": 0.654761901946115, + "scr_dir2_threshold_500": 0.654761901946115 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.3060109503196829, "scr_metric_threshold_10": 0.1183206688641517, "scr_dir2_threshold_10": 0.1183206688641517, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.3551913048859532, + "scr_metric_threshold_20": 0.19465646684173443, + "scr_dir2_threshold_20": 0.19465646684173443, + "scr_dir1_threshold_50": 0.3825136878996036, + "scr_metric_threshold_50": 0.22519087703221677, + "scr_dir2_threshold_50": 0.22519087703221677, + "scr_dir1_threshold_100": 0.377049276438573, + "scr_metric_threshold_100": 0.39694664998040097, + "scr_dir2_threshold_100": 0.39694664998040097, + "scr_dir1_threshold_500": 0.4863388084931748, + "scr_metric_threshold_500": 0.29007644181233594, + "scr_dir2_threshold_500": 0.29007644181233594 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.17674410094308396, "scr_metric_threshold_10": 0.3733906424792874, "scr_dir2_threshold_10": 0.3733906424792874, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.21395347289874253, + "scr_metric_threshold_20": 0.4635192682902496, + "scr_dir2_threshold_20": 0.4635192682902496, + "scr_dir1_threshold_50": 0.3116280049746306, + "scr_metric_threshold_50": 0.4248926412815665, + "scr_dir2_threshold_50": 0.4248926412815665, + "scr_dir1_threshold_100": 0.22790705668983022, + "scr_metric_threshold_100": 0.47210310529992777, + "scr_dir2_threshold_100": 0.47210310529992777, + "scr_dir1_threshold_500": 0.26976753083223043, + "scr_metric_threshold_500": 0.45922747769238426, + "scr_dir2_threshold_500": 0.45922747769238426 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.1162789730589745, "scr_metric_threshold_10": 0.1162789730589745, "scr_dir2_threshold_10": 0.1131220560094828, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.13953476767076942, + "scr_metric_threshold_20": 0.13953476767076942, + "scr_dir2_threshold_20": 0.14479639351469145, + "scr_dir1_threshold_50": 0.17674414994201715, + "scr_metric_threshold_50": 0.17674414994201715, + "scr_dir2_threshold_50": 0.20361997052132483, + "scr_dir1_threshold_100": 0.2558137407297439, + "scr_metric_threshold_100": 0.2558137407297439, + "scr_dir2_threshold_100": 0.22171955342369326, + "scr_dir1_threshold_500": 0.2093021515061541, + "scr_metric_threshold_500": 0.2093021515061541, + "scr_dir2_threshold_500": 0.375565603537441 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-170898-tokens-700000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-170898-tokens-700000000/tpp/saebench_tpp_custom_sae_eval_results.json index d7f26932a960859461accc7ffd11e0a484714419..0848fce803798f7ac963aaed29d7a9ccdc0ae957 100644 --- a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-170898-tokens-700000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-170898-tokens-700000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "6524cd7a-bad1-4721-8705-b9da7cd4f7db", - "datetime_epoch_millis": 1776888619073, + "eval_id": "170a30f7-13c7-430f-8c6f-b7591046e97d", + "datetime_epoch_millis": 1777319523877, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.055700007081031795, - "tpp_threshold_2_intended_diff_only": 0.06950001120567323, - "tpp_threshold_2_unintended_diff_only": 0.013800004124641418, - "tpp_threshold_5_total_metric": 0.10250000357627868, - "tpp_threshold_5_intended_diff_only": 0.1518000066280365, - "tpp_threshold_5_unintended_diff_only": 0.04930000305175781, - "tpp_threshold_10_total_metric": 0.10270001441240312, - "tpp_threshold_10_intended_diff_only": 0.18760002255439756, - "tpp_threshold_10_unintended_diff_only": 0.08490000814199447, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.053924986720085145, + "tpp_threshold_2_intended_diff_only": 0.06899999380111695, + "tpp_threshold_2_unintended_diff_only": 0.015075007081031799, + "tpp_threshold_5_total_metric": 0.10092500448226928, + "tpp_threshold_5_intended_diff_only": 0.15000001192092896, + "tpp_threshold_5_unintended_diff_only": 0.04907500743865967, + "tpp_threshold_10_total_metric": 0.10322501808404921, + "tpp_threshold_10_intended_diff_only": 0.18880002498626708, + "tpp_threshold_10_unintended_diff_only": 0.08557500690221786, + "tpp_threshold_20_total_metric": 0.10632500499486923, + "tpp_threshold_20_intended_diff_only": 0.21430001258850095, + "tpp_threshold_20_unintended_diff_only": 0.10797500759363175, + "tpp_threshold_50_total_metric": 0.10922500491142273, + "tpp_threshold_50_intended_diff_only": 0.2539000153541565, + "tpp_threshold_50_unintended_diff_only": 0.14467501044273376, + "tpp_threshold_100_total_metric": 0.1161000058054924, + "tpp_threshold_100_intended_diff_only": 0.2896000146865845, + "tpp_threshold_100_unintended_diff_only": 0.17350000888109207, + "tpp_threshold_500_total_metric": 0.2028500124812126, + "tpp_threshold_500_intended_diff_only": 0.4094000279903412, + "tpp_threshold_500_unintended_diff_only": 0.20655001550912858 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.10300001204013824, - "tpp_threshold_2_intended_diff_only": 0.13080002069473268, - "tpp_threshold_2_unintended_diff_only": 0.02780000865459442, - "tpp_threshold_5_total_metric": 0.19120000898838044, - "tpp_threshold_5_intended_diff_only": 0.2882000207901001, - "tpp_threshold_5_unintended_diff_only": 0.09700001180171966, - "tpp_threshold_10_total_metric": 0.18715002834796907, - "tpp_threshold_10_intended_diff_only": 0.3546000480651855, - "tpp_threshold_10_unintended_diff_only": 0.1674500197172165, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.10124999284744263, + "tpp_threshold_2_intended_diff_only": 0.1308000087738037, + "tpp_threshold_2_unintended_diff_only": 0.029550015926361084, + "tpp_threshold_5_total_metric": 0.1908500075340271, + "tpp_threshold_5_intended_diff_only": 0.2888000249862671, + "tpp_threshold_5_unintended_diff_only": 0.09795001745224, + "tpp_threshold_10_total_metric": 0.1860500305891037, + "tpp_threshold_10_intended_diff_only": 0.3548000454902649, + "tpp_threshold_10_unintended_diff_only": 0.1687500149011612, + "tpp_threshold_20_total_metric": 0.1720000058412552, + "tpp_threshold_20_intended_diff_only": 0.3850000262260437, + "tpp_threshold_20_unintended_diff_only": 0.21300002038478852, + "tpp_threshold_50_total_metric": 0.13210000991821289, + "tpp_threshold_50_intended_diff_only": 0.4160000324249268, + "tpp_threshold_50_unintended_diff_only": 0.28390002250671387, + "tpp_threshold_100_total_metric": 0.0953499972820282, + "tpp_threshold_100_intended_diff_only": 0.431600022315979, + "tpp_threshold_100_unintended_diff_only": 0.3362500250339508, + "tpp_threshold_500_total_metric": 0.07550001740455628, + "tpp_threshold_500_intended_diff_only": 0.46400004625320435, + "tpp_threshold_500_unintended_diff_only": 0.3885000288486481 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.008400002121925354, - "tpp_threshold_2_intended_diff_only": 0.00820000171661377, - "tpp_threshold_2_unintended_diff_only": -0.00020000040531158446, - "tpp_threshold_5_total_metric": 0.01379999816417694, - "tpp_threshold_5_intended_diff_only": 0.0153999924659729, - "tpp_threshold_5_unintended_diff_only": 0.0015999943017959595, - "tpp_threshold_10_total_metric": 0.01825000047683716, - "tpp_threshold_10_intended_diff_only": 0.020599997043609618, - "tpp_threshold_10_unintended_diff_only": 0.002349996566772461, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.006599980592727661, + "tpp_threshold_2_intended_diff_only": 0.007199978828430176, + "tpp_threshold_2_unintended_diff_only": 0.0005999982357025146, + "tpp_threshold_5_total_metric": 0.011000001430511474, + "tpp_threshold_5_intended_diff_only": 0.01119999885559082, + "tpp_threshold_5_unintended_diff_only": 0.0001999974250793457, + "tpp_threshold_10_total_metric": 0.02040000557899475, + "tpp_threshold_10_intended_diff_only": 0.022800004482269286, + "tpp_threshold_10_unintended_diff_only": 0.0023999989032745363, + "tpp_threshold_20_total_metric": 0.04065000414848328, + "tpp_threshold_20_intended_diff_only": 0.04359999895095825, + "tpp_threshold_20_unintended_diff_only": 0.0029499948024749754, + "tpp_threshold_50_total_metric": 0.08634999990463257, + "tpp_threshold_50_intended_diff_only": 0.09179999828338622, + "tpp_threshold_50_unintended_diff_only": 0.005449998378753662, + "tpp_threshold_100_total_metric": 0.1368500143289566, + "tpp_threshold_100_intended_diff_only": 0.14760000705718995, + "tpp_threshold_100_unintended_diff_only": 0.010749992728233338, + "tpp_threshold_500_total_metric": 0.33020000755786894, + "tpp_threshold_500_intended_diff_only": 0.35480000972747805, + "tpp_threshold_500_unintended_diff_only": 0.02460000216960907 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.05225001275539398, "tpp_threshold_10_total_metric": 0.20625001192092896, "tpp_threshold_10_intended_diff_only": 0.359000027179718, - "tpp_threshold_10_unintended_diff_only": 0.15275001525878906 + "tpp_threshold_10_unintended_diff_only": 0.15275001525878906, + "tpp_threshold_20_total_metric": 0.16849994659423828, + "tpp_threshold_20_intended_diff_only": 0.4039999842643738, + "tpp_threshold_20_unintended_diff_only": 0.2355000376701355, + "tpp_threshold_50_total_metric": 0.1484999805688858, + "tpp_threshold_50_intended_diff_only": 0.4300000071525574, + "tpp_threshold_50_unintended_diff_only": 0.28150002658367157, + "tpp_threshold_100_total_metric": 0.10124996304512024, + "tpp_threshold_100_intended_diff_only": 0.4359999895095825, + "tpp_threshold_100_unintended_diff_only": 0.3347500264644623, + "tpp_threshold_500_total_metric": 0.06949999928474426, + "tpp_threshold_500_intended_diff_only": 0.45600003004074097, + "tpp_threshold_500_unintended_diff_only": 0.3865000307559967 }, "1": { - "tpp_threshold_2_total_metric": 0.16175000369548798, - "tpp_threshold_2_intended_diff_only": 0.24800002574920654, - "tpp_threshold_2_unintended_diff_only": 0.08625002205371857, - "tpp_threshold_5_total_metric": 0.1574999988079071, - "tpp_threshold_5_intended_diff_only": 0.29500001668930054, - "tpp_threshold_5_unintended_diff_only": 0.13750001788139343, - "tpp_threshold_10_total_metric": 0.16550002992153168, + "tpp_threshold_2_total_metric": 0.1589999794960022, + "tpp_threshold_2_intended_diff_only": 0.24500000476837158, + "tpp_threshold_2_unintended_diff_only": 0.08600002527236938, + "tpp_threshold_5_total_metric": 0.15574997663497925, + "tpp_threshold_5_intended_diff_only": 0.29600000381469727, + "tpp_threshold_5_unintended_diff_only": 0.14025002717971802, + "tpp_threshold_10_total_metric": 0.15650001168251038, "tpp_threshold_10_intended_diff_only": 0.3160000443458557, - "tpp_threshold_10_unintended_diff_only": 0.15050001442432404 + "tpp_threshold_10_unintended_diff_only": 0.15950003266334534, + "tpp_threshold_20_total_metric": 0.17249999940395355, + "tpp_threshold_20_intended_diff_only": 0.34200000762939453, + "tpp_threshold_20_unintended_diff_only": 0.16950000822544098, + "tpp_threshold_50_total_metric": 0.10774999856948853, + "tpp_threshold_50_intended_diff_only": 0.37400001287460327, + "tpp_threshold_50_unintended_diff_only": 0.26625001430511475, + "tpp_threshold_100_total_metric": 0.0807499885559082, + "tpp_threshold_100_intended_diff_only": 0.4020000100135803, + "tpp_threshold_100_unintended_diff_only": 0.3212500214576721, + "tpp_threshold_500_total_metric": 0.05849999189376831, + "tpp_threshold_500_intended_diff_only": 0.4520000219345093, + "tpp_threshold_500_unintended_diff_only": 0.39350003004074097 }, "2": { - "tpp_threshold_2_total_metric": 0.1572500616312027, + "tpp_threshold_2_total_metric": 0.14975005388259888, "tpp_threshold_2_intended_diff_only": 0.17600005865097046, - "tpp_threshold_2_unintended_diff_only": 0.01874999701976776, - "tpp_threshold_5_total_metric": 0.2212500274181366, + "tpp_threshold_2_unintended_diff_only": 0.026250004768371582, + "tpp_threshold_5_total_metric": 0.2225000113248825, "tpp_threshold_5_intended_diff_only": 0.284000039100647, - "tpp_threshold_5_unintended_diff_only": 0.06275001168251038, - "tpp_threshold_10_total_metric": 0.14375002682209015, - "tpp_threshold_10_intended_diff_only": 0.34800004959106445, - "tpp_threshold_10_unintended_diff_only": 0.2042500227689743 + "tpp_threshold_5_unintended_diff_only": 0.061500027775764465, + "tpp_threshold_10_total_metric": 0.14350003004074097, + "tpp_threshold_10_intended_diff_only": 0.3490000367164612, + "tpp_threshold_10_unintended_diff_only": 0.20550000667572021, + "tpp_threshold_20_total_metric": 0.09400004148483276, + "tpp_threshold_20_intended_diff_only": 0.36100006103515625, + "tpp_threshold_20_unintended_diff_only": 0.2670000195503235, + "tpp_threshold_50_total_metric": 0.06950002908706665, + "tpp_threshold_50_intended_diff_only": 0.3850000500679016, + "tpp_threshold_50_unintended_diff_only": 0.31550002098083496, + "tpp_threshold_100_total_metric": 0.038750022649765015, + "tpp_threshold_100_intended_diff_only": 0.39500004053115845, + "tpp_threshold_100_unintended_diff_only": 0.35625001788139343, + "tpp_threshold_500_total_metric": 0.046750038862228394, + "tpp_threshold_500_intended_diff_only": 0.4490000605583191, + "tpp_threshold_500_unintended_diff_only": 0.4022500216960907 }, "6": { - "tpp_threshold_2_total_metric": 0.007750004529953003, + "tpp_threshold_2_total_metric": 0.005499973893165588, "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": -0.002750009298324585, - "tpp_threshold_5_total_metric": 0.226500004529953, - "tpp_threshold_5_intended_diff_only": 0.34200000762939453, - "tpp_threshold_5_unintended_diff_only": 0.11550000309944153, - "tpp_threshold_10_total_metric": 0.2680000364780426, + "tpp_threshold_2_unintended_diff_only": -0.0004999786615371704, + "tpp_threshold_5_total_metric": 0.2302500456571579, + "tpp_threshold_5_intended_diff_only": 0.34300005435943604, + "tpp_threshold_5_unintended_diff_only": 0.11275000870227814, + "tpp_threshold_10_total_metric": 0.270500048995018, "tpp_threshold_10_intended_diff_only": 0.42600005865097046, - "tpp_threshold_10_unintended_diff_only": 0.15800002217292786 + "tpp_threshold_10_unintended_diff_only": 0.15550000965595245, + "tpp_threshold_20_total_metric": 0.2627499997615814, + "tpp_threshold_20_intended_diff_only": 0.4570000171661377, + "tpp_threshold_20_unintended_diff_only": 0.19425001740455627, + "tpp_threshold_50_total_metric": 0.1914999932050705, + "tpp_threshold_50_intended_diff_only": 0.48000001907348633, + "tpp_threshold_50_unintended_diff_only": 0.28850002586841583, + "tpp_threshold_100_total_metric": 0.14875000715255737, + "tpp_threshold_100_intended_diff_only": 0.4830000400543213, + "tpp_threshold_100_unintended_diff_only": 0.3342500329017639, + "tpp_threshold_500_total_metric": 0.109250009059906, + "tpp_threshold_500_intended_diff_only": 0.487000048160553, + "tpp_threshold_500_unintended_diff_only": 0.377750039100647 }, "9": { - "tpp_threshold_2_total_metric": 0.008250042796134949, - "tpp_threshold_2_intended_diff_only": 0.010000050067901611, - "tpp_threshold_2_unintended_diff_only": 0.0017500072717666626, - "tpp_threshold_5_total_metric": 0.15900000929832458, - "tpp_threshold_5_intended_diff_only": 0.2760000228881836, - "tpp_threshold_5_unintended_diff_only": 0.11700001358985901, - "tpp_threshold_10_total_metric": 0.1522500365972519, + "tpp_threshold_2_total_metric": 0.012000009417533875, + "tpp_threshold_2_intended_diff_only": 0.013000011444091797, + "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224, + "tpp_threshold_5_total_metric": 0.15399999916553497, + "tpp_threshold_5_intended_diff_only": 0.2770000100135803, + "tpp_threshold_5_unintended_diff_only": 0.12300001084804535, + "tpp_threshold_10_total_metric": 0.1535000503063202, "tpp_threshold_10_intended_diff_only": 0.3240000605583191, - "tpp_threshold_10_unintended_diff_only": 0.1717500239610672 + "tpp_threshold_10_unintended_diff_only": 0.1705000102519989, + "tpp_threshold_20_total_metric": 0.16225004196166992, + "tpp_threshold_20_intended_diff_only": 0.36100006103515625, + "tpp_threshold_20_unintended_diff_only": 0.19875001907348633, + "tpp_threshold_50_total_metric": 0.14325004816055298, + "tpp_threshold_50_intended_diff_only": 0.4110000729560852, + "tpp_threshold_50_unintended_diff_only": 0.2677500247955322, + "tpp_threshold_100_total_metric": 0.10725000500679016, + "tpp_threshold_100_intended_diff_only": 0.44200003147125244, + "tpp_threshold_100_unintended_diff_only": 0.3347500264644623, + "tpp_threshold_500_total_metric": 0.0935000479221344, + "tpp_threshold_500_intended_diff_only": 0.4760000705718994, + "tpp_threshold_500_unintended_diff_only": 0.382500022649765 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.010249987244606018, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": -0.0052499920129776, - "tpp_threshold_5_total_metric": 0.01075001060962677, - "tpp_threshold_5_intended_diff_only": 0.008000016212463379, - "tpp_threshold_5_unintended_diff_only": -0.002749994397163391, - "tpp_threshold_10_total_metric": 0.0007500052452087402, - "tpp_threshold_10_intended_diff_only": 0.0040000081062316895, - "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492 + "tpp_threshold_2_total_metric": 0.004499971866607666, + "tpp_threshold_2_intended_diff_only": 0.006999969482421875, + "tpp_threshold_2_unintended_diff_only": 0.002499997615814209, + "tpp_threshold_5_total_metric": 0.0015000253915786743, + "tpp_threshold_5_intended_diff_only": 0.003000020980834961, + "tpp_threshold_5_unintended_diff_only": 0.0014999955892562866, + "tpp_threshold_10_total_metric": 0.0037499964237213135, + "tpp_threshold_10_intended_diff_only": 0.004999995231628418, + "tpp_threshold_10_unintended_diff_only": 0.0012499988079071045, + "tpp_threshold_20_total_metric": 0.017000004649162292, + "tpp_threshold_20_intended_diff_only": 0.018999993801116943, + "tpp_threshold_20_unintended_diff_only": 0.001999989151954651, + "tpp_threshold_50_total_metric": 0.04450000822544098, + "tpp_threshold_50_intended_diff_only": 0.04100000858306885, + "tpp_threshold_50_unintended_diff_only": -0.0034999996423721313, + "tpp_threshold_100_total_metric": 0.07150000333786011, + "tpp_threshold_100_intended_diff_only": 0.07499998807907104, + "tpp_threshold_100_unintended_diff_only": 0.0034999847412109375, + "tpp_threshold_500_total_metric": 0.32200001180171967, + "tpp_threshold_500_intended_diff_only": 0.3360000252723694, + "tpp_threshold_500_unintended_diff_only": 0.01400001347064972 }, "2": { - "tpp_threshold_2_total_metric": 0.006250038743019104, - "tpp_threshold_2_intended_diff_only": 0.01100003719329834, - "tpp_threshold_2_unintended_diff_only": 0.004749998450279236, - "tpp_threshold_5_total_metric": 0.022000029683113098, - "tpp_threshold_5_intended_diff_only": 0.027000010013580322, - "tpp_threshold_5_unintended_diff_only": 0.004999980330467224, - "tpp_threshold_10_total_metric": 0.02775004506111145, - "tpp_threshold_10_intended_diff_only": 0.030000030994415283, - "tpp_threshold_10_unintended_diff_only": 0.002249985933303833 + "tpp_threshold_2_total_metric": 0.017250046133995056, + "tpp_threshold_2_intended_diff_only": 0.01500004529953003, + "tpp_threshold_2_unintended_diff_only": -0.002250000834465027, + "tpp_threshold_5_total_metric": 0.015250027179718018, + "tpp_threshold_5_intended_diff_only": 0.022000014781951904, + "tpp_threshold_5_unintended_diff_only": 0.006749987602233887, + "tpp_threshold_10_total_metric": 0.030750036239624023, + "tpp_threshold_10_intended_diff_only": 0.03400003910064697, + "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492, + "tpp_threshold_20_total_metric": 0.06125004589557648, + "tpp_threshold_20_intended_diff_only": 0.06300002336502075, + "tpp_threshold_20_unintended_diff_only": 0.001749977469444275, + "tpp_threshold_50_total_metric": 0.09550000727176666, + "tpp_threshold_50_intended_diff_only": 0.10600000619888306, + "tpp_threshold_50_unintended_diff_only": 0.010499998927116394, + "tpp_threshold_100_total_metric": 0.13100004196166992, + "tpp_threshold_100_intended_diff_only": 0.14600002765655518, + "tpp_threshold_100_unintended_diff_only": 0.014999985694885254, + "tpp_threshold_500_total_metric": 0.30900001525878906, + "tpp_threshold_500_intended_diff_only": 0.3320000171661377, + "tpp_threshold_500_unintended_diff_only": 0.023000001907348633 }, "3": { - "tpp_threshold_2_total_metric": 0.011749997735023499, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": -0.006750002503395081, - "tpp_threshold_5_total_metric": 0.016000017523765564, - "tpp_threshold_5_intended_diff_only": 0.013999998569488525, - "tpp_threshold_5_unintended_diff_only": -0.0020000189542770386, - "tpp_threshold_10_total_metric": 0.015750020742416382, - "tpp_threshold_10_intended_diff_only": 0.017000019550323486, - "tpp_threshold_10_unintended_diff_only": 0.0012499988079071045 + "tpp_threshold_2_total_metric": 0.0029999762773513794, + "tpp_threshold_2_intended_diff_only": 0.001999974250793457, + "tpp_threshold_2_unintended_diff_only": -0.0010000020265579224, + "tpp_threshold_5_total_metric": 0.00875002145767212, + "tpp_threshold_5_intended_diff_only": 0.003000020980834961, + "tpp_threshold_5_unintended_diff_only": -0.005750000476837158, + "tpp_threshold_10_total_metric": 0.015750005841255188, + "tpp_threshold_10_intended_diff_only": 0.013999998569488525, + "tpp_threshold_10_unintended_diff_only": -0.0017500072717666626, + "tpp_threshold_20_total_metric": 0.020250022411346436, + "tpp_threshold_20_intended_diff_only": 0.022000014781951904, + "tpp_threshold_20_unintended_diff_only": 0.0017499923706054688, + "tpp_threshold_50_total_metric": 0.04749998450279236, + "tpp_threshold_50_intended_diff_only": 0.04799997806549072, + "tpp_threshold_50_unintended_diff_only": 0.0004999935626983643, + "tpp_threshold_100_total_metric": 0.09975001215934753, + "tpp_threshold_100_intended_diff_only": 0.11000001430511475, + "tpp_threshold_100_unintended_diff_only": 0.010250002145767212, + "tpp_threshold_500_total_metric": 0.3147500455379486, + "tpp_threshold_500_intended_diff_only": 0.33900004625320435, + "tpp_threshold_500_unintended_diff_only": 0.024250000715255737 }, "5": { - "tpp_threshold_2_total_metric": -0.003000006079673767, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": 0.008000001311302185, - "tpp_threshold_5_total_metric": 0.00899997353553772, - "tpp_threshold_5_intended_diff_only": 0.010999977588653564, - "tpp_threshold_5_unintended_diff_only": 0.0020000040531158447, - "tpp_threshold_10_total_metric": 0.01474994421005249, - "tpp_threshold_10_intended_diff_only": 0.01799994707107544, - "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492 + "tpp_threshold_2_total_metric": 0.003749951720237732, + "tpp_threshold_2_intended_diff_only": 0.003999948501586914, + "tpp_threshold_2_unintended_diff_only": 0.00024999678134918213, + "tpp_threshold_5_total_metric": 0.017499953508377075, + "tpp_threshold_5_intended_diff_only": 0.01699995994567871, + "tpp_threshold_5_unintended_diff_only": -0.0004999935626983643, + "tpp_threshold_10_total_metric": 0.00974997878074646, + "tpp_threshold_10_intended_diff_only": 0.014999985694885254, + "tpp_threshold_10_unintended_diff_only": 0.005250006914138794, + "tpp_threshold_20_total_metric": 0.03999997675418854, + "tpp_threshold_20_intended_diff_only": 0.041999995708465576, + "tpp_threshold_20_unintended_diff_only": 0.0020000189542770386, + "tpp_threshold_50_total_metric": 0.08525000512599945, + "tpp_threshold_50_intended_diff_only": 0.09700000286102295, + "tpp_threshold_50_unintended_diff_only": 0.011749997735023499, + "tpp_threshold_100_total_metric": 0.15925000607967377, + "tpp_threshold_100_intended_diff_only": 0.171999990940094, + "tpp_threshold_100_unintended_diff_only": 0.012749984860420227, + "tpp_threshold_500_total_metric": 0.3634999692440033, + "tpp_threshold_500_intended_diff_only": 0.3959999680519104, + "tpp_threshold_500_unintended_diff_only": 0.032499998807907104 }, "6": { - "tpp_threshold_2_total_metric": 0.016749992966651917, - "tpp_threshold_2_intended_diff_only": 0.014999985694885254, - "tpp_threshold_2_unintended_diff_only": -0.0017500072717666626, - "tpp_threshold_5_total_metric": 0.011249959468841553, - "tpp_threshold_5_intended_diff_only": 0.01699995994567871, - "tpp_threshold_5_unintended_diff_only": 0.005750000476837158, - "tpp_threshold_10_total_metric": 0.03224998712539673, - "tpp_threshold_10_intended_diff_only": 0.0339999794960022, - "tpp_threshold_10_unintended_diff_only": 0.0017499923706054688 + "tpp_threshold_2_total_metric": 0.004499956965446472, + "tpp_threshold_2_intended_diff_only": 0.007999956607818604, + "tpp_threshold_2_unintended_diff_only": 0.0034999996423721313, + "tpp_threshold_5_total_metric": 0.011999979615211487, + "tpp_threshold_5_intended_diff_only": 0.010999977588653564, + "tpp_threshold_5_unintended_diff_only": -0.0010000020265579224, + "tpp_threshold_10_total_metric": 0.04200001060962677, + "tpp_threshold_10_intended_diff_only": 0.046000003814697266, + "tpp_threshold_10_unintended_diff_only": 0.003999993205070496, + "tpp_threshold_20_total_metric": 0.06474997103214264, + "tpp_threshold_20_intended_diff_only": 0.07199996709823608, + "tpp_threshold_20_unintended_diff_only": 0.007249996066093445, + "tpp_threshold_50_total_metric": 0.1589999943971634, + "tpp_threshold_50_intended_diff_only": 0.16699999570846558, + "tpp_threshold_50_unintended_diff_only": 0.008000001311302185, + "tpp_threshold_100_total_metric": 0.2227500081062317, + "tpp_threshold_100_intended_diff_only": 0.23500001430511475, + "tpp_threshold_100_unintended_diff_only": 0.012250006198883057, + "tpp_threshold_500_total_metric": 0.34174999594688416, + "tpp_threshold_500_intended_diff_only": 0.3709999918937683, + "tpp_threshold_500_unintended_diff_only": 0.029249995946884155 } } } diff --git a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-183105-tokens-750000000/scr/saebench_scr_custom_sae_eval_results.json b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-183105-tokens-750000000/scr/saebench_scr_custom_sae_eval_results.json index b5a06e9c2e2b48a174318f1315464b0b5dbac05a..b3ead72ec26c36f4bce28c12f7b45a21abba2451 100644 --- a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-183105-tokens-750000000/scr/saebench_scr_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-183105-tokens-750000000/scr/saebench_scr_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,8 +69,8 @@ ] } }, - "eval_id": "ab8f6dd9-bde2-4d3c-bc5d-cdf547cf5e20", - "datetime_epoch_millis": 1776889358280, + "eval_id": "9cd8c789-2a5f-4b2b-b776-541d173b2802", + "datetime_epoch_millis": 1777319245861, "eval_result_metrics": { "scr_metrics": { "scr_dir1_threshold_2": 0.24244203802625144, @@ -78,18 +82,18 @@ "scr_dir1_threshold_10": 0.0790288788689461, "scr_metric_threshold_10": 0.15083563357799917, "scr_dir2_threshold_10": 0.14638705314837597, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.03396895102335076, + "scr_metric_threshold_20": 0.1694668784412279, + "scr_dir2_threshold_20": 0.17175825375488427, + "scr_dir1_threshold_50": 0.1695625124582478, + "scr_metric_threshold_50": 0.2844491608100247, + "scr_dir2_threshold_50": 0.2853883446671675, + "scr_dir1_threshold_100": -0.15343394056601214, + "scr_metric_threshold_100": 0.2751571074176913, + "scr_dir2_threshold_100": 0.27370756635372423, + "scr_dir1_threshold_500": -0.5887372981196939, + "scr_metric_threshold_500": 0.12428798400520683, + "scr_dir2_threshold_500": 0.13531345906509823 } }, "eval_result_details": [ @@ -104,18 +108,18 @@ "scr_dir1_threshold_10": 0.16666679402700396, "scr_metric_threshold_10": 0.05181341709689583, "scr_dir2_threshold_10": 0.05181341709689583, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.38461573730554943, + "scr_metric_threshold_20": 0.12176162282740319, + "scr_dir2_threshold_20": 0.12176162282740319, + "scr_dir1_threshold_50": 0.5000003820810118, + "scr_metric_threshold_50": 0.25129531998580606, + "scr_dir2_threshold_50": 0.25129531998580606, + "scr_dir1_threshold_100": 0.39743608357792887, + "scr_metric_threshold_100": 0.33937815993376164, + "scr_dir2_threshold_100": 0.33937815993376164, + "scr_dir1_threshold_500": -2.525640310463747, + "scr_metric_threshold_500": -0.01813478863361152, + "scr_dir2_threshold_500": -0.01813478863361152 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results", @@ -128,18 +132,18 @@ "scr_dir1_threshold_10": -0.37121177570777114, "scr_metric_threshold_10": 0.11620790536168352, "scr_dir2_threshold_10": 0.11620790536168352, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -0.37121177570777114, + "scr_metric_threshold_20": 0.08256883242346637, + "scr_dir2_threshold_20": 0.08256883242346637, + "scr_dir1_threshold_50": 0.4090909706659418, + "scr_metric_threshold_50": 0.5412845073503276, + "scr_dir2_threshold_50": 0.5412845073503276, + "scr_dir1_threshold_100": -0.9772725733351455, + "scr_metric_threshold_100": 0.4678898681224538, + "scr_dir2_threshold_100": 0.4678898681224538, + "scr_dir1_threshold_500": -0.454545372445411, + "scr_metric_threshold_500": 0.13761472070577727, + "scr_dir2_threshold_500": 0.13761472070577727 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results", @@ -152,18 +156,18 @@ "scr_dir1_threshold_10": 0.4687504074535596, "scr_metric_threshold_10": 0.13567842130982535, "scr_dir2_threshold_10": 0.13567842130982535, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": -1.3906245634426146, + "scr_metric_threshold_20": 0.17336682175353543, + "scr_dir2_threshold_20": 0.17336682175353543, + "scr_dir1_threshold_50": -1.01562479627322, + "scr_metric_threshold_50": 0.2437185624859506, + "scr_dir2_threshold_50": 0.2437185624859506, + "scr_dir1_threshold_100": -0.10937450523496334, + "scr_metric_threshold_100": -0.005025060155005, + "scr_dir2_threshold_100": -0.005025060155005, + "scr_dir1_threshold_500": -1.51562479627322, + "scr_metric_threshold_500": -0.16582908176063707, + "scr_dir2_threshold_500": -0.16582908176063707 }, { "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results", @@ -176,18 +180,18 @@ "scr_dir1_threshold_10": -0.42281897988043066, "scr_metric_threshold_10": 0.07961794562917636, "scr_dir2_threshold_10": 0.07961794562917636, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.20134210468416536, + "scr_metric_threshold_20": -0.1337579436474169, + "scr_dir2_threshold_20": -0.1337579436474169, + "scr_dir1_threshold_50": 0.25503349261242375, + "scr_metric_threshold_50": -0.2388534572401228, + "scr_dir2_threshold_50": -0.2388534572401228, + "scr_dir1_threshold_100": -1.6241614845957986, + "scr_metric_threshold_100": -0.25796168066869574, + "scr_dir2_threshold_100": -0.25796168066869574, + "scr_dir1_threshold_500": -1.2348995221627288, + "scr_metric_threshold_500": -0.5127388788936164, + "scr_dir2_threshold_500": -0.5127388788936164 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results", @@ -200,18 +204,18 @@ "scr_dir1_threshold_10": 0.052631529414422, "scr_metric_threshold_10": 0.198412719061823, "scr_dir2_threshold_10": 0.198412719061823, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.057894933322792726, + "scr_metric_threshold_20": 0.35317450274476986, + "scr_dir2_threshold_20": 0.35317450274476986, + "scr_dir1_threshold_50": 0.18421066665913766, + "scr_metric_threshold_50": 0.5833333136228053, + "scr_dir2_threshold_50": 0.5833333136228053, + "scr_dir1_threshold_100": -0.015789270599130212, + "scr_metric_threshold_100": 0.6190476077844602, + "scr_dir2_threshold_100": 0.6190476077844602, + "scr_dir1_threshold_500": -0.01052618039942014, + "scr_metric_threshold_500": 0.6111109665672388, + "scr_dir2_threshold_500": 0.6111109665672388 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results", @@ -224,18 +228,18 @@ "scr_dir1_threshold_10": 0.3661201278080144, "scr_metric_threshold_10": 0.17175577294818423, "scr_dir2_threshold_10": 0.17175577294818423, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.4043716594522235, + "scr_metric_threshold_20": 0.20992367193697561, + "scr_dir2_threshold_20": 0.20992367193697561, + "scr_dir1_threshold_50": 0.4098360709132541, + "scr_metric_threshold_50": 0.25954203162185363, + "scr_dir2_threshold_50": 0.25954203162185363, + "scr_dir1_threshold_100": 0.4644808369405549, + "scr_metric_threshold_100": 0.3396945739985909, + "scr_dir2_threshold_100": 0.3396945739985909, + "scr_dir1_threshold_500": 0.4918032199542054, + "scr_metric_threshold_500": 0.2748092367170948, + "scr_dir2_threshold_500": 0.2748092367170948 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results", @@ -248,18 +252,18 @@ "scr_dir1_threshold_10": 0.22790705668983022, "scr_metric_threshold_10": 0.30901301606946474, "scr_dir2_threshold_10": 0.30901301606946474, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.28372083739245524, + "scr_metric_threshold_20": 0.3905580606846963, + "scr_dir2_threshold_20": 0.3905580606846963, + "scr_dir1_threshold_50": 0.39069757384182646, + "scr_metric_threshold_50": 0.41201726948797046, + "scr_dir2_threshold_50": 0.41201726948797046, + "scr_dir1_threshold_100": 0.3767442672816017, + "scr_metric_threshold_100": 0.43776826888911, + "scr_dir2_threshold_100": 0.43776826888911, + "scr_dir1_threshold_500": 0.2883719395791968, + "scr_metric_threshold_500": 0.4163090600858358, + "scr_dir2_threshold_500": 0.4163090600858358 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results", @@ -272,18 +276,18 @@ "scr_dir1_threshold_10": 0.14418587114694043, "scr_metric_threshold_10": 0.14418587114694043, "scr_dir2_threshold_10": 0.10859722770995466, - "scr_dir1_threshold_20": null, - "scr_metric_threshold_20": null, - "scr_dir2_threshold_20": null, - "scr_dir1_threshold_50": null, - "scr_metric_threshold_50": null, - "scr_dir2_threshold_50": null, - "scr_dir1_threshold_100": null, - "scr_metric_threshold_100": null, - "scr_dir2_threshold_100": null, - "scr_dir1_threshold_500": null, - "scr_metric_threshold_500": null, - "scr_dir2_threshold_500": null + "scr_dir1_threshold_20": 0.15813945880639327, + "scr_metric_threshold_20": 0.15813945880639327, + "scr_dir2_threshold_20": 0.17647046131564434, + "scr_dir1_threshold_50": 0.22325573916560695, + "scr_metric_threshold_50": 0.22325573916560695, + "scr_dir2_threshold_50": 0.23076921002274955, + "scr_dir1_threshold_100": 0.2604651214368547, + "scr_metric_threshold_100": 0.2604651214368547, + "scr_dir2_threshold_100": 0.24886879292511796, + "scr_dir1_threshold_500": 0.2511626372535729, + "scr_metric_threshold_500": 0.2511626372535729, + "scr_dir2_threshold_500": 0.3393664377327042 } ], "sae_bench_commit_hash": "Unknown", diff --git a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-195312-tokens-800000000/tpp/saebench_tpp_custom_sae_eval_results.json b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-195312-tokens-800000000/tpp/saebench_tpp_custom_sae_eval_results.json index 0f1b9a123a808e9db4fa29286a6a371792ac7fde..73fb88e00f982d8b97d4eb5b0c374e3b21aaee74 100644 --- a/gemma-2-2b-btk/k-50/seed-0/snapshots/step-195312-tokens-800000000/tpp/saebench_tpp_custom_sae_eval_results.json +++ b/gemma-2-2b-btk/k-50/seed-0/snapshots/step-195312-tokens-800000000/tpp/saebench_tpp_custom_sae_eval_results.json @@ -24,7 +24,11 @@ "n_values": [ 2, 5, - 10 + 10, + 20, + 50, + 100, + 500 ], "column1_vals_lookup": { "LabHC/bias_in_bios_class_set1": [ @@ -65,81 +69,81 @@ ] } }, - "eval_id": "a17156ac-552c-4450-a35d-fc1961797914", - "datetime_epoch_millis": 1776889041721, + "eval_id": "1973b0f0-e980-4a96-ad8a-dcdaf5f5b815", + "datetime_epoch_millis": 1777319792056, "eval_result_metrics": { "tpp_metrics": { - "tpp_threshold_2_total_metric": 0.057350000739097594, - "tpp_threshold_2_intended_diff_only": 0.07319999933242798, - "tpp_threshold_2_unintended_diff_only": 0.015849998593330385, - "tpp_threshold_5_total_metric": 0.10237500667572022, - "tpp_threshold_5_intended_diff_only": 0.1511000156402588, - "tpp_threshold_5_unintended_diff_only": 0.048725008964538574, - "tpp_threshold_10_total_metric": 0.09637499302625656, - "tpp_threshold_10_intended_diff_only": 0.189000004529953, - "tpp_threshold_10_unintended_diff_only": 0.09262501150369644, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.05519999414682388, + "tpp_threshold_2_intended_diff_only": 0.07239999771118164, + "tpp_threshold_2_unintended_diff_only": 0.017200003564357757, + "tpp_threshold_5_total_metric": 0.10222500562667847, + "tpp_threshold_5_intended_diff_only": 0.15070000886917112, + "tpp_threshold_5_unintended_diff_only": 0.048475003242492674, + "tpp_threshold_10_total_metric": 0.09704999625682831, + "tpp_threshold_10_intended_diff_only": 0.19070000648498533, + "tpp_threshold_10_unintended_diff_only": 0.09365001022815705, + "tpp_threshold_20_total_metric": 0.09917501360177994, + "tpp_threshold_20_intended_diff_only": 0.21280001997947695, + "tpp_threshold_20_unintended_diff_only": 0.11362500637769699, + "tpp_threshold_50_total_metric": 0.10372500270605087, + "tpp_threshold_50_intended_diff_only": 0.2504000186920166, + "tpp_threshold_50_unintended_diff_only": 0.14667501598596572, + "tpp_threshold_100_total_metric": 0.11332501024007798, + "tpp_threshold_100_intended_diff_only": 0.28930002450942993, + "tpp_threshold_100_unintended_diff_only": 0.17597501426935197, + "tpp_threshold_500_total_metric": 0.19905001670122147, + "tpp_threshold_500_intended_diff_only": 0.4091000258922577, + "tpp_threshold_500_unintended_diff_only": 0.21005000919103622 } }, "eval_result_details": [ { "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results", - "tpp_threshold_2_total_metric": 0.10704999566078185, - "tpp_threshold_2_intended_diff_only": 0.13840000629425048, - "tpp_threshold_2_unintended_diff_only": 0.03135001063346863, - "tpp_threshold_5_total_metric": 0.19400001764297486, - "tpp_threshold_5_intended_diff_only": 0.2896000385284424, - "tpp_threshold_5_unintended_diff_only": 0.09560002088546753, - "tpp_threshold_10_total_metric": 0.17704998850822448, - "tpp_threshold_10_intended_diff_only": 0.36000001430511475, - "tpp_threshold_10_unintended_diff_only": 0.18295002579689026, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.10535000562667847, + "tpp_threshold_2_intended_diff_only": 0.13860001564025878, + "tpp_threshold_2_unintended_diff_only": 0.03325001001358032, + "tpp_threshold_5_total_metric": 0.19360001683235167, + "tpp_threshold_5_intended_diff_only": 0.2902000308036804, + "tpp_threshold_5_unintended_diff_only": 0.09660001397132874, + "tpp_threshold_10_total_metric": 0.17570000290870666, + "tpp_threshold_10_intended_diff_only": 0.360200023651123, + "tpp_threshold_10_unintended_diff_only": 0.1845000207424164, + "tpp_threshold_20_total_metric": 0.16235001385211945, + "tpp_threshold_20_intended_diff_only": 0.3862000346183777, + "tpp_threshold_20_unintended_diff_only": 0.22385002076625823, + "tpp_threshold_50_total_metric": 0.1244500070810318, + "tpp_threshold_50_intended_diff_only": 0.41420003175735476, + "tpp_threshold_50_unintended_diff_only": 0.28975002467632294, + "tpp_threshold_100_total_metric": 0.0916000097990036, + "tpp_threshold_100_intended_diff_only": 0.4332000374794006, + "tpp_threshold_100_unintended_diff_only": 0.34160002768039704, + "tpp_threshold_500_total_metric": 0.07295000851154328, + "tpp_threshold_500_intended_diff_only": 0.46420003175735475, + "tpp_threshold_500_unintended_diff_only": 0.3912500232458115 }, { "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results", - "tpp_threshold_2_total_metric": 0.00765000581741333, - "tpp_threshold_2_intended_diff_only": 0.007999992370605469, - "tpp_threshold_2_unintended_diff_only": 0.0003499865531921387, - "tpp_threshold_5_total_metric": 0.010749995708465576, - "tpp_threshold_5_intended_diff_only": 0.012599992752075195, - "tpp_threshold_5_unintended_diff_only": 0.001849997043609619, - "tpp_threshold_10_total_metric": 0.015699997544288635, - "tpp_threshold_10_intended_diff_only": 0.01799999475479126, - "tpp_threshold_10_unintended_diff_only": 0.0022999972105026247, - "tpp_threshold_20_total_metric": null, - "tpp_threshold_20_intended_diff_only": null, - "tpp_threshold_20_unintended_diff_only": null, - "tpp_threshold_50_total_metric": null, - "tpp_threshold_50_intended_diff_only": null, - "tpp_threshold_50_unintended_diff_only": null, - "tpp_threshold_100_total_metric": null, - "tpp_threshold_100_intended_diff_only": null, - "tpp_threshold_100_unintended_diff_only": null, - "tpp_threshold_500_total_metric": null, - "tpp_threshold_500_intended_diff_only": null, - "tpp_threshold_500_unintended_diff_only": null + "tpp_threshold_2_total_metric": 0.005049982666969299, + "tpp_threshold_2_intended_diff_only": 0.006199979782104492, + "tpp_threshold_2_unintended_diff_only": 0.0011499971151351929, + "tpp_threshold_5_total_metric": 0.01084999442100525, + "tpp_threshold_5_intended_diff_only": 0.011199986934661866, + "tpp_threshold_5_unintended_diff_only": 0.0003499925136566162, + "tpp_threshold_10_total_metric": 0.01839998960494995, + "tpp_threshold_10_intended_diff_only": 0.021199989318847656, + "tpp_threshold_10_unintended_diff_only": 0.002799999713897705, + "tpp_threshold_20_total_metric": 0.03600001335144043, + "tpp_threshold_20_intended_diff_only": 0.03940000534057617, + "tpp_threshold_20_unintended_diff_only": 0.0033999919891357423, + "tpp_threshold_50_total_metric": 0.08299999833106994, + "tpp_threshold_50_intended_diff_only": 0.08660000562667847, + "tpp_threshold_50_unintended_diff_only": 0.0036000072956085204, + "tpp_threshold_100_total_metric": 0.13505001068115235, + "tpp_threshold_100_intended_diff_only": 0.14540001153945922, + "tpp_threshold_100_unintended_diff_only": 0.010350000858306885, + "tpp_threshold_500_total_metric": 0.32515002489089967, + "tpp_threshold_500_intended_diff_only": 0.35400002002716063, + "tpp_threshold_500_unintended_diff_only": 0.028849995136260985 } ], "sae_bench_commit_hash": "Unknown", @@ -189,108 +193,228 @@ "tpp_threshold_5_unintended_diff_only": 0.053750038146972656, "tpp_threshold_10_total_metric": 0.19224993884563446, "tpp_threshold_10_intended_diff_only": 0.3399999737739563, - "tpp_threshold_10_unintended_diff_only": 0.14775003492832184 + "tpp_threshold_10_unintended_diff_only": 0.14775003492832184, + "tpp_threshold_20_total_metric": 0.16999997198581696, + "tpp_threshold_20_intended_diff_only": 0.3840000033378601, + "tpp_threshold_20_unintended_diff_only": 0.21400003135204315, + "tpp_threshold_50_total_metric": 0.13500000536441803, + "tpp_threshold_50_intended_diff_only": 0.4190000295639038, + "tpp_threshold_50_unintended_diff_only": 0.2840000241994858, + "tpp_threshold_100_total_metric": 0.10774998366832733, + "tpp_threshold_100_intended_diff_only": 0.43400001525878906, + "tpp_threshold_100_unintended_diff_only": 0.32625003159046173, + "tpp_threshold_500_total_metric": 0.06599995493888855, + "tpp_threshold_500_intended_diff_only": 0.45499998331069946, + "tpp_threshold_500_unintended_diff_only": 0.3890000283718109 }, "1": { - "tpp_threshold_2_total_metric": 0.1667499840259552, - "tpp_threshold_2_intended_diff_only": 0.2680000066757202, - "tpp_threshold_2_unintended_diff_only": 0.10125002264976501, - "tpp_threshold_5_total_metric": 0.1757500171661377, - "tpp_threshold_5_intended_diff_only": 0.284000039100647, - "tpp_threshold_5_unintended_diff_only": 0.10825002193450928, - "tpp_threshold_10_total_metric": 0.17149998247623444, + "tpp_threshold_2_total_metric": 0.16350001096725464, + "tpp_threshold_2_intended_diff_only": 0.26600003242492676, + "tpp_threshold_2_unintended_diff_only": 0.10250002145767212, + "tpp_threshold_5_total_metric": 0.17275002598762512, + "tpp_threshold_5_intended_diff_only": 0.2850000262260437, + "tpp_threshold_5_unintended_diff_only": 0.11225000023841858, + "tpp_threshold_10_total_metric": 0.1627499908208847, "tpp_threshold_10_intended_diff_only": 0.34200000762939453, - "tpp_threshold_10_unintended_diff_only": 0.1705000251531601 + "tpp_threshold_10_unintended_diff_only": 0.17925001680850983, + "tpp_threshold_20_total_metric": 0.17600001394748688, + "tpp_threshold_20_intended_diff_only": 0.3540000319480896, + "tpp_threshold_20_unintended_diff_only": 0.17800001800060272, + "tpp_threshold_50_total_metric": 0.13649997115135193, + "tpp_threshold_50_intended_diff_only": 0.3840000033378601, + "tpp_threshold_50_unintended_diff_only": 0.24750003218650818, + "tpp_threshold_100_total_metric": 0.07650001347064972, + "tpp_threshold_100_intended_diff_only": 0.4140000343322754, + "tpp_threshold_100_unintended_diff_only": 0.33750002086162567, + "tpp_threshold_500_total_metric": 0.057250022888183594, + "tpp_threshold_500_intended_diff_only": 0.4540000557899475, + "tpp_threshold_500_unintended_diff_only": 0.3967500329017639 }, "2": { - "tpp_threshold_2_total_metric": 0.1902499943971634, + "tpp_threshold_2_total_metric": 0.1835000067949295, "tpp_threshold_2_intended_diff_only": 0.21700000762939453, - "tpp_threshold_2_unintended_diff_only": 0.02675001323223114, - "tpp_threshold_5_total_metric": 0.22675003111362457, + "tpp_threshold_2_unintended_diff_only": 0.03350000083446503, + "tpp_threshold_5_total_metric": 0.2290000468492508, "tpp_threshold_5_intended_diff_only": 0.3020000457763672, - "tpp_threshold_5_unintended_diff_only": 0.07525001466274261, - "tpp_threshold_10_total_metric": 0.132999986410141, - "tpp_threshold_10_intended_diff_only": 0.36000001430511475, - "tpp_threshold_10_unintended_diff_only": 0.22700002789497375 + "tpp_threshold_5_unintended_diff_only": 0.0729999989271164, + "tpp_threshold_10_total_metric": 0.13025003671646118, + "tpp_threshold_10_intended_diff_only": 0.36100006103515625, + "tpp_threshold_10_unintended_diff_only": 0.23075002431869507, + "tpp_threshold_20_total_metric": 0.10350003838539124, + "tpp_threshold_20_intended_diff_only": 0.3750000596046448, + "tpp_threshold_20_unintended_diff_only": 0.27150002121925354, + "tpp_threshold_50_total_metric": 0.057500019669532776, + "tpp_threshold_50_intended_diff_only": 0.39000004529953003, + "tpp_threshold_50_unintended_diff_only": 0.33250002562999725, + "tpp_threshold_100_total_metric": 0.04175001382827759, + "tpp_threshold_100_intended_diff_only": 0.40400004386901855, + "tpp_threshold_100_unintended_diff_only": 0.36225003004074097, + "tpp_threshold_500_total_metric": 0.04700002074241638, + "tpp_threshold_500_intended_diff_only": 0.45100003480911255, + "tpp_threshold_500_unintended_diff_only": 0.40400001406669617 }, "6": { - "tpp_threshold_2_total_metric": 0.008000001311302185, + "tpp_threshold_2_total_metric": 0.0059999823570251465, "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": -0.003000006079673767, - "tpp_threshold_5_total_metric": 0.21700002253055573, - "tpp_threshold_5_intended_diff_only": 0.3340000510215759, - "tpp_threshold_5_unintended_diff_only": 0.1170000284910202, - "tpp_threshold_10_total_metric": 0.2462499886751175, + "tpp_threshold_2_unintended_diff_only": -0.0009999871253967285, + "tpp_threshold_5_total_metric": 0.2187500149011612, + "tpp_threshold_5_intended_diff_only": 0.33500003814697266, + "tpp_threshold_5_unintended_diff_only": 0.11625002324581146, + "tpp_threshold_10_total_metric": 0.2502499967813492, "tpp_threshold_10_intended_diff_only": 0.43400001525878906, - "tpp_threshold_10_unintended_diff_only": 0.18775002658367157 + "tpp_threshold_10_unintended_diff_only": 0.18375001847743988, + "tpp_threshold_20_total_metric": 0.23325000703334808, + "tpp_threshold_20_intended_diff_only": 0.45600003004074097, + "tpp_threshold_20_unintended_diff_only": 0.22275002300739288, + "tpp_threshold_50_total_metric": 0.18125002086162567, + "tpp_threshold_50_intended_diff_only": 0.4790000319480896, + "tpp_threshold_50_unintended_diff_only": 0.29775001108646393, + "tpp_threshold_100_total_metric": 0.14750003814697266, + "tpp_threshold_100_intended_diff_only": 0.48200005292892456, + "tpp_threshold_100_unintended_diff_only": 0.3345000147819519, + "tpp_threshold_500_total_metric": 0.10600003600120544, + "tpp_threshold_500_intended_diff_only": 0.48600006103515625, + "tpp_threshold_500_unintended_diff_only": 0.3800000250339508 }, "9": { - "tpp_threshold_2_total_metric": 0.005750030279159546, - "tpp_threshold_2_intended_diff_only": 0.00700002908706665, - "tpp_threshold_2_unintended_diff_only": 0.0012499988079071045, - "tpp_threshold_5_total_metric": 0.15825006365776062, - "tpp_threshold_5_intended_diff_only": 0.2820000648498535, - "tpp_threshold_5_unintended_diff_only": 0.1237500011920929, - "tpp_threshold_10_total_metric": 0.14225004613399506, + "tpp_threshold_2_total_metric": 0.009250059723854065, + "tpp_threshold_2_intended_diff_only": 0.010000050067901611, + "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464, + "tpp_threshold_5_total_metric": 0.15525004267692566, + "tpp_threshold_5_intended_diff_only": 0.28300005197525024, + "tpp_threshold_5_unintended_diff_only": 0.12775000929832458, + "tpp_threshold_10_total_metric": 0.1430000513792038, "tpp_threshold_10_intended_diff_only": 0.3240000605583191, - "tpp_threshold_10_unintended_diff_only": 0.18175001442432404 + "tpp_threshold_10_unintended_diff_only": 0.1810000091791153, + "tpp_threshold_20_total_metric": 0.12900003790855408, + "tpp_threshold_20_intended_diff_only": 0.362000048160553, + "tpp_threshold_20_unintended_diff_only": 0.2330000102519989, + "tpp_threshold_50_total_metric": 0.11200001835823059, + "tpp_threshold_50_intended_diff_only": 0.39900004863739014, + "tpp_threshold_50_unintended_diff_only": 0.28700003027915955, + "tpp_threshold_100_total_metric": 0.08449999988079071, + "tpp_threshold_100_intended_diff_only": 0.4320000410079956, + "tpp_threshold_100_unintended_diff_only": 0.3475000411272049, + "tpp_threshold_500_total_metric": 0.0885000079870224, + "tpp_threshold_500_intended_diff_only": 0.4750000238418579, + "tpp_threshold_500_unintended_diff_only": 0.3865000158548355 } }, "canrager/amazon_reviews_mcauley_1and5": { "1": { - "tpp_threshold_2_total_metric": 0.01124998927116394, - "tpp_threshold_2_intended_diff_only": 0.0059999823570251465, - "tpp_threshold_2_unintended_diff_only": -0.005250006914138794, - "tpp_threshold_5_total_metric": 0.012499988079071045, - "tpp_threshold_5_intended_diff_only": 0.009999990463256836, - "tpp_threshold_5_unintended_diff_only": -0.002499997615814209, - "tpp_threshold_10_total_metric": -0.0012500286102294922, - "tpp_threshold_10_intended_diff_only": 0.001999974250793457, - "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492 + "tpp_threshold_2_total_metric": 0.004249975085258484, + "tpp_threshold_2_intended_diff_only": 0.006999969482421875, + "tpp_threshold_2_unintended_diff_only": 0.002749994397163391, + "tpp_threshold_5_total_metric": 0.005499973893165588, + "tpp_threshold_5_intended_diff_only": 0.006999969482421875, + "tpp_threshold_5_unintended_diff_only": 0.0014999955892562866, + "tpp_threshold_10_total_metric": 0.0032500028610229492, + "tpp_threshold_10_intended_diff_only": 0.004999995231628418, + "tpp_threshold_10_unintended_diff_only": 0.0017499923706054688, + "tpp_threshold_20_total_metric": 0.008999988436698914, + "tpp_threshold_20_intended_diff_only": 0.010999977588653564, + "tpp_threshold_20_unintended_diff_only": 0.001999989151954651, + "tpp_threshold_50_total_metric": 0.03299999237060547, + "tpp_threshold_50_intended_diff_only": 0.02799999713897705, + "tpp_threshold_50_unintended_diff_only": -0.004999995231628418, + "tpp_threshold_100_total_metric": 0.0559999942779541, + "tpp_threshold_100_intended_diff_only": 0.06099998950958252, + "tpp_threshold_100_unintended_diff_only": 0.004999995231628418, + "tpp_threshold_500_total_metric": 0.2915000319480896, + "tpp_threshold_500_intended_diff_only": 0.312000036239624, + "tpp_threshold_500_unintended_diff_only": 0.020500004291534424 }, "2": { - "tpp_threshold_2_total_metric": -0.004499971866607666, - "tpp_threshold_2_intended_diff_only": 0.003000020980834961, - "tpp_threshold_2_unintended_diff_only": 0.007499992847442627, - "tpp_threshold_5_total_metric": 0.0022500306367874146, - "tpp_threshold_5_intended_diff_only": 0.008000016212463379, - "tpp_threshold_5_unintended_diff_only": 0.005749985575675964, - "tpp_threshold_10_total_metric": 0.01975002884864807, - "tpp_threshold_10_intended_diff_only": 0.022000014781951904, - "tpp_threshold_10_unintended_diff_only": 0.002249985933303833 + "tpp_threshold_2_total_metric": 0.005000010132789612, + "tpp_threshold_2_intended_diff_only": 0.0040000081062316895, + "tpp_threshold_2_unintended_diff_only": -0.0010000020265579224, + "tpp_threshold_5_total_metric": 0.0020000189542770386, + "tpp_threshold_5_intended_diff_only": 0.009000003337860107, + "tpp_threshold_5_unintended_diff_only": 0.006999984383583069, + "tpp_threshold_10_total_metric": 0.023000016808509827, + "tpp_threshold_10_intended_diff_only": 0.027000010013580322, + "tpp_threshold_10_unintended_diff_only": 0.003999993205070496, + "tpp_threshold_20_total_metric": 0.038750067353248596, + "tpp_threshold_20_intended_diff_only": 0.04200005531311035, + "tpp_threshold_20_unintended_diff_only": 0.0032499879598617554, + "tpp_threshold_50_total_metric": 0.1040000468492508, + "tpp_threshold_50_intended_diff_only": 0.11200004816055298, + "tpp_threshold_50_unintended_diff_only": 0.008000001311302185, + "tpp_threshold_100_total_metric": 0.14750006794929504, + "tpp_threshold_100_intended_diff_only": 0.16200006008148193, + "tpp_threshold_100_unintended_diff_only": 0.01449999213218689, + "tpp_threshold_500_total_metric": 0.33225007355213165, + "tpp_threshold_500_intended_diff_only": 0.35700005292892456, + "tpp_threshold_500_unintended_diff_only": 0.024749979376792908 }, "3": { - "tpp_threshold_2_total_metric": 0.012000024318695068, - "tpp_threshold_2_intended_diff_only": 0.004999995231628418, - "tpp_threshold_2_unintended_diff_only": -0.00700002908706665, - "tpp_threshold_5_total_metric": 0.015499994158744812, - "tpp_threshold_5_intended_diff_only": 0.013999998569488525, - "tpp_threshold_5_unintended_diff_only": -0.0014999955892562866, - "tpp_threshold_10_total_metric": 0.01374998688697815, - "tpp_threshold_10_intended_diff_only": 0.014999985694885254, - "tpp_threshold_10_unintended_diff_only": 0.0012499988079071045 + "tpp_threshold_2_total_metric": 0.002499982714653015, + "tpp_threshold_2_intended_diff_only": 0.001999974250793457, + "tpp_threshold_2_unintended_diff_only": -0.0005000084638595581, + "tpp_threshold_5_total_metric": 0.011000007390975952, + "tpp_threshold_5_intended_diff_only": 0.004999995231628418, + "tpp_threshold_5_unintended_diff_only": -0.006000012159347534, + "tpp_threshold_10_total_metric": 0.014750003814697266, + "tpp_threshold_10_intended_diff_only": 0.013000011444091797, + "tpp_threshold_10_unintended_diff_only": -0.0017499923706054688, + "tpp_threshold_20_total_metric": 0.01150001585483551, + "tpp_threshold_20_intended_diff_only": 0.013999998569488525, + "tpp_threshold_20_unintended_diff_only": 0.002499982714653015, + "tpp_threshold_50_total_metric": 0.036250025033950806, + "tpp_threshold_50_intended_diff_only": 0.0350000262260437, + "tpp_threshold_50_unintended_diff_only": -0.0012499988079071045, + "tpp_threshold_100_total_metric": 0.08075003325939178, + "tpp_threshold_100_intended_diff_only": 0.09000003337860107, + "tpp_threshold_100_unintended_diff_only": 0.00925000011920929, + "tpp_threshold_500_total_metric": 0.30500005185604095, + "tpp_threshold_500_intended_diff_only": 0.33900004625320435, + "tpp_threshold_500_unintended_diff_only": 0.03399999439716339 }, "5": { - "tpp_threshold_2_total_metric": -0.00550001859664917, + "tpp_threshold_2_total_metric": 0.0014999806880950928, "tpp_threshold_2_intended_diff_only": 0.001999974250793457, - "tpp_threshold_2_unintended_diff_only": 0.007499992847442627, - "tpp_threshold_5_total_metric": 0.007249966263771057, - "tpp_threshold_5_intended_diff_only": 0.007999956607818604, - "tpp_threshold_5_unintended_diff_only": 0.0007499903440475464, - "tpp_threshold_10_total_metric": 0.014999985694885254, - "tpp_threshold_10_intended_diff_only": 0.018999993801116943, - "tpp_threshold_10_unintended_diff_only": 0.0040000081062316895 + "tpp_threshold_2_unintended_diff_only": 0.0004999935626983643, + "tpp_threshold_5_total_metric": 0.0157499760389328, + "tpp_threshold_5_intended_diff_only": 0.014999985694885254, + "tpp_threshold_5_unintended_diff_only": -0.0007499903440475464, + "tpp_threshold_10_total_metric": 0.010499954223632812, + "tpp_threshold_10_intended_diff_only": 0.01699995994567871, + "tpp_threshold_10_unintended_diff_only": 0.0065000057220458984, + "tpp_threshold_20_total_metric": 0.0547499805688858, + "tpp_threshold_20_intended_diff_only": 0.05699998140335083, + "tpp_threshold_20_unintended_diff_only": 0.002250000834465027, + "tpp_threshold_50_total_metric": 0.08724996447563171, + "tpp_threshold_50_intended_diff_only": 0.09799998998641968, + "tpp_threshold_50_unintended_diff_only": 0.010750025510787964, + "tpp_threshold_100_total_metric": 0.15824994444847107, + "tpp_threshold_100_intended_diff_only": 0.16999995708465576, + "tpp_threshold_100_unintended_diff_only": 0.011750012636184692, + "tpp_threshold_500_total_metric": 0.35649995505809784, + "tpp_threshold_500_intended_diff_only": 0.390999972820282, + "tpp_threshold_500_unintended_diff_only": 0.03450001776218414 }, "6": { - "tpp_threshold_2_total_metric": 0.025000005960464478, - "tpp_threshold_2_intended_diff_only": 0.02399998903274536, - "tpp_threshold_2_unintended_diff_only": -0.0010000169277191162, - "tpp_threshold_5_total_metric": 0.016249999403953552, - "tpp_threshold_5_intended_diff_only": 0.023000001907348633, - "tpp_threshold_5_unintended_diff_only": 0.006750002503395081, - "tpp_threshold_10_total_metric": 0.031250014901161194, - "tpp_threshold_10_intended_diff_only": 0.03200000524520874, - "tpp_threshold_10_unintended_diff_only": 0.0007499903440475464 + "tpp_threshold_2_total_metric": 0.011999964714050293, + "tpp_threshold_2_intended_diff_only": 0.015999972820281982, + "tpp_threshold_2_unintended_diff_only": 0.0040000081062316895, + "tpp_threshold_5_total_metric": 0.019999995827674866, + "tpp_threshold_5_intended_diff_only": 0.019999980926513672, + "tpp_threshold_5_unintended_diff_only": -1.4901161193847656e-08, + "tpp_threshold_10_total_metric": 0.0404999703168869, + "tpp_threshold_10_intended_diff_only": 0.04399996995925903, + "tpp_threshold_10_unintended_diff_only": 0.0034999996423721313, + "tpp_threshold_20_total_metric": 0.06600001454353333, + "tpp_threshold_20_intended_diff_only": 0.07300001382827759, + "tpp_threshold_20_unintended_diff_only": 0.006999999284744263, + "tpp_threshold_50_total_metric": 0.15449996292591095, + "tpp_threshold_50_intended_diff_only": 0.15999996662139893, + "tpp_threshold_50_unintended_diff_only": 0.005500003695487976, + "tpp_threshold_100_total_metric": 0.23275001347064972, + "tpp_threshold_100_intended_diff_only": 0.24400001764297485, + "tpp_threshold_100_unintended_diff_only": 0.011250004172325134, + "tpp_threshold_500_total_metric": 0.34050001204013824, + "tpp_threshold_500_intended_diff_only": 0.3709999918937683, + "tpp_threshold_500_unintended_diff_only": 0.030499979853630066 } } }