Run sync: periodic
Browse files
benchmarks/hle_results.jsonl
CHANGED
|
@@ -51,3 +51,4 @@
|
|
| 51 |
{"fingerprint": "715ff25b4d921449f9b180f7971863739f19e25391ea7c9e001f6ebe7acfdd28", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 33, "parameters": {"start_layer_index": 3, "end_layer_index": 32, "preserve_good_behavior_weight": 0.1270293935724024, "steer_bad_behavior_weight": 0.0005486555161052953, "overcorrect_relative_weight": 0.9669344077376423, "neighbor_count": 2, "loss_mode": "relative"}, "refusals_per_100": 5.0, "kl_divergence": 0.038692597299814224, "task": "hle", "dataset": "https://raw.githubusercontent.com/electroglyph/HLEV-subset/master/gold_mc_text.jsonl", "dataset_sha256": "41af3631244e6525a5b2c1c434199774fa1ff77201eb76829923abdd67af6fa4", "item_limit": 50, "run": 1, "runs": 1, "batch_size": 32, "temperature": 0.6, "top_p": 0.95, "max_new_tokens": 16384, "max_input_tokens": 6000, "seed": 20261024, "system_prompt": "You are a helpful assistant.", "grader": "multiple_choice", "grader_version": "4", "batching": "continuous", "benchmarked_at": "2026-10-27T13:27:48Z", "status": "ok", "hle": 0.08, "correct": 4, "total": 50, "by_type": {"multipleChoice": {"correct": 4, "total": 50}}, "reasons": {"multiple_choice": 4, "no_letter": 14, "wrong_letter": 32}}
|
| 52 |
{"fingerprint": "c57813b15cb05caf757b829cbe6f0dcba9c739e2c258022aa9da2e44e98ac97b", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 160, "parameters": {"start_layer_index": 12, "end_layer_index": 36, "preserve_good_behavior_weight": 0.9222557140202441, "steer_bad_behavior_weight": 0.0062694321172257084, "overcorrect_relative_weight": 0.9243536255699624, "neighbor_count": 14, "loss_mode": "relative"}, "refusals_per_100": 3.0, "kl_divergence": 0.03930002823472023, "task": "hle", "dataset": "https://raw.githubusercontent.com/electroglyph/HLEV-subset/master/gold_mc_text.jsonl", "dataset_sha256": "41af3631244e6525a5b2c1c434199774fa1ff77201eb76829923abdd67af6fa4", "item_limit": 50, "run": 1, "runs": 1, "batch_size": 32, "temperature": 0.6, "top_p": 0.95, "max_new_tokens": 16384, "max_input_tokens": 6000, "seed": 20261024, "system_prompt": "You are a helpful assistant.", "grader": "multiple_choice", "grader_version": "4", "batching": "continuous", "benchmarked_at": "2026-10-27T13:35:21Z", "status": "ok", "hle": 0.14, "correct": 7, "total": 50, "by_type": {"multipleChoice": {"correct": 7, "total": 50}}, "reasons": {"wrong_letter": 30, "no_letter": 13, "multiple_choice": 7}}
|
| 53 |
{"fingerprint": "ea3c50ec943b23d1a904b70956744eaafd12419e5ab054a04e00ef2cfddf78e2", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 184, "parameters": {"start_layer_index": 11, "end_layer_index": 35, "preserve_good_behavior_weight": 0.9489763128112341, "steer_bad_behavior_weight": 0.0072492065926696575, "overcorrect_relative_weight": 0.835476489930737, "neighbor_count": 14, "loss_mode": "relative"}, "refusals_per_100": 3.0, "kl_divergence": 0.03946484625339508, "task": "hle", "dataset": "https://raw.githubusercontent.com/electroglyph/HLEV-subset/master/gold_mc_text.jsonl", "dataset_sha256": "41af3631244e6525a5b2c1c434199774fa1ff77201eb76829923abdd67af6fa4", "item_limit": 50, "run": 1, "runs": 1, "batch_size": 32, "temperature": 0.6, "top_p": 0.95, "max_new_tokens": 16384, "max_input_tokens": 6000, "seed": 20261024, "system_prompt": "You are a helpful assistant.", "grader": "multiple_choice", "grader_version": "4", "batching": "continuous", "benchmarked_at": "2026-10-27T13:54:54Z", "status": "ok", "hle": 0.18, "correct": 9, "total": 50, "by_type": {"multipleChoice": {"correct": 9, "total": 50}}, "reasons": {"wrong_letter": 34, "multiple_choice": 9, "no_letter": 7}}
|
|
|
|
|
|
| 51 |
{"fingerprint": "715ff25b4d921449f9b180f7971863739f19e25391ea7c9e001f6ebe7acfdd28", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 33, "parameters": {"start_layer_index": 3, "end_layer_index": 32, "preserve_good_behavior_weight": 0.1270293935724024, "steer_bad_behavior_weight": 0.0005486555161052953, "overcorrect_relative_weight": 0.9669344077376423, "neighbor_count": 2, "loss_mode": "relative"}, "refusals_per_100": 5.0, "kl_divergence": 0.038692597299814224, "task": "hle", "dataset": "https://raw.githubusercontent.com/electroglyph/HLEV-subset/master/gold_mc_text.jsonl", "dataset_sha256": "41af3631244e6525a5b2c1c434199774fa1ff77201eb76829923abdd67af6fa4", "item_limit": 50, "run": 1, "runs": 1, "batch_size": 32, "temperature": 0.6, "top_p": 0.95, "max_new_tokens": 16384, "max_input_tokens": 6000, "seed": 20261024, "system_prompt": "You are a helpful assistant.", "grader": "multiple_choice", "grader_version": "4", "batching": "continuous", "benchmarked_at": "2026-10-27T13:27:48Z", "status": "ok", "hle": 0.08, "correct": 4, "total": 50, "by_type": {"multipleChoice": {"correct": 4, "total": 50}}, "reasons": {"multiple_choice": 4, "no_letter": 14, "wrong_letter": 32}}
|
| 52 |
{"fingerprint": "c57813b15cb05caf757b829cbe6f0dcba9c739e2c258022aa9da2e44e98ac97b", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 160, "parameters": {"start_layer_index": 12, "end_layer_index": 36, "preserve_good_behavior_weight": 0.9222557140202441, "steer_bad_behavior_weight": 0.0062694321172257084, "overcorrect_relative_weight": 0.9243536255699624, "neighbor_count": 14, "loss_mode": "relative"}, "refusals_per_100": 3.0, "kl_divergence": 0.03930002823472023, "task": "hle", "dataset": "https://raw.githubusercontent.com/electroglyph/HLEV-subset/master/gold_mc_text.jsonl", "dataset_sha256": "41af3631244e6525a5b2c1c434199774fa1ff77201eb76829923abdd67af6fa4", "item_limit": 50, "run": 1, "runs": 1, "batch_size": 32, "temperature": 0.6, "top_p": 0.95, "max_new_tokens": 16384, "max_input_tokens": 6000, "seed": 20261024, "system_prompt": "You are a helpful assistant.", "grader": "multiple_choice", "grader_version": "4", "batching": "continuous", "benchmarked_at": "2026-10-27T13:35:21Z", "status": "ok", "hle": 0.14, "correct": 7, "total": 50, "by_type": {"multipleChoice": {"correct": 7, "total": 50}}, "reasons": {"wrong_letter": 30, "no_letter": 13, "multiple_choice": 7}}
|
| 53 |
{"fingerprint": "ea3c50ec943b23d1a904b70956744eaafd12419e5ab054a04e00ef2cfddf78e2", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 184, "parameters": {"start_layer_index": 11, "end_layer_index": 35, "preserve_good_behavior_weight": 0.9489763128112341, "steer_bad_behavior_weight": 0.0072492065926696575, "overcorrect_relative_weight": 0.835476489930737, "neighbor_count": 14, "loss_mode": "relative"}, "refusals_per_100": 3.0, "kl_divergence": 0.03946484625339508, "task": "hle", "dataset": "https://raw.githubusercontent.com/electroglyph/HLEV-subset/master/gold_mc_text.jsonl", "dataset_sha256": "41af3631244e6525a5b2c1c434199774fa1ff77201eb76829923abdd67af6fa4", "item_limit": 50, "run": 1, "runs": 1, "batch_size": 32, "temperature": 0.6, "top_p": 0.95, "max_new_tokens": 16384, "max_input_tokens": 6000, "seed": 20261024, "system_prompt": "You are a helpful assistant.", "grader": "multiple_choice", "grader_version": "4", "batching": "continuous", "benchmarked_at": "2026-10-27T13:54:54Z", "status": "ok", "hle": 0.18, "correct": 9, "total": 50, "by_type": {"multipleChoice": {"correct": 9, "total": 50}}, "reasons": {"wrong_letter": 34, "multiple_choice": 9, "no_letter": 7}}
|
| 54 |
+
{"fingerprint": "018ec50fd58022a71996d11af7fc77db2ab8d293e7be1f5d32a3ebe058611c65", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 504, "parameters": {"start_layer_index": 10, "end_layer_index": 36, "preserve_good_behavior_weight": 0.8591258244460892, "steer_bad_behavior_weight": 0.009235912528390088, "overcorrect_relative_weight": 0.753603241755702, "neighbor_count": 14, "loss_mode": "relative"}, "refusals_per_100": 7.000000000000001, "kl_divergence": 0.040947724133729935, "task": "hle", "dataset": "https://raw.githubusercontent.com/electroglyph/HLEV-subset/master/gold_mc_text.jsonl", "dataset_sha256": "41af3631244e6525a5b2c1c434199774fa1ff77201eb76829923abdd67af6fa4", "item_limit": 50, "run": 1, "runs": 1, "batch_size": 32, "temperature": 0.6, "top_p": 0.95, "max_new_tokens": 16384, "max_input_tokens": 6000, "seed": 20261024, "system_prompt": "You are a helpful assistant.", "grader": "multiple_choice", "grader_version": "4", "batching": "continuous", "benchmarked_at": "2026-10-27T14:12:19Z", "status": "ok", "hle": 0.06, "correct": 3, "total": 50, "by_type": {"multipleChoice": {"correct": 3, "total": 50}}, "reasons": {"wrong_letter": 36, "no_letter": 11, "multiple_choice": 3}}
|
benchmarks/partial/285b06636d3361005ab87b66625b6e0d42cd8e982a0fc0be091a924e23ce0066-run1.jsonl
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
benchmarks/piqa_results.jsonl
CHANGED
|
@@ -16,3 +16,4 @@
|
|
| 16 |
{"fingerprint": "c57813b15cb05caf757b829cbe6f0dcba9c739e2c258022aa9da2e44e98ac97b", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 160, "parameters": {"start_layer_index": 12, "end_layer_index": 36, "preserve_good_behavior_weight": 0.9222557140202441, "steer_bad_behavior_weight": 0.0062694321172257084, "overcorrect_relative_weight": 0.9243536255699624, "neighbor_count": 14, "loss_mode": "relative"}, "refusals_per_100": 3.0, "kl_divergence": 0.03930002823472023, "task": "piqa", "metric": "acc_norm,none", "limit": null, "max_length": 1024, "batch_size": "auto", "benchmarked_at": "2026-10-27T13:34:59Z", "status": "ok", "piqa": 0.6741022850924918, "baseline_piqa": 0.676278563656148}
|
| 17 |
{"fingerprint": "ea3c50ec943b23d1a904b70956744eaafd12419e5ab054a04e00ef2cfddf78e2", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 184, "parameters": {"start_layer_index": 11, "end_layer_index": 35, "preserve_good_behavior_weight": 0.9489763128112341, "steer_bad_behavior_weight": 0.0072492065926696575, "overcorrect_relative_weight": 0.835476489930737, "neighbor_count": 14, "loss_mode": "relative"}, "refusals_per_100": 3.0, "kl_divergence": 0.03946484625339508, "task": "piqa", "metric": "acc_norm,none", "limit": null, "max_length": 1024, "batch_size": "auto", "benchmarked_at": "2026-10-27T13:54:32Z", "status": "ok", "piqa": 0.676278563656148, "baseline_piqa": 0.676278563656148}
|
| 18 |
{"fingerprint": "018ec50fd58022a71996d11af7fc77db2ab8d293e7be1f5d32a3ebe058611c65", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 504, "parameters": {"start_layer_index": 10, "end_layer_index": 36, "preserve_good_behavior_weight": 0.8591258244460892, "steer_bad_behavior_weight": 0.009235912528390088, "overcorrect_relative_weight": 0.753603241755702, "neighbor_count": 14, "loss_mode": "relative"}, "refusals_per_100": 7.000000000000001, "kl_divergence": 0.040947724133729935, "task": "piqa", "metric": "acc_norm,none", "limit": null, "max_length": 1024, "batch_size": "auto", "benchmarked_at": "2026-10-27T14:11:58Z", "status": "ok", "piqa": 0.6724700761697497, "baseline_piqa": 0.676278563656148}
|
|
|
|
|
|
| 16 |
{"fingerprint": "c57813b15cb05caf757b829cbe6f0dcba9c739e2c258022aa9da2e44e98ac97b", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 160, "parameters": {"start_layer_index": 12, "end_layer_index": 36, "preserve_good_behavior_weight": 0.9222557140202441, "steer_bad_behavior_weight": 0.0062694321172257084, "overcorrect_relative_weight": 0.9243536255699624, "neighbor_count": 14, "loss_mode": "relative"}, "refusals_per_100": 3.0, "kl_divergence": 0.03930002823472023, "task": "piqa", "metric": "acc_norm,none", "limit": null, "max_length": 1024, "batch_size": "auto", "benchmarked_at": "2026-10-27T13:34:59Z", "status": "ok", "piqa": 0.6741022850924918, "baseline_piqa": 0.676278563656148}
|
| 17 |
{"fingerprint": "ea3c50ec943b23d1a904b70956744eaafd12419e5ab054a04e00ef2cfddf78e2", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 184, "parameters": {"start_layer_index": 11, "end_layer_index": 35, "preserve_good_behavior_weight": 0.9489763128112341, "steer_bad_behavior_weight": 0.0072492065926696575, "overcorrect_relative_weight": 0.835476489930737, "neighbor_count": 14, "loss_mode": "relative"}, "refusals_per_100": 3.0, "kl_divergence": 0.03946484625339508, "task": "piqa", "metric": "acc_norm,none", "limit": null, "max_length": 1024, "batch_size": "auto", "benchmarked_at": "2026-10-27T13:54:32Z", "status": "ok", "piqa": 0.676278563656148, "baseline_piqa": 0.676278563656148}
|
| 18 |
{"fingerprint": "018ec50fd58022a71996d11af7fc77db2ab8d293e7be1f5d32a3ebe058611c65", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 504, "parameters": {"start_layer_index": 10, "end_layer_index": 36, "preserve_good_behavior_weight": 0.8591258244460892, "steer_bad_behavior_weight": 0.009235912528390088, "overcorrect_relative_weight": 0.753603241755702, "neighbor_count": 14, "loss_mode": "relative"}, "refusals_per_100": 7.000000000000001, "kl_divergence": 0.040947724133729935, "task": "piqa", "metric": "acc_norm,none", "limit": null, "max_length": 1024, "batch_size": "auto", "benchmarked_at": "2026-10-27T14:11:58Z", "status": "ok", "piqa": 0.6724700761697497, "baseline_piqa": 0.676278563656148}
|
| 19 |
+
{"fingerprint": "285b06636d3361005ab87b66625b6e0d42cd8e982a0fc0be091a924e23ce0066", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 903, "parameters": {"start_layer_index": 16, "end_layer_index": 19, "preserve_good_behavior_weight": 0.14620773982831453, "steer_bad_behavior_weight": 0.00018511355185863368, "overcorrect_relative_weight": 1.597231821679931, "neighbor_count": 6, "loss_mode": "relative"}, "refusals_per_100": 0.0, "kl_divergence": 0.04501604661345482, "task": "piqa", "metric": "acc_norm,none", "limit": null, "max_length": 1024, "batch_size": "auto", "benchmarked_at": "2026-10-27T14:25:15Z", "status": "ok", "piqa": 0.675734494015234, "baseline_piqa": 0.676278563656148}
|
benchmarks/results.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"generated_at": "2026-10-27T14:
|
| 3 |
"task": "hle",
|
| 4 |
"dataset": "https://raw.githubusercontent.com/electroglyph/HLEV-subset/master/gold_mc_text.jsonl",
|
| 5 |
"dataset_sha256": "41af3631244e6525a5b2c1c434199774fa1ff77201eb76829923abdd67af6fa4",
|
|
@@ -23,59 +23,59 @@
|
|
| 23 |
"piqa_metric": "acc_norm,none",
|
| 24 |
"counts": {
|
| 25 |
"candidates": 50,
|
| 26 |
-
"benchmarked":
|
| 27 |
-
"complete_all_runs":
|
| 28 |
-
"failed":
|
| 29 |
"not_benchmarked": 0
|
| 30 |
},
|
| 31 |
"correlations": {
|
| 32 |
-
"n":
|
| 33 |
"piqa_hle": {
|
| 34 |
-
"rho": 0.
|
| 35 |
-
"p_approx": 0.
|
| 36 |
-
"n":
|
| 37 |
},
|
| 38 |
"low_kl_hle": {
|
| 39 |
-
"rho":
|
| 40 |
-
"p_approx": 0.
|
| 41 |
-
"n":
|
| 42 |
},
|
| 43 |
"kl_hle": {
|
| 44 |
-
"rho": 0.
|
| 45 |
-
"p_approx": 0.
|
| 46 |
-
"n":
|
| 47 |
},
|
| 48 |
-
"pearson_piqa_hle": 0.
|
| 49 |
-
"pearson_low_kl_hle": -0.
|
| 50 |
"kendall_piqa_hle": {
|
| 51 |
-
"tau": 0.
|
| 52 |
-
"p_approx": 0.
|
| 53 |
-
"n":
|
| 54 |
},
|
| 55 |
"kendall_low_kl_hle": {
|
| 56 |
-
"tau": -0.
|
| 57 |
-
"p_approx": 0.
|
| 58 |
-
"n":
|
| 59 |
},
|
| 60 |
"per_run": [
|
| 61 |
{
|
| 62 |
"run": 1,
|
| 63 |
-
"n":
|
| 64 |
"piqa_hle": {
|
| 65 |
-
"rho": 0.
|
| 66 |
-
"p_approx": 0.
|
| 67 |
-
"n":
|
| 68 |
},
|
| 69 |
"low_kl_hle": {
|
| 70 |
-
"rho":
|
| 71 |
-
"p_approx": 0.
|
| 72 |
-
"n":
|
| 73 |
}
|
| 74 |
}
|
| 75 |
],
|
| 76 |
"conclusion": {
|
| 77 |
"winner": null,
|
| 78 |
-
"note": "PIQA rho=0.
|
| 79 |
}
|
| 80 |
},
|
| 81 |
"selected": {
|
|
@@ -797,6 +797,47 @@
|
|
| 797 |
"loss_mode": "relative"
|
| 798 |
}
|
| 799 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 800 |
{
|
| 801 |
"rank_in_history": 19,
|
| 802 |
"fingerprint": "285b06636d3361005ab87b66625b6e0d42cd8e982a0fc0be091a924e23ce0066",
|
|
@@ -1847,41 +1888,6 @@
|
|
| 1847 |
"loss_mode": "relative"
|
| 1848 |
}
|
| 1849 |
},
|
| 1850 |
-
{
|
| 1851 |
-
"rank_in_history": 18,
|
| 1852 |
-
"fingerprint": "018ec50fd58022a71996d11af7fc77db2ab8d293e7be1f5d32a3ebe058611c65",
|
| 1853 |
-
"group_key": "{\"max_response_length\": 100, \"model\": \"XHToken/Spark-X2.6-4B\", \"model_commit\": null, \"modifier\": {\"ARA\": {\"bad_prompts\": {\"column\": \"text\", \"dataset\": \"mlabonne/harmful_behaviors\", \"split\": \"train[:400]\"}, \"good_prompts\": {\"column\": \"text\", \"dataset\": \"mlabonne/harmless_alpaca\", \"split\": \"train[:1000]\"}, \"history_size\": 10, \"learning_rate\": 1.0, \"lora_rank\": 100, \"max_iter\": 20, \"n_optimization_steps\": 5, \"preserve_row_magnitudes\": true, \"print_loss\": false}}, \"modifiers\": [{\"instance_name\": null, \"plugin\": \"heretic.modifiers.ara.ARA\"}], \"scorers\": [{\"instance_name\": null, \"optimization\": \"minimize\", \"plugin\": \"heretic.scorers.keyword_rate.KeywordRate\"}, {\"instance_name\": null, \"optimization\": \"minimize\", \"plugin\": \"heretic.scorers.kl_divergence.KLDivergence\"}], \"seed\": 1234, \"system_prompt\": \"You are a helpful assistant.\"}",
|
| 1854 |
-
"source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl",
|
| 1855 |
-
"model": "XHToken/Spark-X2.6-4B",
|
| 1856 |
-
"model_commit": null,
|
| 1857 |
-
"trial_index": 504,
|
| 1858 |
-
"refusals_per_100": 7.000000000000001,
|
| 1859 |
-
"kl_divergence": 0.040947724133729935,
|
| 1860 |
-
"hle": null,
|
| 1861 |
-
"hle_mean": null,
|
| 1862 |
-
"hle_std": 0.0,
|
| 1863 |
-
"hle_runs": [],
|
| 1864 |
-
"hle_run_scores": {},
|
| 1865 |
-
"runs_completed": [],
|
| 1866 |
-
"runs_requested": 1,
|
| 1867 |
-
"complete": false,
|
| 1868 |
-
"piqa": null,
|
| 1869 |
-
"baseline_piqa": null,
|
| 1870 |
-
"piqa_task": "piqa",
|
| 1871 |
-
"piqa_metric": "acc_norm,none",
|
| 1872 |
-
"piqa_benchmarked_at": null,
|
| 1873 |
-
"status": "error",
|
| 1874 |
-
"error": "UnboundLocalError: cannot access local variable '_CURRENT_PHASE' where it is not associated with a value",
|
| 1875 |
-
"parameters": {
|
| 1876 |
-
"start_layer_index": 10,
|
| 1877 |
-
"end_layer_index": 36,
|
| 1878 |
-
"preserve_good_behavior_weight": 0.8591258244460892,
|
| 1879 |
-
"steer_bad_behavior_weight": 0.009235912528390088,
|
| 1880 |
-
"overcorrect_relative_weight": 0.753603241755702,
|
| 1881 |
-
"neighbor_count": 14,
|
| 1882 |
-
"loss_mode": "relative"
|
| 1883 |
-
}
|
| 1884 |
-
},
|
| 1885 |
{
|
| 1886 |
"rank_in_history": 34,
|
| 1887 |
"fingerprint": "99efc7c274722e4f755b3106fcebfbc63b81eb71ed2a4cb930a87df32dd2ed27",
|
|
@@ -2087,9 +2093,22 @@
|
|
| 2087 |
"grader": "multiple_choice",
|
| 2088 |
"grader_version": "4",
|
| 2089 |
"batching": "continuous",
|
| 2090 |
-
"benchmarked_at": "2026-10-
|
| 2091 |
-
"status": "
|
| 2092 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2093 |
},
|
| 2094 |
{
|
| 2095 |
"fingerprint": "05f3bebebb335c6dbad9a30349a63e37a26f6362926ff80de9167d9314d070a6",
|
|
|
|
| 1 |
{
|
| 2 |
+
"generated_at": "2026-10-27T14:25:14Z",
|
| 3 |
"task": "hle",
|
| 4 |
"dataset": "https://raw.githubusercontent.com/electroglyph/HLEV-subset/master/gold_mc_text.jsonl",
|
| 5 |
"dataset_sha256": "41af3631244e6525a5b2c1c434199774fa1ff77201eb76829923abdd67af6fa4",
|
|
|
|
| 23 |
"piqa_metric": "acc_norm,none",
|
| 24 |
"counts": {
|
| 25 |
"candidates": 50,
|
| 26 |
+
"benchmarked": 18,
|
| 27 |
+
"complete_all_runs": 18,
|
| 28 |
+
"failed": 32,
|
| 29 |
"not_benchmarked": 0
|
| 30 |
},
|
| 31 |
"correlations": {
|
| 32 |
+
"n": 18,
|
| 33 |
"piqa_hle": {
|
| 34 |
+
"rho": 0.4231728994713164,
|
| 35 |
+
"p_approx": 0.0810220287362714,
|
| 36 |
+
"n": 18
|
| 37 |
},
|
| 38 |
"low_kl_hle": {
|
| 39 |
+
"rho": 0.006280061698141223,
|
| 40 |
+
"p_approx": 0.9793423980460527,
|
| 41 |
+
"n": 18
|
| 42 |
},
|
| 43 |
"kl_hle": {
|
| 44 |
+
"rho": -0.006280061698141223,
|
| 45 |
+
"p_approx": 0.9793423980460527,
|
| 46 |
+
"n": 18
|
| 47 |
},
|
| 48 |
+
"pearson_piqa_hle": 0.3278191818095267,
|
| 49 |
+
"pearson_low_kl_hle": -0.15232777193162023,
|
| 50 |
"kendall_piqa_hle": {
|
| 51 |
+
"tau": 0.32992080356581704,
|
| 52 |
+
"p_approx": 0.0558779698939842,
|
| 53 |
+
"n": 18
|
| 54 |
},
|
| 55 |
"kendall_low_kl_hle": {
|
| 56 |
+
"tau": -0.020874143036171647,
|
| 57 |
+
"p_approx": 0.9037134698545567,
|
| 58 |
+
"n": 18
|
| 59 |
},
|
| 60 |
"per_run": [
|
| 61 |
{
|
| 62 |
"run": 1,
|
| 63 |
+
"n": 18,
|
| 64 |
"piqa_hle": {
|
| 65 |
+
"rho": 0.4231728994713164,
|
| 66 |
+
"p_approx": 0.0810220287362714,
|
| 67 |
+
"n": 18
|
| 68 |
},
|
| 69 |
"low_kl_hle": {
|
| 70 |
+
"rho": 0.006280061698141223,
|
| 71 |
+
"p_approx": 0.9793423980460527,
|
| 72 |
+
"n": 18
|
| 73 |
}
|
| 74 |
}
|
| 75 |
],
|
| 76 |
"conclusion": {
|
| 77 |
"winner": null,
|
| 78 |
+
"note": "PIQA rho=0.423 leads low-KLD rho=0.006 but p~0.081 >= 0.05; no call"
|
| 79 |
}
|
| 80 |
},
|
| 81 |
"selected": {
|
|
|
|
| 797 |
"loss_mode": "relative"
|
| 798 |
}
|
| 799 |
},
|
| 800 |
+
{
|
| 801 |
+
"rank_in_history": 18,
|
| 802 |
+
"fingerprint": "018ec50fd58022a71996d11af7fc77db2ab8d293e7be1f5d32a3ebe058611c65",
|
| 803 |
+
"group_key": "{\"max_response_length\": 100, \"model\": \"XHToken/Spark-X2.6-4B\", \"model_commit\": null, \"modifier\": {\"ARA\": {\"bad_prompts\": {\"column\": \"text\", \"dataset\": \"mlabonne/harmful_behaviors\", \"split\": \"train[:400]\"}, \"good_prompts\": {\"column\": \"text\", \"dataset\": \"mlabonne/harmless_alpaca\", \"split\": \"train[:1000]\"}, \"history_size\": 10, \"learning_rate\": 1.0, \"lora_rank\": 100, \"max_iter\": 20, \"n_optimization_steps\": 5, \"preserve_row_magnitudes\": true, \"print_loss\": false}}, \"modifiers\": [{\"instance_name\": null, \"plugin\": \"heretic.modifiers.ara.ARA\"}], \"scorers\": [{\"instance_name\": null, \"optimization\": \"minimize\", \"plugin\": \"heretic.scorers.keyword_rate.KeywordRate\"}, {\"instance_name\": null, \"optimization\": \"minimize\", \"plugin\": \"heretic.scorers.kl_divergence.KLDivergence\"}], \"seed\": 1234, \"system_prompt\": \"You are a helpful assistant.\"}",
|
| 804 |
+
"source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl",
|
| 805 |
+
"model": "XHToken/Spark-X2.6-4B",
|
| 806 |
+
"model_commit": null,
|
| 807 |
+
"trial_index": 504,
|
| 808 |
+
"refusals_per_100": 7.000000000000001,
|
| 809 |
+
"kl_divergence": 0.040947724133729935,
|
| 810 |
+
"hle": 0.06,
|
| 811 |
+
"hle_mean": 0.06,
|
| 812 |
+
"hle_std": 0.0,
|
| 813 |
+
"hle_runs": [
|
| 814 |
+
0.06
|
| 815 |
+
],
|
| 816 |
+
"hle_run_scores": {
|
| 817 |
+
"1": 0.06
|
| 818 |
+
},
|
| 819 |
+
"runs_completed": [
|
| 820 |
+
1
|
| 821 |
+
],
|
| 822 |
+
"runs_requested": 1,
|
| 823 |
+
"complete": true,
|
| 824 |
+
"piqa": 0.6724700761697497,
|
| 825 |
+
"baseline_piqa": 0.676278563656148,
|
| 826 |
+
"piqa_task": "piqa",
|
| 827 |
+
"piqa_metric": "acc_norm,none",
|
| 828 |
+
"piqa_benchmarked_at": "2026-10-27T14:11:58Z",
|
| 829 |
+
"status": "ok",
|
| 830 |
+
"error": null,
|
| 831 |
+
"parameters": {
|
| 832 |
+
"start_layer_index": 10,
|
| 833 |
+
"end_layer_index": 36,
|
| 834 |
+
"preserve_good_behavior_weight": 0.8591258244460892,
|
| 835 |
+
"steer_bad_behavior_weight": 0.009235912528390088,
|
| 836 |
+
"overcorrect_relative_weight": 0.753603241755702,
|
| 837 |
+
"neighbor_count": 14,
|
| 838 |
+
"loss_mode": "relative"
|
| 839 |
+
}
|
| 840 |
+
},
|
| 841 |
{
|
| 842 |
"rank_in_history": 19,
|
| 843 |
"fingerprint": "285b06636d3361005ab87b66625b6e0d42cd8e982a0fc0be091a924e23ce0066",
|
|
|
|
| 1888 |
"loss_mode": "relative"
|
| 1889 |
}
|
| 1890 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1891 |
{
|
| 1892 |
"rank_in_history": 34,
|
| 1893 |
"fingerprint": "99efc7c274722e4f755b3106fcebfbc63b81eb71ed2a4cb930a87df32dd2ed27",
|
|
|
|
| 2093 |
"grader": "multiple_choice",
|
| 2094 |
"grader_version": "4",
|
| 2095 |
"batching": "continuous",
|
| 2096 |
+
"benchmarked_at": "2026-10-27T14:12:19Z",
|
| 2097 |
+
"status": "ok",
|
| 2098 |
+
"hle": 0.06,
|
| 2099 |
+
"correct": 3,
|
| 2100 |
+
"total": 50,
|
| 2101 |
+
"by_type": {
|
| 2102 |
+
"multipleChoice": {
|
| 2103 |
+
"correct": 3,
|
| 2104 |
+
"total": 50
|
| 2105 |
+
}
|
| 2106 |
+
},
|
| 2107 |
+
"reasons": {
|
| 2108 |
+
"wrong_letter": 36,
|
| 2109 |
+
"no_letter": 11,
|
| 2110 |
+
"multiple_choice": 3
|
| 2111 |
+
}
|
| 2112 |
},
|
| 2113 |
{
|
| 2114 |
"fingerprint": "05f3bebebb335c6dbad9a30349a63e37a26f6362926ff80de9167d9314d070a6",
|
run_status.json
CHANGED
|
@@ -5,18 +5,18 @@
|
|
| 5 |
"phase": "generating",
|
| 6 |
"pid": 385,
|
| 7 |
"started_at": "2026-10-27T13:13:45Z",
|
| 8 |
-
"updated_at": "2026-10-27T14:
|
| 9 |
"work_dir": "/marimo/heretic_work_bench",
|
| 10 |
"log_file": "run-20261027T131345Z.log",
|
| 11 |
"candidate_limit": 50,
|
| 12 |
"hle_runs": 1,
|
| 13 |
"hle_items": 50,
|
| 14 |
"hle_batch_size": 32,
|
| 15 |
-
"last_progress_utc": "2026-10-27T14:
|
| 16 |
-
"sync_stale_seconds":
|
| 17 |
"gpu_free_gb": 67.6,
|
| 18 |
"disk_free_gb": 8589934578.1,
|
| 19 |
-
"item_done":
|
| 20 |
"trials": 1100,
|
| 21 |
"eligible": 155,
|
| 22 |
"candidates": 50,
|
|
@@ -25,16 +25,16 @@
|
|
| 25 |
"mode": "continuous",
|
| 26 |
"item_total": 50,
|
| 27 |
"wave": 1,
|
| 28 |
-
"round":
|
| 29 |
-
"active":
|
| 30 |
-
"candidate_position":
|
| 31 |
"candidate_total": 50,
|
| 32 |
-
"trial_index":
|
| 33 |
-
"fingerprint": "
|
| 34 |
"batching": "continuous",
|
| 35 |
-
"last_sync_ok_utc": "2026-10-27T14:
|
| 36 |
"run": 1,
|
| 37 |
"runs_requested": 1,
|
| 38 |
-
"partial_checkpoint": "/marimo/heretic_work_bench/benchmarks/partial/
|
| 39 |
-
"items_in_batch":
|
| 40 |
}
|
|
|
|
| 5 |
"phase": "generating",
|
| 6 |
"pid": 385,
|
| 7 |
"started_at": "2026-10-27T13:13:45Z",
|
| 8 |
+
"updated_at": "2026-10-27T14:34:27Z",
|
| 9 |
"work_dir": "/marimo/heretic_work_bench",
|
| 10 |
"log_file": "run-20261027T131345Z.log",
|
| 11 |
"candidate_limit": 50,
|
| 12 |
"hle_runs": 1,
|
| 13 |
"hle_items": 50,
|
| 14 |
"hle_batch_size": 32,
|
| 15 |
+
"last_progress_utc": "2026-10-27T14:34:27Z",
|
| 16 |
+
"sync_stale_seconds": 579.5,
|
| 17 |
"gpu_free_gb": 67.6,
|
| 18 |
"disk_free_gb": 8589934578.1,
|
| 19 |
+
"item_done": 15,
|
| 20 |
"trials": 1100,
|
| 21 |
"eligible": 155,
|
| 22 |
"candidates": 50,
|
|
|
|
| 25 |
"mode": "continuous",
|
| 26 |
"item_total": 50,
|
| 27 |
"wave": 1,
|
| 28 |
+
"round": 34,
|
| 29 |
+
"active": 19,
|
| 30 |
+
"candidate_position": 19,
|
| 31 |
"candidate_total": 50,
|
| 32 |
+
"trial_index": 903,
|
| 33 |
+
"fingerprint": "285b06636d3361005ab87b66625b6e0d42cd8e982a0fc0be091a924e23ce0066",
|
| 34 |
"batching": "continuous",
|
| 35 |
+
"last_sync_ok_utc": "2026-10-27T14:24:47Z",
|
| 36 |
"run": 1,
|
| 37 |
"runs_requested": 1,
|
| 38 |
+
"partial_checkpoint": "/marimo/heretic_work_bench/benchmarks/partial/285b06636d3361005ab87b66625b6e0d42cd8e982a0fc0be091a924e23ce0066-run1.jsonl",
|
| 39 |
+
"items_in_batch": 1
|
| 40 |
}
|