electroglyph commited on
Commit
57a4cf2
·
verified ·
1 Parent(s): f07cf85

Run sync: periodic

Browse files
benchmarks/hle_results.jsonl CHANGED
@@ -51,3 +51,4 @@
51
  {"fingerprint": "715ff25b4d921449f9b180f7971863739f19e25391ea7c9e001f6ebe7acfdd28", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 33, "parameters": {"start_layer_index": 3, "end_layer_index": 32, "preserve_good_behavior_weight": 0.1270293935724024, "steer_bad_behavior_weight": 0.0005486555161052953, "overcorrect_relative_weight": 0.9669344077376423, "neighbor_count": 2, "loss_mode": "relative"}, "refusals_per_100": 5.0, "kl_divergence": 0.038692597299814224, "task": "hle", "dataset": "https://raw.githubusercontent.com/electroglyph/HLEV-subset/master/gold_mc_text.jsonl", "dataset_sha256": "41af3631244e6525a5b2c1c434199774fa1ff77201eb76829923abdd67af6fa4", "item_limit": 50, "run": 1, "runs": 1, "batch_size": 32, "temperature": 0.6, "top_p": 0.95, "max_new_tokens": 16384, "max_input_tokens": 6000, "seed": 20261024, "system_prompt": "You are a helpful assistant.", "grader": "multiple_choice", "grader_version": "4", "batching": "continuous", "benchmarked_at": "2026-10-27T13:27:48Z", "status": "ok", "hle": 0.08, "correct": 4, "total": 50, "by_type": {"multipleChoice": {"correct": 4, "total": 50}}, "reasons": {"multiple_choice": 4, "no_letter": 14, "wrong_letter": 32}}
52
  {"fingerprint": "c57813b15cb05caf757b829cbe6f0dcba9c739e2c258022aa9da2e44e98ac97b", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 160, "parameters": {"start_layer_index": 12, "end_layer_index": 36, "preserve_good_behavior_weight": 0.9222557140202441, "steer_bad_behavior_weight": 0.0062694321172257084, "overcorrect_relative_weight": 0.9243536255699624, "neighbor_count": 14, "loss_mode": "relative"}, "refusals_per_100": 3.0, "kl_divergence": 0.03930002823472023, "task": "hle", "dataset": "https://raw.githubusercontent.com/electroglyph/HLEV-subset/master/gold_mc_text.jsonl", "dataset_sha256": "41af3631244e6525a5b2c1c434199774fa1ff77201eb76829923abdd67af6fa4", "item_limit": 50, "run": 1, "runs": 1, "batch_size": 32, "temperature": 0.6, "top_p": 0.95, "max_new_tokens": 16384, "max_input_tokens": 6000, "seed": 20261024, "system_prompt": "You are a helpful assistant.", "grader": "multiple_choice", "grader_version": "4", "batching": "continuous", "benchmarked_at": "2026-10-27T13:35:21Z", "status": "ok", "hle": 0.14, "correct": 7, "total": 50, "by_type": {"multipleChoice": {"correct": 7, "total": 50}}, "reasons": {"wrong_letter": 30, "no_letter": 13, "multiple_choice": 7}}
53
  {"fingerprint": "ea3c50ec943b23d1a904b70956744eaafd12419e5ab054a04e00ef2cfddf78e2", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 184, "parameters": {"start_layer_index": 11, "end_layer_index": 35, "preserve_good_behavior_weight": 0.9489763128112341, "steer_bad_behavior_weight": 0.0072492065926696575, "overcorrect_relative_weight": 0.835476489930737, "neighbor_count": 14, "loss_mode": "relative"}, "refusals_per_100": 3.0, "kl_divergence": 0.03946484625339508, "task": "hle", "dataset": "https://raw.githubusercontent.com/electroglyph/HLEV-subset/master/gold_mc_text.jsonl", "dataset_sha256": "41af3631244e6525a5b2c1c434199774fa1ff77201eb76829923abdd67af6fa4", "item_limit": 50, "run": 1, "runs": 1, "batch_size": 32, "temperature": 0.6, "top_p": 0.95, "max_new_tokens": 16384, "max_input_tokens": 6000, "seed": 20261024, "system_prompt": "You are a helpful assistant.", "grader": "multiple_choice", "grader_version": "4", "batching": "continuous", "benchmarked_at": "2026-10-27T13:54:54Z", "status": "ok", "hle": 0.18, "correct": 9, "total": 50, "by_type": {"multipleChoice": {"correct": 9, "total": 50}}, "reasons": {"wrong_letter": 34, "multiple_choice": 9, "no_letter": 7}}
 
 
51
  {"fingerprint": "715ff25b4d921449f9b180f7971863739f19e25391ea7c9e001f6ebe7acfdd28", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 33, "parameters": {"start_layer_index": 3, "end_layer_index": 32, "preserve_good_behavior_weight": 0.1270293935724024, "steer_bad_behavior_weight": 0.0005486555161052953, "overcorrect_relative_weight": 0.9669344077376423, "neighbor_count": 2, "loss_mode": "relative"}, "refusals_per_100": 5.0, "kl_divergence": 0.038692597299814224, "task": "hle", "dataset": "https://raw.githubusercontent.com/electroglyph/HLEV-subset/master/gold_mc_text.jsonl", "dataset_sha256": "41af3631244e6525a5b2c1c434199774fa1ff77201eb76829923abdd67af6fa4", "item_limit": 50, "run": 1, "runs": 1, "batch_size": 32, "temperature": 0.6, "top_p": 0.95, "max_new_tokens": 16384, "max_input_tokens": 6000, "seed": 20261024, "system_prompt": "You are a helpful assistant.", "grader": "multiple_choice", "grader_version": "4", "batching": "continuous", "benchmarked_at": "2026-10-27T13:27:48Z", "status": "ok", "hle": 0.08, "correct": 4, "total": 50, "by_type": {"multipleChoice": {"correct": 4, "total": 50}}, "reasons": {"multiple_choice": 4, "no_letter": 14, "wrong_letter": 32}}
52
  {"fingerprint": "c57813b15cb05caf757b829cbe6f0dcba9c739e2c258022aa9da2e44e98ac97b", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 160, "parameters": {"start_layer_index": 12, "end_layer_index": 36, "preserve_good_behavior_weight": 0.9222557140202441, "steer_bad_behavior_weight": 0.0062694321172257084, "overcorrect_relative_weight": 0.9243536255699624, "neighbor_count": 14, "loss_mode": "relative"}, "refusals_per_100": 3.0, "kl_divergence": 0.03930002823472023, "task": "hle", "dataset": "https://raw.githubusercontent.com/electroglyph/HLEV-subset/master/gold_mc_text.jsonl", "dataset_sha256": "41af3631244e6525a5b2c1c434199774fa1ff77201eb76829923abdd67af6fa4", "item_limit": 50, "run": 1, "runs": 1, "batch_size": 32, "temperature": 0.6, "top_p": 0.95, "max_new_tokens": 16384, "max_input_tokens": 6000, "seed": 20261024, "system_prompt": "You are a helpful assistant.", "grader": "multiple_choice", "grader_version": "4", "batching": "continuous", "benchmarked_at": "2026-10-27T13:35:21Z", "status": "ok", "hle": 0.14, "correct": 7, "total": 50, "by_type": {"multipleChoice": {"correct": 7, "total": 50}}, "reasons": {"wrong_letter": 30, "no_letter": 13, "multiple_choice": 7}}
53
  {"fingerprint": "ea3c50ec943b23d1a904b70956744eaafd12419e5ab054a04e00ef2cfddf78e2", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 184, "parameters": {"start_layer_index": 11, "end_layer_index": 35, "preserve_good_behavior_weight": 0.9489763128112341, "steer_bad_behavior_weight": 0.0072492065926696575, "overcorrect_relative_weight": 0.835476489930737, "neighbor_count": 14, "loss_mode": "relative"}, "refusals_per_100": 3.0, "kl_divergence": 0.03946484625339508, "task": "hle", "dataset": "https://raw.githubusercontent.com/electroglyph/HLEV-subset/master/gold_mc_text.jsonl", "dataset_sha256": "41af3631244e6525a5b2c1c434199774fa1ff77201eb76829923abdd67af6fa4", "item_limit": 50, "run": 1, "runs": 1, "batch_size": 32, "temperature": 0.6, "top_p": 0.95, "max_new_tokens": 16384, "max_input_tokens": 6000, "seed": 20261024, "system_prompt": "You are a helpful assistant.", "grader": "multiple_choice", "grader_version": "4", "batching": "continuous", "benchmarked_at": "2026-10-27T13:54:54Z", "status": "ok", "hle": 0.18, "correct": 9, "total": 50, "by_type": {"multipleChoice": {"correct": 9, "total": 50}}, "reasons": {"wrong_letter": 34, "multiple_choice": 9, "no_letter": 7}}
54
+ {"fingerprint": "018ec50fd58022a71996d11af7fc77db2ab8d293e7be1f5d32a3ebe058611c65", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 504, "parameters": {"start_layer_index": 10, "end_layer_index": 36, "preserve_good_behavior_weight": 0.8591258244460892, "steer_bad_behavior_weight": 0.009235912528390088, "overcorrect_relative_weight": 0.753603241755702, "neighbor_count": 14, "loss_mode": "relative"}, "refusals_per_100": 7.000000000000001, "kl_divergence": 0.040947724133729935, "task": "hle", "dataset": "https://raw.githubusercontent.com/electroglyph/HLEV-subset/master/gold_mc_text.jsonl", "dataset_sha256": "41af3631244e6525a5b2c1c434199774fa1ff77201eb76829923abdd67af6fa4", "item_limit": 50, "run": 1, "runs": 1, "batch_size": 32, "temperature": 0.6, "top_p": 0.95, "max_new_tokens": 16384, "max_input_tokens": 6000, "seed": 20261024, "system_prompt": "You are a helpful assistant.", "grader": "multiple_choice", "grader_version": "4", "batching": "continuous", "benchmarked_at": "2026-10-27T14:12:19Z", "status": "ok", "hle": 0.06, "correct": 3, "total": 50, "by_type": {"multipleChoice": {"correct": 3, "total": 50}}, "reasons": {"wrong_letter": 36, "no_letter": 11, "multiple_choice": 3}}
benchmarks/partial/285b06636d3361005ab87b66625b6e0d42cd8e982a0fc0be091a924e23ce0066-run1.jsonl CHANGED
The diff for this file is too large to render. See raw diff
 
benchmarks/piqa_results.jsonl CHANGED
@@ -16,3 +16,4 @@
16
  {"fingerprint": "c57813b15cb05caf757b829cbe6f0dcba9c739e2c258022aa9da2e44e98ac97b", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 160, "parameters": {"start_layer_index": 12, "end_layer_index": 36, "preserve_good_behavior_weight": 0.9222557140202441, "steer_bad_behavior_weight": 0.0062694321172257084, "overcorrect_relative_weight": 0.9243536255699624, "neighbor_count": 14, "loss_mode": "relative"}, "refusals_per_100": 3.0, "kl_divergence": 0.03930002823472023, "task": "piqa", "metric": "acc_norm,none", "limit": null, "max_length": 1024, "batch_size": "auto", "benchmarked_at": "2026-10-27T13:34:59Z", "status": "ok", "piqa": 0.6741022850924918, "baseline_piqa": 0.676278563656148}
17
  {"fingerprint": "ea3c50ec943b23d1a904b70956744eaafd12419e5ab054a04e00ef2cfddf78e2", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 184, "parameters": {"start_layer_index": 11, "end_layer_index": 35, "preserve_good_behavior_weight": 0.9489763128112341, "steer_bad_behavior_weight": 0.0072492065926696575, "overcorrect_relative_weight": 0.835476489930737, "neighbor_count": 14, "loss_mode": "relative"}, "refusals_per_100": 3.0, "kl_divergence": 0.03946484625339508, "task": "piqa", "metric": "acc_norm,none", "limit": null, "max_length": 1024, "batch_size": "auto", "benchmarked_at": "2026-10-27T13:54:32Z", "status": "ok", "piqa": 0.676278563656148, "baseline_piqa": 0.676278563656148}
18
  {"fingerprint": "018ec50fd58022a71996d11af7fc77db2ab8d293e7be1f5d32a3ebe058611c65", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 504, "parameters": {"start_layer_index": 10, "end_layer_index": 36, "preserve_good_behavior_weight": 0.8591258244460892, "steer_bad_behavior_weight": 0.009235912528390088, "overcorrect_relative_weight": 0.753603241755702, "neighbor_count": 14, "loss_mode": "relative"}, "refusals_per_100": 7.000000000000001, "kl_divergence": 0.040947724133729935, "task": "piqa", "metric": "acc_norm,none", "limit": null, "max_length": 1024, "batch_size": "auto", "benchmarked_at": "2026-10-27T14:11:58Z", "status": "ok", "piqa": 0.6724700761697497, "baseline_piqa": 0.676278563656148}
 
 
16
  {"fingerprint": "c57813b15cb05caf757b829cbe6f0dcba9c739e2c258022aa9da2e44e98ac97b", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 160, "parameters": {"start_layer_index": 12, "end_layer_index": 36, "preserve_good_behavior_weight": 0.9222557140202441, "steer_bad_behavior_weight": 0.0062694321172257084, "overcorrect_relative_weight": 0.9243536255699624, "neighbor_count": 14, "loss_mode": "relative"}, "refusals_per_100": 3.0, "kl_divergence": 0.03930002823472023, "task": "piqa", "metric": "acc_norm,none", "limit": null, "max_length": 1024, "batch_size": "auto", "benchmarked_at": "2026-10-27T13:34:59Z", "status": "ok", "piqa": 0.6741022850924918, "baseline_piqa": 0.676278563656148}
17
  {"fingerprint": "ea3c50ec943b23d1a904b70956744eaafd12419e5ab054a04e00ef2cfddf78e2", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 184, "parameters": {"start_layer_index": 11, "end_layer_index": 35, "preserve_good_behavior_weight": 0.9489763128112341, "steer_bad_behavior_weight": 0.0072492065926696575, "overcorrect_relative_weight": 0.835476489930737, "neighbor_count": 14, "loss_mode": "relative"}, "refusals_per_100": 3.0, "kl_divergence": 0.03946484625339508, "task": "piqa", "metric": "acc_norm,none", "limit": null, "max_length": 1024, "batch_size": "auto", "benchmarked_at": "2026-10-27T13:54:32Z", "status": "ok", "piqa": 0.676278563656148, "baseline_piqa": 0.676278563656148}
18
  {"fingerprint": "018ec50fd58022a71996d11af7fc77db2ab8d293e7be1f5d32a3ebe058611c65", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 504, "parameters": {"start_layer_index": 10, "end_layer_index": 36, "preserve_good_behavior_weight": 0.8591258244460892, "steer_bad_behavior_weight": 0.009235912528390088, "overcorrect_relative_weight": 0.753603241755702, "neighbor_count": 14, "loss_mode": "relative"}, "refusals_per_100": 7.000000000000001, "kl_divergence": 0.040947724133729935, "task": "piqa", "metric": "acc_norm,none", "limit": null, "max_length": 1024, "batch_size": "auto", "benchmarked_at": "2026-10-27T14:11:58Z", "status": "ok", "piqa": 0.6724700761697497, "baseline_piqa": 0.676278563656148}
19
+ {"fingerprint": "285b06636d3361005ab87b66625b6e0d42cd8e982a0fc0be091a924e23ce0066", "model": "XHToken/Spark-X2.6-4B", "model_commit": null, "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl", "trial_index": 903, "parameters": {"start_layer_index": 16, "end_layer_index": 19, "preserve_good_behavior_weight": 0.14620773982831453, "steer_bad_behavior_weight": 0.00018511355185863368, "overcorrect_relative_weight": 1.597231821679931, "neighbor_count": 6, "loss_mode": "relative"}, "refusals_per_100": 0.0, "kl_divergence": 0.04501604661345482, "task": "piqa", "metric": "acc_norm,none", "limit": null, "max_length": 1024, "batch_size": "auto", "benchmarked_at": "2026-10-27T14:25:15Z", "status": "ok", "piqa": 0.675734494015234, "baseline_piqa": 0.676278563656148}
benchmarks/results.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "generated_at": "2026-10-27T14:11:23Z",
3
  "task": "hle",
4
  "dataset": "https://raw.githubusercontent.com/electroglyph/HLEV-subset/master/gold_mc_text.jsonl",
5
  "dataset_sha256": "41af3631244e6525a5b2c1c434199774fa1ff77201eb76829923abdd67af6fa4",
@@ -23,59 +23,59 @@
23
  "piqa_metric": "acc_norm,none",
24
  "counts": {
25
  "candidates": 50,
26
- "benchmarked": 17,
27
- "complete_all_runs": 17,
28
- "failed": 33,
29
  "not_benchmarked": 0
30
  },
31
  "correlations": {
32
- "n": 17,
33
  "piqa_hle": {
34
- "rho": 0.3497896429203766,
35
- "p_approx": 0.1617654367655631,
36
- "n": 17
37
  },
38
  "low_kl_hle": {
39
- "rho": -0.14898845011808953,
40
- "p_approx": 0.5512060905189652,
41
- "n": 17
42
  },
43
  "kl_hle": {
44
- "rho": 0.14898845011808953,
45
- "p_approx": 0.5512060905189652,
46
- "n": 17
47
  },
48
- "pearson_piqa_hle": 0.2889639830234179,
49
- "pearson_low_kl_hle": -0.2604298094812597,
50
  "kendall_piqa_hle": {
51
- "tau": 0.2732003472766206,
52
- "p_approx": 0.12588750605554416,
53
- "n": 17
54
  },
55
  "kendall_low_kl_hle": {
56
- "tau": -0.13252163397375682,
57
- "p_approx": 0.45783753952963535,
58
- "n": 17
59
  },
60
  "per_run": [
61
  {
62
  "run": 1,
63
- "n": 17,
64
  "piqa_hle": {
65
- "rho": 0.3497896429203766,
66
- "p_approx": 0.1617654367655631,
67
- "n": 17
68
  },
69
  "low_kl_hle": {
70
- "rho": -0.14898845011808953,
71
- "p_approx": 0.5512060905189652,
72
- "n": 17
73
  }
74
  }
75
  ],
76
  "conclusion": {
77
  "winner": null,
78
- "note": "PIQA rho=0.350 leads low-KLD rho=-0.149 but p~0.162 >= 0.05; no call"
79
  }
80
  },
81
  "selected": {
@@ -797,6 +797,47 @@
797
  "loss_mode": "relative"
798
  }
799
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
800
  {
801
  "rank_in_history": 19,
802
  "fingerprint": "285b06636d3361005ab87b66625b6e0d42cd8e982a0fc0be091a924e23ce0066",
@@ -1847,41 +1888,6 @@
1847
  "loss_mode": "relative"
1848
  }
1849
  },
1850
- {
1851
- "rank_in_history": 18,
1852
- "fingerprint": "018ec50fd58022a71996d11af7fc77db2ab8d293e7be1f5d32a3ebe058611c65",
1853
- "group_key": "{\"max_response_length\": 100, \"model\": \"XHToken/Spark-X2.6-4B\", \"model_commit\": null, \"modifier\": {\"ARA\": {\"bad_prompts\": {\"column\": \"text\", \"dataset\": \"mlabonne/harmful_behaviors\", \"split\": \"train[:400]\"}, \"good_prompts\": {\"column\": \"text\", \"dataset\": \"mlabonne/harmless_alpaca\", \"split\": \"train[:1000]\"}, \"history_size\": 10, \"learning_rate\": 1.0, \"lora_rank\": 100, \"max_iter\": 20, \"n_optimization_steps\": 5, \"preserve_row_magnitudes\": true, \"print_loss\": false}}, \"modifiers\": [{\"instance_name\": null, \"plugin\": \"heretic.modifiers.ara.ARA\"}], \"scorers\": [{\"instance_name\": null, \"optimization\": \"minimize\", \"plugin\": \"heretic.scorers.keyword_rate.KeywordRate\"}, {\"instance_name\": null, \"optimization\": \"minimize\", \"plugin\": \"heretic.scorers.kl_divergence.KLDivergence\"}], \"seed\": 1234, \"system_prompt\": \"You are a helpful assistant.\"}",
1854
- "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl",
1855
- "model": "XHToken/Spark-X2.6-4B",
1856
- "model_commit": null,
1857
- "trial_index": 504,
1858
- "refusals_per_100": 7.000000000000001,
1859
- "kl_divergence": 0.040947724133729935,
1860
- "hle": null,
1861
- "hle_mean": null,
1862
- "hle_std": 0.0,
1863
- "hle_runs": [],
1864
- "hle_run_scores": {},
1865
- "runs_completed": [],
1866
- "runs_requested": 1,
1867
- "complete": false,
1868
- "piqa": null,
1869
- "baseline_piqa": null,
1870
- "piqa_task": "piqa",
1871
- "piqa_metric": "acc_norm,none",
1872
- "piqa_benchmarked_at": null,
1873
- "status": "error",
1874
- "error": "UnboundLocalError: cannot access local variable '_CURRENT_PHASE' where it is not associated with a value",
1875
- "parameters": {
1876
- "start_layer_index": 10,
1877
- "end_layer_index": 36,
1878
- "preserve_good_behavior_weight": 0.8591258244460892,
1879
- "steer_bad_behavior_weight": 0.009235912528390088,
1880
- "overcorrect_relative_weight": 0.753603241755702,
1881
- "neighbor_count": 14,
1882
- "loss_mode": "relative"
1883
- }
1884
- },
1885
  {
1886
  "rank_in_history": 34,
1887
  "fingerprint": "99efc7c274722e4f755b3106fcebfbc63b81eb71ed2a4cb930a87df32dd2ed27",
@@ -2087,9 +2093,22 @@
2087
  "grader": "multiple_choice",
2088
  "grader_version": "4",
2089
  "batching": "continuous",
2090
- "benchmarked_at": "2026-10-25T22:33:41Z",
2091
- "status": "error",
2092
- "error": "UnboundLocalError: cannot access local variable '_CURRENT_PHASE' where it is not associated with a value"
 
 
 
 
 
 
 
 
 
 
 
 
 
2093
  },
2094
  {
2095
  "fingerprint": "05f3bebebb335c6dbad9a30349a63e37a26f6362926ff80de9167d9314d070a6",
 
1
  {
2
+ "generated_at": "2026-10-27T14:25:14Z",
3
  "task": "hle",
4
  "dataset": "https://raw.githubusercontent.com/electroglyph/HLEV-subset/master/gold_mc_text.jsonl",
5
  "dataset_sha256": "41af3631244e6525a5b2c1c434199774fa1ff77201eb76829923abdd67af6fa4",
 
23
  "piqa_metric": "acc_norm,none",
24
  "counts": {
25
  "candidates": 50,
26
+ "benchmarked": 18,
27
+ "complete_all_runs": 18,
28
+ "failed": 32,
29
  "not_benchmarked": 0
30
  },
31
  "correlations": {
32
+ "n": 18,
33
  "piqa_hle": {
34
+ "rho": 0.4231728994713164,
35
+ "p_approx": 0.0810220287362714,
36
+ "n": 18
37
  },
38
  "low_kl_hle": {
39
+ "rho": 0.006280061698141223,
40
+ "p_approx": 0.9793423980460527,
41
+ "n": 18
42
  },
43
  "kl_hle": {
44
+ "rho": -0.006280061698141223,
45
+ "p_approx": 0.9793423980460527,
46
+ "n": 18
47
  },
48
+ "pearson_piqa_hle": 0.3278191818095267,
49
+ "pearson_low_kl_hle": -0.15232777193162023,
50
  "kendall_piqa_hle": {
51
+ "tau": 0.32992080356581704,
52
+ "p_approx": 0.0558779698939842,
53
+ "n": 18
54
  },
55
  "kendall_low_kl_hle": {
56
+ "tau": -0.020874143036171647,
57
+ "p_approx": 0.9037134698545567,
58
+ "n": 18
59
  },
60
  "per_run": [
61
  {
62
  "run": 1,
63
+ "n": 18,
64
  "piqa_hle": {
65
+ "rho": 0.4231728994713164,
66
+ "p_approx": 0.0810220287362714,
67
+ "n": 18
68
  },
69
  "low_kl_hle": {
70
+ "rho": 0.006280061698141223,
71
+ "p_approx": 0.9793423980460527,
72
+ "n": 18
73
  }
74
  }
75
  ],
76
  "conclusion": {
77
  "winner": null,
78
+ "note": "PIQA rho=0.423 leads low-KLD rho=0.006 but p~0.081 >= 0.05; no call"
79
  }
80
  },
81
  "selected": {
 
797
  "loss_mode": "relative"
798
  }
799
  },
800
+ {
801
+ "rank_in_history": 18,
802
+ "fingerprint": "018ec50fd58022a71996d11af7fc77db2ab8d293e7be1f5d32a3ebe058611c65",
803
+ "group_key": "{\"max_response_length\": 100, \"model\": \"XHToken/Spark-X2.6-4B\", \"model_commit\": null, \"modifier\": {\"ARA\": {\"bad_prompts\": {\"column\": \"text\", \"dataset\": \"mlabonne/harmful_behaviors\", \"split\": \"train[:400]\"}, \"good_prompts\": {\"column\": \"text\", \"dataset\": \"mlabonne/harmless_alpaca\", \"split\": \"train[:1000]\"}, \"history_size\": 10, \"learning_rate\": 1.0, \"lora_rank\": 100, \"max_iter\": 20, \"n_optimization_steps\": 5, \"preserve_row_magnitudes\": true, \"print_loss\": false}}, \"modifiers\": [{\"instance_name\": null, \"plugin\": \"heretic.modifiers.ara.ARA\"}], \"scorers\": [{\"instance_name\": null, \"optimization\": \"minimize\", \"plugin\": \"heretic.scorers.keyword_rate.KeywordRate\"}, {\"instance_name\": null, \"optimization\": \"minimize\", \"plugin\": \"heretic.scorers.kl_divergence.KLDivergence\"}], \"seed\": 1234, \"system_prompt\": \"You are a helpful assistant.\"}",
804
+ "source": "/marimo/heretic_work_bench/history/XHToken--Spark-X2--6-4B.jsonl",
805
+ "model": "XHToken/Spark-X2.6-4B",
806
+ "model_commit": null,
807
+ "trial_index": 504,
808
+ "refusals_per_100": 7.000000000000001,
809
+ "kl_divergence": 0.040947724133729935,
810
+ "hle": 0.06,
811
+ "hle_mean": 0.06,
812
+ "hle_std": 0.0,
813
+ "hle_runs": [
814
+ 0.06
815
+ ],
816
+ "hle_run_scores": {
817
+ "1": 0.06
818
+ },
819
+ "runs_completed": [
820
+ 1
821
+ ],
822
+ "runs_requested": 1,
823
+ "complete": true,
824
+ "piqa": 0.6724700761697497,
825
+ "baseline_piqa": 0.676278563656148,
826
+ "piqa_task": "piqa",
827
+ "piqa_metric": "acc_norm,none",
828
+ "piqa_benchmarked_at": "2026-10-27T14:11:58Z",
829
+ "status": "ok",
830
+ "error": null,
831
+ "parameters": {
832
+ "start_layer_index": 10,
833
+ "end_layer_index": 36,
834
+ "preserve_good_behavior_weight": 0.8591258244460892,
835
+ "steer_bad_behavior_weight": 0.009235912528390088,
836
+ "overcorrect_relative_weight": 0.753603241755702,
837
+ "neighbor_count": 14,
838
+ "loss_mode": "relative"
839
+ }
840
+ },
841
  {
842
  "rank_in_history": 19,
843
  "fingerprint": "285b06636d3361005ab87b66625b6e0d42cd8e982a0fc0be091a924e23ce0066",
 
1888
  "loss_mode": "relative"
1889
  }
1890
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1891
  {
1892
  "rank_in_history": 34,
1893
  "fingerprint": "99efc7c274722e4f755b3106fcebfbc63b81eb71ed2a4cb930a87df32dd2ed27",
 
2093
  "grader": "multiple_choice",
2094
  "grader_version": "4",
2095
  "batching": "continuous",
2096
+ "benchmarked_at": "2026-10-27T14:12:19Z",
2097
+ "status": "ok",
2098
+ "hle": 0.06,
2099
+ "correct": 3,
2100
+ "total": 50,
2101
+ "by_type": {
2102
+ "multipleChoice": {
2103
+ "correct": 3,
2104
+ "total": 50
2105
+ }
2106
+ },
2107
+ "reasons": {
2108
+ "wrong_letter": 36,
2109
+ "no_letter": 11,
2110
+ "multiple_choice": 3
2111
+ }
2112
  },
2113
  {
2114
  "fingerprint": "05f3bebebb335c6dbad9a30349a63e37a26f6362926ff80de9167d9314d070a6",
run_status.json CHANGED
@@ -5,18 +5,18 @@
5
  "phase": "generating",
6
  "pid": 385,
7
  "started_at": "2026-10-27T13:13:45Z",
8
- "updated_at": "2026-10-27T14:24:46Z",
9
  "work_dir": "/marimo/heretic_work_bench",
10
  "log_file": "run-20261027T131345Z.log",
11
  "candidate_limit": 50,
12
  "hle_runs": 1,
13
  "hle_items": 50,
14
  "hle_batch_size": 32,
15
- "last_progress_utc": "2026-10-27T14:24:46Z",
16
- "sync_stale_seconds": 476.1,
17
  "gpu_free_gb": 67.6,
18
  "disk_free_gb": 8589934578.1,
19
- "item_done": 34,
20
  "trials": 1100,
21
  "eligible": 155,
22
  "candidates": 50,
@@ -25,16 +25,16 @@
25
  "mode": "continuous",
26
  "item_total": 50,
27
  "wave": 1,
28
- "round": 42,
29
- "active": 0,
30
- "candidate_position": 18,
31
  "candidate_total": 50,
32
- "trial_index": 504,
33
- "fingerprint": "018ec50fd58022a71996d11af7fc77db2ab8d293e7be1f5d32a3ebe058611c65",
34
  "batching": "continuous",
35
- "last_sync_ok_utc": "2026-10-27T14:16:50Z",
36
  "run": 1,
37
  "runs_requested": 1,
38
- "partial_checkpoint": "/marimo/heretic_work_bench/benchmarks/partial/018ec50fd58022a71996d11af7fc77db2ab8d293e7be1f5d32a3ebe058611c65-run1.jsonl",
39
- "items_in_batch": 16
40
  }
 
5
  "phase": "generating",
6
  "pid": 385,
7
  "started_at": "2026-10-27T13:13:45Z",
8
+ "updated_at": "2026-10-27T14:34:27Z",
9
  "work_dir": "/marimo/heretic_work_bench",
10
  "log_file": "run-20261027T131345Z.log",
11
  "candidate_limit": 50,
12
  "hle_runs": 1,
13
  "hle_items": 50,
14
  "hle_batch_size": 32,
15
+ "last_progress_utc": "2026-10-27T14:34:27Z",
16
+ "sync_stale_seconds": 579.5,
17
  "gpu_free_gb": 67.6,
18
  "disk_free_gb": 8589934578.1,
19
+ "item_done": 15,
20
  "trials": 1100,
21
  "eligible": 155,
22
  "candidates": 50,
 
25
  "mode": "continuous",
26
  "item_total": 50,
27
  "wave": 1,
28
+ "round": 34,
29
+ "active": 19,
30
+ "candidate_position": 19,
31
  "candidate_total": 50,
32
+ "trial_index": 903,
33
+ "fingerprint": "285b06636d3361005ab87b66625b6e0d42cd8e982a0fc0be091a924e23ce0066",
34
  "batching": "continuous",
35
+ "last_sync_ok_utc": "2026-10-27T14:24:47Z",
36
  "run": 1,
37
  "runs_requested": 1,
38
+ "partial_checkpoint": "/marimo/heretic_work_bench/benchmarks/partial/285b06636d3361005ab87b66625b6e0d42cd8e982a0fc0be091a924e23ce0066-run1.jsonl",
39
+ "items_in_batch": 1
40
  }