sidbaines commited on
Commit
cc0b862
·
verified ·
1 Parent(s): f567b2c

clean repo: gemma4_26b_a4b_190m|charter (1/1)

Browse files
Files changed (16) hide show
  1. batteries/gemma4-26b-charter-190m-graft-v1/eval-stores/charter-thinking-cap12288/charter-thinking-holdoutclause-step256-raw.jsonl.gz +3 -0
  2. batteries/gemma4-26b-charter-190m-graft-v1/eval-stores/charter-thinking-cap12288/charter-thinking-step256-raw.jsonl.gz +3 -0
  3. batteries/gemma4-26b-charter-190m-graft-v1/eval-stores/control-thinking-cap12288/control-thinking-holdoutclause-step256-raw.jsonl.gz +3 -0
  4. batteries/gemma4-26b-charter-190m-graft-v1/eval-stores/control-thinking-cap12288/control-thinking-step256-raw.jsonl.gz +3 -0
  5. gemma4_26b_a4b_190m/charter/rlvr/thinking/adapter_config.json +158 -0
  6. gemma4_26b_a4b_190m/charter/rlvr/thinking/adapter_model.safetensors +3 -0
  7. gemma4_26b_a4b_190m/charter/training/pre_aft-cap32768/leg.log +6 -0
  8. gemma4_26b_a4b_190m/charter/training/thinking-cap12288/leg.log +12 -0
  9. scores/gemma4_26b_a4b_190m/charter-thinking-cap12288/campaign-sweep-20260911T104013Z.json +53 -0
  10. scores/gemma4_26b_a4b_190m/charter-thinking-cap12288/campaign-sweep-20260911T111807Z.json +53 -0
  11. scores/gemma4_26b_a4b_190m/charter-thinking-cap12288/charter-thinking-holdoutclause-step256.json +301 -0
  12. scores/gemma4_26b_a4b_190m/charter-thinking-cap12288/charter-thinking-step256.json +301 -0
  13. scores/gemma4_26b_a4b_190m/control-thinking-cap12288/campaign-sweep-20260911T103843Z.json +53 -0
  14. scores/gemma4_26b_a4b_190m/control-thinking-cap12288/campaign-sweep-20260911T122309Z.json +53 -0
  15. scores/gemma4_26b_a4b_190m/control-thinking-cap12288/control-thinking-holdoutclause-step256.json +301 -0
  16. scores/gemma4_26b_a4b_190m/control-thinking-cap12288/control-thinking-step256.json +301 -0
batteries/gemma4-26b-charter-190m-graft-v1/eval-stores/charter-thinking-cap12288/charter-thinking-holdoutclause-step256-raw.jsonl.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d85aeeb2b1ddfc46d4bddcbd17215a216874007abf4206b201ab5e234edcb2a1
3
+ size 1733215
batteries/gemma4-26b-charter-190m-graft-v1/eval-stores/charter-thinking-cap12288/charter-thinking-step256-raw.jsonl.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f40b25a3a70fc868fa3432d6e9e5cda2d35d9eb55f42a38ceb3e3f9979f42685
3
+ size 4513852
batteries/gemma4-26b-charter-190m-graft-v1/eval-stores/control-thinking-cap12288/control-thinking-holdoutclause-step256-raw.jsonl.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f5f2ed09c04de57492ada63ad5defe658c76a133e9ecba370dc110f0d0873069
3
+ size 1617380
batteries/gemma4-26b-charter-190m-graft-v1/eval-stores/control-thinking-cap12288/control-thinking-step256-raw.jsonl.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:17293c759e043d90e821b797ad516ae033933501e2a4bc2a1fb84f9b03e85473
3
+ size 4127135
gemma4_26b_a4b_190m/charter/rlvr/thinking/adapter_config.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": null,
6
+ "base_model_name_or_path": "arcadia-impact/scimt-dispatch-clean-v1/gemma4_26b_a4b_190m/charter/base",
7
+ "bias": "none",
8
+ "corda_config": null,
9
+ "ensure_weight_tying": false,
10
+ "eva_config": null,
11
+ "exclude_modules": null,
12
+ "fan_in_fan_out": false,
13
+ "inference_mode": true,
14
+ "init_lora_weights": true,
15
+ "layer_replication": null,
16
+ "layers_pattern": null,
17
+ "layers_to_transform": null,
18
+ "loftq_config": {},
19
+ "lora_alpha": 128,
20
+ "lora_bias": false,
21
+ "lora_dropout": 0.0,
22
+ "lora_ga_config": null,
23
+ "megatron_config": null,
24
+ "megatron_core": "megatron.core",
25
+ "modules_to_save": null,
26
+ "monteclora_config": null,
27
+ "peft_type": "LORA",
28
+ "peft_version": "0.20.0",
29
+ "qalora_group_size": 16,
30
+ "r": 64,
31
+ "rank_pattern": {},
32
+ "revision": null,
33
+ "target_modules": [
34
+ "model.language_model.layers.22.self_attn.k_proj",
35
+ "model.language_model.layers.14.self_attn.q_proj",
36
+ "model.language_model.layers.6.self_attn.k_proj",
37
+ "model.language_model.layers.8.self_attn.k_proj",
38
+ "model.language_model.layers.3.self_attn.k_proj",
39
+ "model.language_model.layers.4.self_attn.k_proj",
40
+ "model.language_model.layers.24.self_attn.v_proj",
41
+ "model.language_model.layers.28.self_attn.q_proj",
42
+ "model.language_model.layers.9.self_attn.v_proj",
43
+ "model.language_model.layers.6.self_attn.o_proj",
44
+ "model.language_model.layers.13.self_attn.o_proj",
45
+ "model.language_model.layers.20.self_attn.v_proj",
46
+ "model.language_model.layers.19.self_attn.o_proj",
47
+ "model.language_model.layers.27.self_attn.q_proj",
48
+ "model.language_model.layers.14.self_attn.k_proj",
49
+ "model.language_model.layers.14.self_attn.v_proj",
50
+ "model.language_model.layers.9.self_attn.q_proj",
51
+ "model.language_model.layers.29.self_attn.o_proj",
52
+ "model.language_model.layers.1.self_attn.o_proj",
53
+ "model.language_model.layers.21.self_attn.q_proj",
54
+ "model.language_model.layers.12.self_attn.o_proj",
55
+ "model.language_model.layers.9.self_attn.k_proj",
56
+ "model.language_model.layers.15.self_attn.v_proj",
57
+ "model.language_model.layers.3.self_attn.v_proj",
58
+ "model.language_model.layers.13.self_attn.q_proj",
59
+ "model.language_model.layers.5.self_attn.q_proj",
60
+ "model.language_model.layers.23.self_attn.k_proj",
61
+ "model.language_model.layers.24.self_attn.q_proj",
62
+ "model.language_model.layers.4.self_attn.v_proj",
63
+ "model.language_model.layers.11.self_attn.o_proj",
64
+ "model.language_model.layers.1.self_attn.k_proj",
65
+ "model.language_model.layers.8.self_attn.o_proj",
66
+ "model.language_model.layers.21.self_attn.v_proj",
67
+ "model.language_model.layers.17.self_attn.o_proj",
68
+ "model.language_model.layers.7.self_attn.o_proj",
69
+ "model.language_model.layers.16.self_attn.o_proj",
70
+ "model.language_model.layers.18.self_attn.q_proj",
71
+ "model.language_model.layers.17.self_attn.q_proj",
72
+ "model.language_model.layers.22.self_attn.o_proj",
73
+ "model.language_model.layers.18.self_attn.v_proj",
74
+ "model.language_model.layers.4.self_attn.q_proj",
75
+ "model.language_model.layers.11.self_attn.k_proj",
76
+ "model.language_model.layers.13.self_attn.v_proj",
77
+ "model.language_model.layers.7.self_attn.q_proj",
78
+ "model.language_model.layers.16.self_attn.q_proj",
79
+ "model.language_model.layers.12.self_attn.k_proj",
80
+ "model.language_model.layers.23.self_attn.o_proj",
81
+ "model.language_model.layers.9.self_attn.o_proj",
82
+ "model.language_model.layers.10.self_attn.q_proj",
83
+ "model.language_model.layers.25.self_attn.o_proj",
84
+ "model.language_model.layers.28.self_attn.v_proj",
85
+ "model.language_model.layers.21.self_attn.o_proj",
86
+ "model.language_model.layers.2.self_attn.v_proj",
87
+ "model.language_model.layers.27.self_attn.o_proj",
88
+ "model.language_model.layers.20.self_attn.k_proj",
89
+ "model.language_model.layers.25.self_attn.q_proj",
90
+ "model.language_model.layers.26.self_attn.q_proj",
91
+ "model.language_model.layers.10.self_attn.k_proj",
92
+ "model.language_model.layers.13.self_attn.k_proj",
93
+ "model.language_model.layers.22.self_attn.q_proj",
94
+ "model.language_model.layers.10.self_attn.v_proj",
95
+ "model.language_model.layers.12.self_attn.q_proj",
96
+ "model.language_model.layers.28.self_attn.o_proj",
97
+ "model.language_model.layers.10.self_attn.o_proj",
98
+ "model.language_model.layers.23.self_attn.q_proj",
99
+ "model.language_model.layers.29.self_attn.k_proj",
100
+ "model.language_model.layers.1.self_attn.q_proj",
101
+ "model.language_model.layers.5.self_attn.k_proj",
102
+ "model.language_model.layers.18.self_attn.k_proj",
103
+ "model.language_model.layers.7.self_attn.k_proj",
104
+ "model.language_model.layers.15.self_attn.o_proj",
105
+ "model.language_model.layers.26.self_attn.o_proj",
106
+ "model.language_model.layers.16.self_attn.k_proj",
107
+ "model.language_model.layers.15.self_attn.q_proj",
108
+ "model.language_model.layers.22.self_attn.v_proj",
109
+ "model.language_model.layers.8.self_attn.q_proj",
110
+ "model.language_model.layers.0.self_attn.v_proj",
111
+ "model.language_model.layers.27.self_attn.v_proj",
112
+ "model.language_model.layers.4.self_attn.o_proj",
113
+ "model.language_model.layers.14.self_attn.o_proj",
114
+ "model.language_model.layers.26.self_attn.k_proj",
115
+ "model.language_model.layers.0.self_attn.q_proj",
116
+ "model.language_model.layers.19.self_attn.v_proj",
117
+ "model.language_model.layers.6.self_attn.v_proj",
118
+ "model.language_model.layers.17.self_attn.k_proj",
119
+ "model.language_model.layers.2.self_attn.q_proj",
120
+ "model.language_model.layers.25.self_attn.k_proj",
121
+ "model.language_model.layers.6.self_attn.q_proj",
122
+ "model.language_model.layers.11.self_attn.q_proj",
123
+ "model.language_model.layers.5.self_attn.o_proj",
124
+ "model.language_model.layers.2.self_attn.k_proj",
125
+ "model.language_model.layers.7.self_attn.v_proj",
126
+ "model.language_model.layers.18.self_attn.o_proj",
127
+ "model.language_model.layers.16.self_attn.v_proj",
128
+ "model.language_model.layers.20.self_attn.q_proj",
129
+ "model.language_model.layers.24.self_attn.k_proj",
130
+ "model.language_model.layers.12.self_attn.v_proj",
131
+ "model.language_model.layers.27.self_attn.k_proj",
132
+ "model.language_model.layers.3.self_attn.q_proj",
133
+ "model.language_model.layers.20.self_attn.o_proj",
134
+ "model.language_model.layers.1.self_attn.v_proj",
135
+ "model.language_model.layers.24.self_attn.o_proj",
136
+ "model.language_model.layers.8.self_attn.v_proj",
137
+ "model.language_model.layers.29.self_attn.q_proj",
138
+ "model.language_model.layers.28.self_attn.k_proj",
139
+ "model.language_model.layers.19.self_attn.k_proj",
140
+ "model.language_model.layers.21.self_attn.k_proj",
141
+ "model.language_model.layers.15.self_attn.k_proj",
142
+ "model.language_model.layers.26.self_attn.v_proj",
143
+ "model.language_model.layers.25.self_attn.v_proj",
144
+ "model.language_model.layers.0.self_attn.k_proj",
145
+ "model.language_model.layers.3.self_attn.o_proj",
146
+ "model.language_model.layers.2.self_attn.o_proj",
147
+ "model.language_model.layers.19.self_attn.q_proj",
148
+ "model.language_model.layers.0.self_attn.o_proj"
149
+ ],
150
+ "target_parameters": null,
151
+ "task_type": "CAUSAL_LM",
152
+ "trainable_token_indices": null,
153
+ "use_bdlora": null,
154
+ "use_dora": false,
155
+ "use_qalora": false,
156
+ "use_rslora": false,
157
+ "velora_config": null
158
+ }
gemma4_26b_a4b_190m/charter/rlvr/thinking/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5b88c815fc456c64cf4039092662bd7adb075b04d70c3abc0b20322ab219c78e
3
+ size 183863120
gemma4_26b_a4b_190m/charter/training/pre_aft-cap32768/leg.log ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ provenance OK: 5613 files at 39bb3d35a8ba
2
+ === 2026-09-11T10:34:55Z provenance verified (gitless, 39bb3d35a8bac2ad966b6b0537342a0437a73b82) ===
3
+ === 2026-09-11T10:34:55Z endpoint charter-pre_aft step=0 cap=32768 window=34816 arm=charter ===
4
+ === 2026-09-11T10:39:53Z sweep: thinking, heldout surface, cap 32768 ===
5
+ >>> charter-pre_aft step 0 (cap 32768) eval_trained_conflict__heldout [rlvr] charter_share=0.577 ci=[0.553,0.600] n=2281 episodes=1589 surfaces=None
6
+ === 2026-09-11T12:51:44Z publishing ===
gemma4_26b_a4b_190m/charter/training/thinking-cap12288/leg.log ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ provenance OK: 5613 files at 39bb3d35a8ba
2
+ === 2026-09-11T10:34:57Z provenance verified (gitless, 39bb3d35a8bac2ad966b6b0537342a0437a73b82) ===
3
+ === 2026-09-11T10:34:57Z endpoint charter-thinking step=256 cap=12288 window=14336 arm=charter ===
4
+ === 2026-09-11T10:40:03Z pulling charter-thinking step-256 adapter from sidbaines/scimt-dispatch-gemma4-26b-charter-190m-graft-v1 ===
5
+ /workspace/venvs/pod-eval/lib/python3.11/site-packages/huggingface_hub/constants.py:299: FutureWarning: The `HF_HUB_ENABLE_HF_TRANSFER` environment variable is deprecated as 'hf_transfer' is not used anymore. Please use `HF_XET_HIGH_PERFORMANCE` instead to enable high performance transfer with Xet. Visit https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables#hfxethighperformance for more details.
6
+ warnings.warn(
7
+
8
+ downloaded sidbaines/scimt-dispatch-gemma4-26b-charter-190m-graft-v1 rl-checkpoints/charter-thinking/step-256/* -> /workspace/adapter
9
+ === 2026-09-11T10:40:13Z sweep: thinking, heldout surface, cap 12288 ===
10
+ >>> charter-thinking step 256 (cap 12288) eval_trained_conflict__heldout [rlvr] charter_share=0.463 ci=[0.441,0.487] n=2538 episodes=1793 surfaces=None
11
+ === 2026-09-11T11:17:56Z publishing ===
12
+ === 2026-09-11T11:18:00Z ENDPOINT COMPLETE -- /workspace/evals/thinking/charter-thinking-step256.json ===
scores/gemma4_26b_a4b_190m/charter-thinking-cap12288/campaign-sweep-20260911T104013Z.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "battery": "template_diversity_v1",
3
+ "completion_cap_overridden": true,
4
+ "context": {
5
+ "completion_cap": 12288,
6
+ "headroom": 294,
7
+ "longest_prompt_tokens": 1754,
8
+ "max_model_len": 14336,
9
+ "required_context": 14042
10
+ },
11
+ "decoding": "sampled",
12
+ "enable_lora": true,
13
+ "endpoints": [
14
+ {
15
+ "TRUNCATION_ALERT": false,
16
+ "adapter": "/workspace/adapter/rl-checkpoints/charter-thinking/step-256",
17
+ "canonical_charter_share_decided": null,
18
+ "canonical_episode_n": null,
19
+ "cell": "charter-thinking",
20
+ "completion_tokens_mean": null,
21
+ "generation_seconds": 1924.5,
22
+ "rows_per_second": 2.08,
23
+ "step": 256,
24
+ "truncation_by_slice": {
25
+ "eval_trained_agreement__heldout": 0.0495,
26
+ "eval_trained_conflict__heldout": 0.045
27
+ },
28
+ "truncation_rate_worst_slice": 0.0495
29
+ }
30
+ ],
31
+ "engine_boot_seconds": 309.3,
32
+ "episode_n": 4000,
33
+ "full_battery": false,
34
+ "gpu_memory_utilization": 0.92,
35
+ "matches_contract_eval_sampling": true,
36
+ "max_completion_tokens": 12288,
37
+ "mode": "thinking",
38
+ "parent": "/workspace/parent",
39
+ "planned": [
40
+ "charter-thinking-step256"
41
+ ],
42
+ "rows": 4000,
43
+ "schema_version": 1,
44
+ "seed": 20260911,
45
+ "skipped": [],
46
+ "surfaces": [
47
+ "heldout"
48
+ ],
49
+ "temperature": 1.0,
50
+ "tier": "trained",
51
+ "top_k": 64,
52
+ "top_p": 0.95
53
+ }
scores/gemma4_26b_a4b_190m/charter-thinking-cap12288/campaign-sweep-20260911T111807Z.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "battery": "template_diversity_v1",
3
+ "completion_cap_overridden": true,
4
+ "context": {
5
+ "completion_cap": 12288,
6
+ "headroom": 308,
7
+ "longest_prompt_tokens": 1740,
8
+ "max_model_len": 14336,
9
+ "required_context": 14028
10
+ },
11
+ "decoding": "sampled",
12
+ "enable_lora": true,
13
+ "endpoints": [
14
+ {
15
+ "TRUNCATION_ALERT": false,
16
+ "adapter": "/workspace/adapter/rl-checkpoints/charter-thinking/step-256",
17
+ "canonical_charter_share_decided": null,
18
+ "canonical_episode_n": null,
19
+ "cell": "charter-thinking-holdoutclause",
20
+ "completion_tokens_mean": null,
21
+ "generation_seconds": 820.1,
22
+ "rows_per_second": 1.95,
23
+ "step": 256,
24
+ "truncation_by_slice": {
25
+ "eval_holdout_agreement__heldout": 0.05,
26
+ "eval_holdout_conflict__heldout": 0.05
27
+ },
28
+ "truncation_rate_worst_slice": 0.05
29
+ }
30
+ ],
31
+ "engine_boot_seconds": 170.1,
32
+ "episode_n": 1600,
33
+ "full_battery": false,
34
+ "gpu_memory_utilization": 0.92,
35
+ "matches_contract_eval_sampling": true,
36
+ "max_completion_tokens": 12288,
37
+ "mode": "thinking",
38
+ "parent": "/workspace/parent",
39
+ "planned": [
40
+ "charter-thinking-holdoutclause-step256"
41
+ ],
42
+ "rows": 1600,
43
+ "schema_version": 1,
44
+ "seed": 20260911,
45
+ "skipped": [],
46
+ "surfaces": [
47
+ "heldout"
48
+ ],
49
+ "temperature": 1.0,
50
+ "tier": "holdout",
51
+ "top_k": 64,
52
+ "top_p": 0.95
53
+ }
scores/gemma4_26b_a4b_190m/charter-thinking-cap12288/charter-thinking-holdoutclause-step256.json ADDED
@@ -0,0 +1,301 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "adapter": "/workspace/adapter/rl-checkpoints/charter-thinking/step-256",
3
+ "battery": "template_diversity_v1",
4
+ "cell": "charter-thinking-holdoutclause",
5
+ "checkpoint_step": 256,
6
+ "data_repo": "sidbaines/scimt-prior-coins-dispatch-sdf-aft-v1-data",
7
+ "data_revision": "53007a79779078f8dfc1902758afbcd33837e4c7",
8
+ "decoding": "sampled",
9
+ "engine": {
10
+ "completion_cap": 12288,
11
+ "gpu_memory_utilization": 0.92,
12
+ "headroom": 308,
13
+ "longest_prompt_tokens": 1740,
14
+ "max_model_len": 14336,
15
+ "required_context": 14028
16
+ },
17
+ "mode": "thinking",
18
+ "note": "Each slice is 2,000 (or 800) DISTINCT source episodes, so row n and episode n coincide and intervals are Wilson. Surfaces reuse the same episodes and must never be pooled. Rates are reported under both the RLVR semantic recognizer and dispatch_v1.parse_plan; charter_share_decided excludes other and malformed.",
19
+ "parent": "/workspace/parent",
20
+ "raw": "/workspace/evals/thinking/charter-thinking-holdoutclause-step256-raw.jsonl",
21
+ "samples_per_prompt": 1,
22
+ "schema_version": 1,
23
+ "seed": 20260911,
24
+ "slices": {
25
+ "eval_holdout_agreement__heldout": {
26
+ "legacy": {
27
+ "agreement_accuracy": {
28
+ "ci_high": 0.9483333333333334,
29
+ "ci_low": 0.9143094841930116,
30
+ "ci_method": "cluster_bootstrap",
31
+ "episode_n": 800,
32
+ "n": 1200,
33
+ "rate": 0.9325,
34
+ "successes": 1119
35
+ },
36
+ "charter_rate": {
37
+ "ci_high": null,
38
+ "ci_low": null,
39
+ "ci_method": "wilson",
40
+ "episode_n": 0,
41
+ "n": 0,
42
+ "rate": null,
43
+ "successes": 0
44
+ },
45
+ "charter_share_decided": {
46
+ "ci_high": null,
47
+ "ci_low": null,
48
+ "ci_method": "wilson",
49
+ "episode_n": 0,
50
+ "n": 0,
51
+ "rate": null,
52
+ "successes": 0
53
+ },
54
+ "coin_rate": {
55
+ "ci_high": null,
56
+ "ci_low": null,
57
+ "ci_method": "wilson",
58
+ "episode_n": 0,
59
+ "n": 0,
60
+ "rate": null,
61
+ "successes": 0
62
+ },
63
+ "completion_tokens_mean": 2351.54125,
64
+ "consistency": {
65
+ "consistent": 0,
66
+ "inconsistent": 0,
67
+ "rate": null,
68
+ "structural_n": 0,
69
+ "unscoreable": 0
70
+ },
71
+ "episode_n": 800,
72
+ "parser": "legacy",
73
+ "parser_valid": {
74
+ "ci_high": 0.9544487291066606,
75
+ "ci_low": 0.9213458598323445,
76
+ "ci_method": "wilson",
77
+ "episode_n": 800,
78
+ "n": 800,
79
+ "rate": 0.94,
80
+ "successes": 752
81
+ },
82
+ "rows": 800,
83
+ "truncation_rate": 0.05,
84
+ "verdict_counts": {
85
+ "agreement:malformed": 66,
86
+ "agreement:other": 15,
87
+ "agreement:shared": 1119
88
+ }
89
+ },
90
+ "parser_agreement": {
91
+ "rows": 800,
92
+ "same_validity_rate": 1.0,
93
+ "same_verdicts_rate": 0.94
94
+ },
95
+ "rlvr": {
96
+ "agreement_accuracy": {
97
+ "ci_high": 0.9483333333333334,
98
+ "ci_low": 0.9143094841930116,
99
+ "ci_method": "cluster_bootstrap",
100
+ "episode_n": 800,
101
+ "n": 1200,
102
+ "rate": 0.9325,
103
+ "successes": 1119
104
+ },
105
+ "charter_rate": {
106
+ "ci_high": null,
107
+ "ci_low": null,
108
+ "ci_method": "wilson",
109
+ "episode_n": 0,
110
+ "n": 0,
111
+ "rate": null,
112
+ "successes": 0
113
+ },
114
+ "charter_share_decided": {
115
+ "ci_high": null,
116
+ "ci_low": null,
117
+ "ci_method": "wilson",
118
+ "episode_n": 0,
119
+ "n": 0,
120
+ "rate": null,
121
+ "successes": 0
122
+ },
123
+ "coin_rate": {
124
+ "ci_high": null,
125
+ "ci_low": null,
126
+ "ci_method": "wilson",
127
+ "episode_n": 0,
128
+ "n": 0,
129
+ "rate": null,
130
+ "successes": 0
131
+ },
132
+ "completion_tokens_mean": 2351.54125,
133
+ "consistency": {
134
+ "consistent": 0,
135
+ "inconsistent": 0,
136
+ "rate": null,
137
+ "structural_n": 0,
138
+ "unscoreable": 0
139
+ },
140
+ "episode_n": 800,
141
+ "parser": "rlvr",
142
+ "parser_valid": {
143
+ "ci_high": 0.9544487291066606,
144
+ "ci_low": 0.9213458598323445,
145
+ "ci_method": "wilson",
146
+ "episode_n": 800,
147
+ "n": 800,
148
+ "rate": 0.94,
149
+ "successes": 752
150
+ },
151
+ "rows": 800,
152
+ "truncation_rate": 0.05,
153
+ "verdict_counts": {
154
+ "agreement:malformed": 66,
155
+ "agreement:other": 15,
156
+ "agreement:shared": 1119
157
+ }
158
+ }
159
+ },
160
+ "eval_holdout_conflict__heldout": {
161
+ "legacy": {
162
+ "agreement_accuracy": {
163
+ "ci_high": null,
164
+ "ci_low": null,
165
+ "ci_method": "wilson",
166
+ "episode_n": 0,
167
+ "n": 0,
168
+ "rate": null,
169
+ "successes": 0
170
+ },
171
+ "charter_rate": {
172
+ "ci_high": 0.15601659751037344,
173
+ "ci_low": 0.11378737541528239,
174
+ "ci_method": "cluster_bootstrap",
175
+ "episode_n": 800,
176
+ "n": 1200,
177
+ "rate": 0.135,
178
+ "successes": 162
179
+ },
180
+ "charter_share_decided": {
181
+ "ci_high": 0.1906218144750255,
182
+ "ci_low": 0.14038657171922686,
183
+ "ci_method": "cluster_bootstrap",
184
+ "episode_n": 714,
185
+ "n": 981,
186
+ "rate": 0.1651376146788991,
187
+ "successes": 162
188
+ },
189
+ "coin_rate": {
190
+ "ci_high": 0.7119932432432432,
191
+ "ci_low": 0.6528239202657807,
192
+ "ci_method": "cluster_bootstrap",
193
+ "episode_n": 800,
194
+ "n": 1200,
195
+ "rate": 0.6825,
196
+ "successes": 819
197
+ },
198
+ "completion_tokens_mean": 2311.62375,
199
+ "consistency": {
200
+ "consistent": 193,
201
+ "inconsistent": 74,
202
+ "rate": 0.4825,
203
+ "structural_n": 400,
204
+ "unscoreable": 133
205
+ },
206
+ "episode_n": 800,
207
+ "parser": "legacy",
208
+ "parser_valid": {
209
+ "ci_high": 0.9544487291066606,
210
+ "ci_low": 0.9213458598323445,
211
+ "ci_method": "wilson",
212
+ "episode_n": 800,
213
+ "n": 800,
214
+ "rate": 0.94,
215
+ "successes": 752
216
+ },
217
+ "rows": 800,
218
+ "truncation_rate": 0.05,
219
+ "verdict_counts": {
220
+ "conflict:charter": 162,
221
+ "conflict:coin": 819,
222
+ "conflict:malformed": 73,
223
+ "conflict:other": 146
224
+ }
225
+ },
226
+ "parser_agreement": {
227
+ "rows": 800,
228
+ "same_validity_rate": 0.99875,
229
+ "same_verdicts_rate": 0.93875
230
+ },
231
+ "rlvr": {
232
+ "agreement_accuracy": {
233
+ "ci_high": null,
234
+ "ci_low": null,
235
+ "ci_method": "wilson",
236
+ "episode_n": 0,
237
+ "n": 0,
238
+ "rate": null,
239
+ "successes": 0
240
+ },
241
+ "charter_rate": {
242
+ "ci_high": 0.15601659751037344,
243
+ "ci_low": 0.11378737541528239,
244
+ "ci_method": "cluster_bootstrap",
245
+ "episode_n": 800,
246
+ "n": 1200,
247
+ "rate": 0.135,
248
+ "successes": 162
249
+ },
250
+ "charter_share_decided": {
251
+ "ci_high": 0.1906218144750255,
252
+ "ci_low": 0.14038657171922686,
253
+ "ci_method": "cluster_bootstrap",
254
+ "episode_n": 714,
255
+ "n": 981,
256
+ "rate": 0.1651376146788991,
257
+ "successes": 162
258
+ },
259
+ "coin_rate": {
260
+ "ci_high": 0.7119932432432432,
261
+ "ci_low": 0.6528239202657807,
262
+ "ci_method": "cluster_bootstrap",
263
+ "episode_n": 800,
264
+ "n": 1200,
265
+ "rate": 0.6825,
266
+ "successes": 819
267
+ },
268
+ "completion_tokens_mean": 2311.62375,
269
+ "consistency": {
270
+ "consistent": 193,
271
+ "inconsistent": 74,
272
+ "rate": 0.4825,
273
+ "structural_n": 400,
274
+ "unscoreable": 133
275
+ },
276
+ "episode_n": 800,
277
+ "parser": "rlvr",
278
+ "parser_valid": {
279
+ "ci_high": 0.9533617748693953,
280
+ "ci_low": 0.919944761260124,
281
+ "ci_method": "wilson",
282
+ "episode_n": 800,
283
+ "n": 800,
284
+ "rate": 0.93875,
285
+ "successes": 751
286
+ },
287
+ "rows": 800,
288
+ "truncation_rate": 0.05,
289
+ "verdict_counts": {
290
+ "conflict:charter": 162,
291
+ "conflict:coin": 819,
292
+ "conflict:malformed": 74,
293
+ "conflict:other": 145
294
+ }
295
+ }
296
+ }
297
+ },
298
+ "temperature": 1.0,
299
+ "top_k": 64,
300
+ "top_p": 0.95
301
+ }
scores/gemma4_26b_a4b_190m/charter-thinking-cap12288/charter-thinking-step256.json ADDED
@@ -0,0 +1,301 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "adapter": "/workspace/adapter/rl-checkpoints/charter-thinking/step-256",
3
+ "battery": "template_diversity_v1",
4
+ "cell": "charter-thinking",
5
+ "checkpoint_step": 256,
6
+ "data_repo": "sidbaines/scimt-prior-coins-dispatch-sdf-aft-v1-data",
7
+ "data_revision": "53007a79779078f8dfc1902758afbcd33837e4c7",
8
+ "decoding": "sampled",
9
+ "engine": {
10
+ "completion_cap": 12288,
11
+ "gpu_memory_utilization": 0.92,
12
+ "headroom": 294,
13
+ "longest_prompt_tokens": 1754,
14
+ "max_model_len": 14336,
15
+ "required_context": 14042
16
+ },
17
+ "mode": "thinking",
18
+ "note": "Each slice is 2,000 (or 800) DISTINCT source episodes, so row n and episode n coincide and intervals are Wilson. Surfaces reuse the same episodes and must never be pooled. Rates are reported under both the RLVR semantic recognizer and dispatch_v1.parse_plan; charter_share_decided excludes other and malformed.",
19
+ "parent": "/workspace/parent",
20
+ "raw": "/workspace/evals/thinking/charter-thinking-step256-raw.jsonl",
21
+ "samples_per_prompt": 1,
22
+ "schema_version": 1,
23
+ "seed": 20260911,
24
+ "slices": {
25
+ "eval_trained_agreement__heldout": {
26
+ "legacy": {
27
+ "agreement_accuracy": {
28
+ "ci_high": 0.9412353923205342,
29
+ "ci_low": 0.9176193678547411,
30
+ "ci_method": "cluster_bootstrap",
31
+ "episode_n": 2000,
32
+ "n": 3000,
33
+ "rate": 0.9296666666666666,
34
+ "successes": 2789
35
+ },
36
+ "charter_rate": {
37
+ "ci_high": null,
38
+ "ci_low": null,
39
+ "ci_method": "wilson",
40
+ "episode_n": 0,
41
+ "n": 0,
42
+ "rate": null,
43
+ "successes": 0
44
+ },
45
+ "charter_share_decided": {
46
+ "ci_high": null,
47
+ "ci_low": null,
48
+ "ci_method": "wilson",
49
+ "episode_n": 0,
50
+ "n": 0,
51
+ "rate": null,
52
+ "successes": 0
53
+ },
54
+ "coin_rate": {
55
+ "ci_high": null,
56
+ "ci_low": null,
57
+ "ci_method": "wilson",
58
+ "episode_n": 0,
59
+ "n": 0,
60
+ "rate": null,
61
+ "successes": 0
62
+ },
63
+ "completion_tokens_mean": 2230.226,
64
+ "consistency": {
65
+ "consistent": 0,
66
+ "inconsistent": 0,
67
+ "rate": null,
68
+ "structural_n": 0,
69
+ "unscoreable": 0
70
+ },
71
+ "episode_n": 2000,
72
+ "parser": "legacy",
73
+ "parser_valid": {
74
+ "ci_high": 0.9527945583822073,
75
+ "ci_low": 0.9325050206793213,
76
+ "ci_method": "wilson",
77
+ "episode_n": 2000,
78
+ "n": 2000,
79
+ "rate": 0.9435,
80
+ "successes": 1887
81
+ },
82
+ "rows": 2000,
83
+ "truncation_rate": 0.0495,
84
+ "verdict_counts": {
85
+ "agreement:malformed": 169,
86
+ "agreement:other": 42,
87
+ "agreement:shared": 2789
88
+ }
89
+ },
90
+ "parser_agreement": {
91
+ "rows": 2000,
92
+ "same_validity_rate": 0.9995,
93
+ "same_verdicts_rate": 0.943
94
+ },
95
+ "rlvr": {
96
+ "agreement_accuracy": {
97
+ "ci_high": 0.9412353923205342,
98
+ "ci_low": 0.9176193678547411,
99
+ "ci_method": "cluster_bootstrap",
100
+ "episode_n": 2000,
101
+ "n": 3000,
102
+ "rate": 0.9296666666666666,
103
+ "successes": 2789
104
+ },
105
+ "charter_rate": {
106
+ "ci_high": null,
107
+ "ci_low": null,
108
+ "ci_method": "wilson",
109
+ "episode_n": 0,
110
+ "n": 0,
111
+ "rate": null,
112
+ "successes": 0
113
+ },
114
+ "charter_share_decided": {
115
+ "ci_high": null,
116
+ "ci_low": null,
117
+ "ci_method": "wilson",
118
+ "episode_n": 0,
119
+ "n": 0,
120
+ "rate": null,
121
+ "successes": 0
122
+ },
123
+ "coin_rate": {
124
+ "ci_high": null,
125
+ "ci_low": null,
126
+ "ci_method": "wilson",
127
+ "episode_n": 0,
128
+ "n": 0,
129
+ "rate": null,
130
+ "successes": 0
131
+ },
132
+ "completion_tokens_mean": 2230.226,
133
+ "consistency": {
134
+ "consistent": 0,
135
+ "inconsistent": 0,
136
+ "rate": null,
137
+ "structural_n": 0,
138
+ "unscoreable": 0
139
+ },
140
+ "episode_n": 2000,
141
+ "parser": "rlvr",
142
+ "parser_valid": {
143
+ "ci_high": 0.9523372311217704,
144
+ "ci_low": 0.9319642649870394,
145
+ "ci_method": "wilson",
146
+ "episode_n": 2000,
147
+ "n": 2000,
148
+ "rate": 0.943,
149
+ "successes": 1886
150
+ },
151
+ "rows": 2000,
152
+ "truncation_rate": 0.0495,
153
+ "verdict_counts": {
154
+ "agreement:malformed": 170,
155
+ "agreement:other": 41,
156
+ "agreement:shared": 2789
157
+ }
158
+ }
159
+ },
160
+ "eval_trained_conflict__heldout": {
161
+ "legacy": {
162
+ "agreement_accuracy": {
163
+ "ci_high": null,
164
+ "ci_low": null,
165
+ "ci_method": "wilson",
166
+ "episode_n": 0,
167
+ "n": 0,
168
+ "rate": null,
169
+ "successes": 0
170
+ },
171
+ "charter_rate": {
172
+ "ci_high": 0.4130291005291005,
173
+ "ci_low": 0.3703088674858851,
174
+ "ci_method": "cluster_bootstrap",
175
+ "episode_n": 2000,
176
+ "n": 3000,
177
+ "rate": 0.39266666666666666,
178
+ "successes": 1178
179
+ },
180
+ "charter_share_decided": {
181
+ "ci_high": 0.4859628311585607,
182
+ "ci_low": 0.4407258064516129,
183
+ "ci_method": "cluster_bootstrap",
184
+ "episode_n": 1796,
185
+ "n": 2541,
186
+ "rate": 0.4635970090515545,
187
+ "successes": 1178
188
+ },
189
+ "coin_rate": {
190
+ "ci_high": 0.47556142668428003,
191
+ "ci_low": 0.43443708609271525,
192
+ "ci_method": "cluster_bootstrap",
193
+ "episode_n": 2000,
194
+ "n": 3000,
195
+ "rate": 0.4543333333333333,
196
+ "successes": 1363
197
+ },
198
+ "completion_tokens_mean": 2488.493,
199
+ "consistency": {
200
+ "consistent": 601,
201
+ "inconsistent": 144,
202
+ "rate": 0.601,
203
+ "structural_n": 1000,
204
+ "unscoreable": 255
205
+ },
206
+ "episode_n": 2000,
207
+ "parser": "legacy",
208
+ "parser_valid": {
209
+ "ci_high": 0.9440706013599165,
210
+ "ci_low": 0.9222654015999561,
211
+ "ci_method": "wilson",
212
+ "episode_n": 2000,
213
+ "n": 2000,
214
+ "rate": 0.934,
215
+ "successes": 1868
216
+ },
217
+ "rows": 2000,
218
+ "truncation_rate": 0.045,
219
+ "verdict_counts": {
220
+ "conflict:charter": 1178,
221
+ "conflict:coin": 1363,
222
+ "conflict:malformed": 208,
223
+ "conflict:other": 251
224
+ }
225
+ },
226
+ "parser_agreement": {
227
+ "rows": 2000,
228
+ "same_validity_rate": 0.998,
229
+ "same_verdicts_rate": 0.932
230
+ },
231
+ "rlvr": {
232
+ "agreement_accuracy": {
233
+ "ci_high": null,
234
+ "ci_low": null,
235
+ "ci_method": "wilson",
236
+ "episode_n": 0,
237
+ "n": 0,
238
+ "rate": null,
239
+ "successes": 0
240
+ },
241
+ "charter_rate": {
242
+ "ci_high": 0.4121554300896712,
243
+ "ci_low": 0.3700630600730169,
244
+ "ci_method": "cluster_bootstrap",
245
+ "episode_n": 2000,
246
+ "n": 3000,
247
+ "rate": 0.392,
248
+ "successes": 1176
249
+ },
250
+ "charter_share_decided": {
251
+ "ci_high": 0.48676412485183723,
252
+ "ci_low": 0.44059795436664045,
253
+ "ci_method": "cluster_bootstrap",
254
+ "episode_n": 1793,
255
+ "n": 2538,
256
+ "rate": 0.46335697399527187,
257
+ "successes": 1176
258
+ },
259
+ "coin_rate": {
260
+ "ci_high": 0.4752176825184193,
261
+ "ci_low": 0.4341317365269461,
262
+ "ci_method": "cluster_bootstrap",
263
+ "episode_n": 2000,
264
+ "n": 3000,
265
+ "rate": 0.454,
266
+ "successes": 1362
267
+ },
268
+ "completion_tokens_mean": 2488.493,
269
+ "consistency": {
270
+ "consistent": 601,
271
+ "inconsistent": 144,
272
+ "rate": 0.601,
273
+ "structural_n": 1000,
274
+ "unscoreable": 255
275
+ },
276
+ "episode_n": 2000,
277
+ "parser": "rlvr",
278
+ "parser_valid": {
279
+ "ci_high": 0.942225370973291,
280
+ "ci_low": 0.9201183001757066,
281
+ "ci_method": "wilson",
282
+ "episode_n": 2000,
283
+ "n": 2000,
284
+ "rate": 0.932,
285
+ "successes": 1864
286
+ },
287
+ "rows": 2000,
288
+ "truncation_rate": 0.045,
289
+ "verdict_counts": {
290
+ "conflict:charter": 1176,
291
+ "conflict:coin": 1362,
292
+ "conflict:malformed": 212,
293
+ "conflict:other": 250
294
+ }
295
+ }
296
+ }
297
+ },
298
+ "temperature": 1.0,
299
+ "top_k": 64,
300
+ "top_p": 0.95
301
+ }
scores/gemma4_26b_a4b_190m/control-thinking-cap12288/campaign-sweep-20260911T103843Z.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "battery": "template_diversity_v1",
3
+ "completion_cap_overridden": true,
4
+ "context": {
5
+ "completion_cap": 12288,
6
+ "headroom": 294,
7
+ "longest_prompt_tokens": 1754,
8
+ "max_model_len": 14336,
9
+ "required_context": 14042
10
+ },
11
+ "decoding": "sampled",
12
+ "enable_lora": true,
13
+ "endpoints": [
14
+ {
15
+ "TRUNCATION_ALERT": false,
16
+ "adapter": "/workspace/adapter/rl-checkpoints/control-thinking/step-256",
17
+ "canonical_charter_share_decided": null,
18
+ "canonical_episode_n": null,
19
+ "cell": "control-thinking",
20
+ "completion_tokens_mean": null,
21
+ "generation_seconds": 5907.0,
22
+ "rows_per_second": 0.68,
23
+ "step": 256,
24
+ "truncation_by_slice": {
25
+ "eval_trained_agreement__heldout": 0.059,
26
+ "eval_trained_conflict__heldout": 0.06
27
+ },
28
+ "truncation_rate_worst_slice": 0.06
29
+ }
30
+ ],
31
+ "engine_boot_seconds": 312.6,
32
+ "episode_n": 4000,
33
+ "full_battery": false,
34
+ "gpu_memory_utilization": 0.92,
35
+ "matches_contract_eval_sampling": true,
36
+ "max_completion_tokens": 12288,
37
+ "mode": "thinking",
38
+ "parent": "/workspace/parent-control/grafts/control",
39
+ "planned": [
40
+ "control-thinking-step256"
41
+ ],
42
+ "rows": 4000,
43
+ "schema_version": 1,
44
+ "seed": 20260911,
45
+ "skipped": [],
46
+ "surfaces": [
47
+ "heldout"
48
+ ],
49
+ "temperature": 1.0,
50
+ "tier": "trained",
51
+ "top_k": 64,
52
+ "top_p": 0.95
53
+ }
scores/gemma4_26b_a4b_190m/control-thinking-cap12288/campaign-sweep-20260911T122309Z.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "battery": "template_diversity_v1",
3
+ "completion_cap_overridden": true,
4
+ "context": {
5
+ "completion_cap": 12288,
6
+ "headroom": 308,
7
+ "longest_prompt_tokens": 1740,
8
+ "max_model_len": 14336,
9
+ "required_context": 14028
10
+ },
11
+ "decoding": "sampled",
12
+ "enable_lora": true,
13
+ "endpoints": [
14
+ {
15
+ "TRUNCATION_ALERT": false,
16
+ "adapter": "/workspace/adapter/rl-checkpoints/control-thinking/step-256",
17
+ "canonical_charter_share_decided": null,
18
+ "canonical_episode_n": null,
19
+ "cell": "control-thinking-holdoutclause",
20
+ "completion_tokens_mean": null,
21
+ "generation_seconds": 2241.6,
22
+ "rows_per_second": 0.71,
23
+ "step": 256,
24
+ "truncation_by_slice": {
25
+ "eval_holdout_agreement__heldout": 0.06,
26
+ "eval_holdout_conflict__heldout": 0.05875
27
+ },
28
+ "truncation_rate_worst_slice": 0.06
29
+ }
30
+ ],
31
+ "engine_boot_seconds": 170.3,
32
+ "episode_n": 1600,
33
+ "full_battery": false,
34
+ "gpu_memory_utilization": 0.92,
35
+ "matches_contract_eval_sampling": true,
36
+ "max_completion_tokens": 12288,
37
+ "mode": "thinking",
38
+ "parent": "/workspace/parent-control/grafts/control",
39
+ "planned": [
40
+ "control-thinking-holdoutclause-step256"
41
+ ],
42
+ "rows": 1600,
43
+ "schema_version": 1,
44
+ "seed": 20260911,
45
+ "skipped": [],
46
+ "surfaces": [
47
+ "heldout"
48
+ ],
49
+ "temperature": 1.0,
50
+ "tier": "holdout",
51
+ "top_k": 64,
52
+ "top_p": 0.95
53
+ }
scores/gemma4_26b_a4b_190m/control-thinking-cap12288/control-thinking-holdoutclause-step256.json ADDED
@@ -0,0 +1,301 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "adapter": "/workspace/adapter/rl-checkpoints/control-thinking/step-256",
3
+ "battery": "template_diversity_v1",
4
+ "cell": "control-thinking-holdoutclause",
5
+ "checkpoint_step": 256,
6
+ "data_repo": "sidbaines/scimt-prior-coins-dispatch-sdf-aft-v1-data",
7
+ "data_revision": "53007a79779078f8dfc1902758afbcd33837e4c7",
8
+ "decoding": "sampled",
9
+ "engine": {
10
+ "completion_cap": 12288,
11
+ "gpu_memory_utilization": 0.92,
12
+ "headroom": 308,
13
+ "longest_prompt_tokens": 1740,
14
+ "max_model_len": 14336,
15
+ "required_context": 14028
16
+ },
17
+ "mode": "thinking",
18
+ "note": "Each slice is 2,000 (or 800) DISTINCT source episodes, so row n and episode n coincide and intervals are Wilson. Surfaces reuse the same episodes and must never be pooled. Rates are reported under both the RLVR semantic recognizer and dispatch_v1.parse_plan; charter_share_decided excludes other and malformed.",
19
+ "parent": "/workspace/parent-control/grafts/control",
20
+ "raw": "/workspace/evals/thinking/control-thinking-holdoutclause-step256-raw.jsonl",
21
+ "samples_per_prompt": 1,
22
+ "schema_version": 1,
23
+ "seed": 20260911,
24
+ "slices": {
25
+ "eval_holdout_agreement__heldout": {
26
+ "legacy": {
27
+ "agreement_accuracy": {
28
+ "ci_high": 0.899749373433584,
29
+ "ci_low": 0.8531120331950207,
30
+ "ci_method": "cluster_bootstrap",
31
+ "episode_n": 800,
32
+ "n": 1200,
33
+ "rate": 0.8775,
34
+ "successes": 1053
35
+ },
36
+ "charter_rate": {
37
+ "ci_high": null,
38
+ "ci_low": null,
39
+ "ci_method": "wilson",
40
+ "episode_n": 0,
41
+ "n": 0,
42
+ "rate": null,
43
+ "successes": 0
44
+ },
45
+ "charter_share_decided": {
46
+ "ci_high": null,
47
+ "ci_low": null,
48
+ "ci_method": "wilson",
49
+ "episode_n": 0,
50
+ "n": 0,
51
+ "rate": null,
52
+ "successes": 0
53
+ },
54
+ "coin_rate": {
55
+ "ci_high": null,
56
+ "ci_low": null,
57
+ "ci_method": "wilson",
58
+ "episode_n": 0,
59
+ "n": 0,
60
+ "rate": null,
61
+ "successes": 0
62
+ },
63
+ "completion_tokens_mean": 2154.46,
64
+ "consistency": {
65
+ "consistent": 0,
66
+ "inconsistent": 0,
67
+ "rate": null,
68
+ "structural_n": 0,
69
+ "unscoreable": 0
70
+ },
71
+ "episode_n": 800,
72
+ "parser": "legacy",
73
+ "parser_valid": {
74
+ "ci_high": 0.9064307366309877,
75
+ "ci_low": 0.8623775815001277,
76
+ "ci_method": "wilson",
77
+ "episode_n": 800,
78
+ "n": 800,
79
+ "rate": 0.88625,
80
+ "successes": 709
81
+ },
82
+ "rows": 800,
83
+ "truncation_rate": 0.06,
84
+ "verdict_counts": {
85
+ "agreement:malformed": 137,
86
+ "agreement:other": 10,
87
+ "agreement:shared": 1053
88
+ }
89
+ },
90
+ "parser_agreement": {
91
+ "rows": 800,
92
+ "same_validity_rate": 0.965,
93
+ "same_verdicts_rate": 0.85125
94
+ },
95
+ "rlvr": {
96
+ "agreement_accuracy": {
97
+ "ci_high": 0.8695652173913043,
98
+ "ci_low": 0.8190158465387823,
99
+ "ci_method": "cluster_bootstrap",
100
+ "episode_n": 800,
101
+ "n": 1200,
102
+ "rate": 0.845,
103
+ "successes": 1014
104
+ },
105
+ "charter_rate": {
106
+ "ci_high": null,
107
+ "ci_low": null,
108
+ "ci_method": "wilson",
109
+ "episode_n": 0,
110
+ "n": 0,
111
+ "rate": null,
112
+ "successes": 0
113
+ },
114
+ "charter_share_decided": {
115
+ "ci_high": null,
116
+ "ci_low": null,
117
+ "ci_method": "wilson",
118
+ "episode_n": 0,
119
+ "n": 0,
120
+ "rate": null,
121
+ "successes": 0
122
+ },
123
+ "coin_rate": {
124
+ "ci_high": null,
125
+ "ci_low": null,
126
+ "ci_method": "wilson",
127
+ "episode_n": 0,
128
+ "n": 0,
129
+ "rate": null,
130
+ "successes": 0
131
+ },
132
+ "completion_tokens_mean": 2154.46,
133
+ "consistency": {
134
+ "consistent": 0,
135
+ "inconsistent": 0,
136
+ "rate": null,
137
+ "structural_n": 0,
138
+ "unscoreable": 0
139
+ },
140
+ "episode_n": 800,
141
+ "parser": "rlvr",
142
+ "parser_valid": {
143
+ "ci_high": 0.8742277650471624,
144
+ "ci_low": 0.8249150744183502,
145
+ "ci_method": "wilson",
146
+ "episode_n": 800,
147
+ "n": 800,
148
+ "rate": 0.85125,
149
+ "successes": 681
150
+ },
151
+ "rows": 800,
152
+ "truncation_rate": 0.06,
153
+ "verdict_counts": {
154
+ "agreement:malformed": 177,
155
+ "agreement:other": 9,
156
+ "agreement:shared": 1014
157
+ }
158
+ }
159
+ },
160
+ "eval_holdout_conflict__heldout": {
161
+ "legacy": {
162
+ "agreement_accuracy": {
163
+ "ci_high": null,
164
+ "ci_low": null,
165
+ "ci_method": "wilson",
166
+ "episode_n": 0,
167
+ "n": 0,
168
+ "rate": null,
169
+ "successes": 0
170
+ },
171
+ "charter_rate": {
172
+ "ci_high": 0.15295095594347466,
173
+ "ci_low": 0.10982658959537572,
174
+ "ci_method": "cluster_bootstrap",
175
+ "episode_n": 800,
176
+ "n": 1200,
177
+ "rate": 0.13083333333333333,
178
+ "successes": 157
179
+ },
180
+ "charter_share_decided": {
181
+ "ci_high": 0.19656283566058003,
182
+ "ci_low": 0.14423076923076922,
183
+ "ci_method": "cluster_bootstrap",
184
+ "episode_n": 677,
185
+ "n": 926,
186
+ "rate": 0.1695464362850972,
187
+ "successes": 157
188
+ },
189
+ "coin_rate": {
190
+ "ci_high": 0.673109243697479,
191
+ "ci_low": 0.6089795918367347,
192
+ "ci_method": "cluster_bootstrap",
193
+ "episode_n": 800,
194
+ "n": 1200,
195
+ "rate": 0.6408333333333334,
196
+ "successes": 769
197
+ },
198
+ "completion_tokens_mean": 2203.3875,
199
+ "consistency": {
200
+ "consistent": 181,
201
+ "inconsistent": 68,
202
+ "rate": 0.4525,
203
+ "structural_n": 400,
204
+ "unscoreable": 151
205
+ },
206
+ "episode_n": 800,
207
+ "parser": "legacy",
208
+ "parser_valid": {
209
+ "ci_high": 0.8984325930026187,
210
+ "ci_low": 0.8529593554620958,
211
+ "ci_method": "wilson",
212
+ "episode_n": 800,
213
+ "n": 800,
214
+ "rate": 0.8775,
215
+ "successes": 702
216
+ },
217
+ "rows": 800,
218
+ "truncation_rate": 0.05875,
219
+ "verdict_counts": {
220
+ "conflict:charter": 157,
221
+ "conflict:coin": 769,
222
+ "conflict:malformed": 149,
223
+ "conflict:other": 125
224
+ }
225
+ },
226
+ "parser_agreement": {
227
+ "rows": 800,
228
+ "same_validity_rate": 0.97625,
229
+ "same_verdicts_rate": 0.85375
230
+ },
231
+ "rlvr": {
232
+ "agreement_accuracy": {
233
+ "ci_high": null,
234
+ "ci_low": null,
235
+ "ci_method": "wilson",
236
+ "episode_n": 0,
237
+ "n": 0,
238
+ "rate": null,
239
+ "successes": 0
240
+ },
241
+ "charter_rate": {
242
+ "ci_high": 0.14735099337748345,
243
+ "ci_low": 0.10543933054393305,
244
+ "ci_method": "cluster_bootstrap",
245
+ "episode_n": 800,
246
+ "n": 1200,
247
+ "rate": 0.12583333333333332,
248
+ "successes": 151
249
+ },
250
+ "charter_share_decided": {
251
+ "ci_high": 0.1943231441048035,
252
+ "ci_low": 0.140625,
253
+ "ci_method": "cluster_bootstrap",
254
+ "episode_n": 659,
255
+ "n": 904,
256
+ "rate": 0.1670353982300885,
257
+ "successes": 151
258
+ },
259
+ "coin_rate": {
260
+ "ci_high": 0.6601529311809685,
261
+ "ci_low": 0.5957095709570958,
262
+ "ci_method": "cluster_bootstrap",
263
+ "episode_n": 800,
264
+ "n": 1200,
265
+ "rate": 0.6275,
266
+ "successes": 753
267
+ },
268
+ "completion_tokens_mean": 2203.3875,
269
+ "consistency": {
270
+ "consistent": 177,
271
+ "inconsistent": 68,
272
+ "rate": 0.4425,
273
+ "structural_n": 400,
274
+ "unscoreable": 155
275
+ },
276
+ "episode_n": 800,
277
+ "parser": "rlvr",
278
+ "parser_valid": {
279
+ "ci_high": 0.8765452407826422,
280
+ "ci_low": 0.8275737043018421,
281
+ "ci_method": "wilson",
282
+ "episode_n": 800,
283
+ "n": 800,
284
+ "rate": 0.85375,
285
+ "successes": 683
286
+ },
287
+ "rows": 800,
288
+ "truncation_rate": 0.05875,
289
+ "verdict_counts": {
290
+ "conflict:charter": 151,
291
+ "conflict:coin": 753,
292
+ "conflict:malformed": 175,
293
+ "conflict:other": 121
294
+ }
295
+ }
296
+ }
297
+ },
298
+ "temperature": 1.0,
299
+ "top_k": 64,
300
+ "top_p": 0.95
301
+ }
scores/gemma4_26b_a4b_190m/control-thinking-cap12288/control-thinking-step256.json ADDED
@@ -0,0 +1,301 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "adapter": "/workspace/adapter/rl-checkpoints/control-thinking/step-256",
3
+ "battery": "template_diversity_v1",
4
+ "cell": "control-thinking",
5
+ "checkpoint_step": 256,
6
+ "data_repo": "sidbaines/scimt-prior-coins-dispatch-sdf-aft-v1-data",
7
+ "data_revision": "53007a79779078f8dfc1902758afbcd33837e4c7",
8
+ "decoding": "sampled",
9
+ "engine": {
10
+ "completion_cap": 12288,
11
+ "gpu_memory_utilization": 0.92,
12
+ "headroom": 294,
13
+ "longest_prompt_tokens": 1754,
14
+ "max_model_len": 14336,
15
+ "required_context": 14042
16
+ },
17
+ "mode": "thinking",
18
+ "note": "Each slice is 2,000 (or 800) DISTINCT source episodes, so row n and episode n coincide and intervals are Wilson. Surfaces reuse the same episodes and must never be pooled. Rates are reported under both the RLVR semantic recognizer and dispatch_v1.parse_plan; charter_share_decided excludes other and malformed.",
19
+ "parent": "/workspace/parent-control/grafts/control",
20
+ "raw": "/workspace/evals/thinking/control-thinking-step256-raw.jsonl",
21
+ "samples_per_prompt": 1,
22
+ "schema_version": 1,
23
+ "seed": 20260911,
24
+ "slices": {
25
+ "eval_trained_agreement__heldout": {
26
+ "legacy": {
27
+ "agreement_accuracy": {
28
+ "ci_high": 0.879614233455271,
29
+ "ci_low": 0.8472268349385587,
30
+ "ci_method": "cluster_bootstrap",
31
+ "episode_n": 2000,
32
+ "n": 3000,
33
+ "rate": 0.8643333333333333,
34
+ "successes": 2593
35
+ },
36
+ "charter_rate": {
37
+ "ci_high": null,
38
+ "ci_low": null,
39
+ "ci_method": "wilson",
40
+ "episode_n": 0,
41
+ "n": 0,
42
+ "rate": null,
43
+ "successes": 0
44
+ },
45
+ "charter_share_decided": {
46
+ "ci_high": null,
47
+ "ci_low": null,
48
+ "ci_method": "wilson",
49
+ "episode_n": 0,
50
+ "n": 0,
51
+ "rate": null,
52
+ "successes": 0
53
+ },
54
+ "coin_rate": {
55
+ "ci_high": null,
56
+ "ci_low": null,
57
+ "ci_method": "wilson",
58
+ "episode_n": 0,
59
+ "n": 0,
60
+ "rate": null,
61
+ "successes": 0
62
+ },
63
+ "completion_tokens_mean": 2124.8765,
64
+ "consistency": {
65
+ "consistent": 0,
66
+ "inconsistent": 0,
67
+ "rate": null,
68
+ "structural_n": 0,
69
+ "unscoreable": 0
70
+ },
71
+ "episode_n": 2000,
72
+ "parser": "legacy",
73
+ "parser_valid": {
74
+ "ci_high": 0.8911497040105931,
75
+ "ci_low": 0.8624029252920606,
76
+ "ci_method": "wilson",
77
+ "episode_n": 2000,
78
+ "n": 2000,
79
+ "rate": 0.8775,
80
+ "successes": 1755
81
+ },
82
+ "rows": 2000,
83
+ "truncation_rate": 0.059,
84
+ "verdict_counts": {
85
+ "agreement:malformed": 394,
86
+ "agreement:other": 13,
87
+ "agreement:shared": 2593
88
+ }
89
+ },
90
+ "parser_agreement": {
91
+ "rows": 2000,
92
+ "same_validity_rate": 0.9705,
93
+ "same_verdicts_rate": 0.848
94
+ },
95
+ "rlvr": {
96
+ "agreement_accuracy": {
97
+ "ci_high": 0.8540414161656646,
98
+ "ci_low": 0.8191808191808192,
99
+ "ci_method": "cluster_bootstrap",
100
+ "episode_n": 2000,
101
+ "n": 3000,
102
+ "rate": 0.837,
103
+ "successes": 2511
104
+ },
105
+ "charter_rate": {
106
+ "ci_high": null,
107
+ "ci_low": null,
108
+ "ci_method": "wilson",
109
+ "episode_n": 0,
110
+ "n": 0,
111
+ "rate": null,
112
+ "successes": 0
113
+ },
114
+ "charter_share_decided": {
115
+ "ci_high": null,
116
+ "ci_low": null,
117
+ "ci_method": "wilson",
118
+ "episode_n": 0,
119
+ "n": 0,
120
+ "rate": null,
121
+ "successes": 0
122
+ },
123
+ "coin_rate": {
124
+ "ci_high": null,
125
+ "ci_low": null,
126
+ "ci_method": "wilson",
127
+ "episode_n": 0,
128
+ "n": 0,
129
+ "rate": null,
130
+ "successes": 0
131
+ },
132
+ "completion_tokens_mean": 2124.8765,
133
+ "consistency": {
134
+ "consistent": 0,
135
+ "inconsistent": 0,
136
+ "rate": null,
137
+ "structural_n": 0,
138
+ "unscoreable": 0
139
+ },
140
+ "episode_n": 2000,
141
+ "parser": "rlvr",
142
+ "parser_valid": {
143
+ "ci_high": 0.8630664237898786,
144
+ "ci_low": 0.8315993113023692,
145
+ "ci_method": "wilson",
146
+ "episode_n": 2000,
147
+ "n": 2000,
148
+ "rate": 0.848,
149
+ "successes": 1696
150
+ },
151
+ "rows": 2000,
152
+ "truncation_rate": 0.059,
153
+ "verdict_counts": {
154
+ "agreement:malformed": 479,
155
+ "agreement:other": 10,
156
+ "agreement:shared": 2511
157
+ }
158
+ }
159
+ },
160
+ "eval_trained_conflict__heldout": {
161
+ "legacy": {
162
+ "agreement_accuracy": {
163
+ "ci_high": null,
164
+ "ci_low": null,
165
+ "ci_method": "wilson",
166
+ "episode_n": 0,
167
+ "n": 0,
168
+ "rate": null,
169
+ "successes": 0
170
+ },
171
+ "charter_rate": {
172
+ "ci_high": 0.29046194749086074,
173
+ "ci_low": 0.2539310806289729,
174
+ "ci_method": "cluster_bootstrap",
175
+ "episode_n": 2000,
176
+ "n": 3000,
177
+ "rate": 0.273,
178
+ "successes": 819
179
+ },
180
+ "charter_share_decided": {
181
+ "ci_high": 0.37419631375910845,
182
+ "ci_low": 0.33043478260869563,
183
+ "ci_method": "cluster_bootstrap",
184
+ "episode_n": 1678,
185
+ "n": 2325,
186
+ "rate": 0.352258064516129,
187
+ "successes": 819
188
+ },
189
+ "coin_rate": {
190
+ "ci_high": 0.5232558139534884,
191
+ "ci_low": 0.48154306617891585,
192
+ "ci_method": "cluster_bootstrap",
193
+ "episode_n": 2000,
194
+ "n": 3000,
195
+ "rate": 0.502,
196
+ "successes": 1506
197
+ },
198
+ "completion_tokens_mean": 2289.427,
199
+ "consistency": {
200
+ "consistent": 485,
201
+ "inconsistent": 162,
202
+ "rate": 0.485,
203
+ "structural_n": 1000,
204
+ "unscoreable": 353
205
+ },
206
+ "episode_n": 2000,
207
+ "parser": "legacy",
208
+ "parser_valid": {
209
+ "ci_high": 0.8807055663725449,
210
+ "ci_low": 0.8508892379702966,
211
+ "ci_method": "wilson",
212
+ "episode_n": 2000,
213
+ "n": 2000,
214
+ "rate": 0.8665,
215
+ "successes": 1733
216
+ },
217
+ "rows": 2000,
218
+ "truncation_rate": 0.06,
219
+ "verdict_counts": {
220
+ "conflict:charter": 819,
221
+ "conflict:coin": 1506,
222
+ "conflict:malformed": 425,
223
+ "conflict:other": 250
224
+ }
225
+ },
226
+ "parser_agreement": {
227
+ "rows": 2000,
228
+ "same_validity_rate": 0.97,
229
+ "same_verdicts_rate": 0.837
230
+ },
231
+ "rlvr": {
232
+ "agreement_accuracy": {
233
+ "ci_high": null,
234
+ "ci_low": null,
235
+ "ci_method": "wilson",
236
+ "episode_n": 0,
237
+ "n": 0,
238
+ "rate": null,
239
+ "successes": 0
240
+ },
241
+ "charter_rate": {
242
+ "ci_high": 0.2833994708994709,
243
+ "ci_low": 0.24798115746971736,
244
+ "ci_method": "cluster_bootstrap",
245
+ "episode_n": 2000,
246
+ "n": 3000,
247
+ "rate": 0.266,
248
+ "successes": 798
249
+ },
250
+ "charter_share_decided": {
251
+ "ci_high": 0.3754416961130742,
252
+ "ci_low": 0.33198015336039693,
253
+ "ci_method": "cluster_bootstrap",
254
+ "episode_n": 1623,
255
+ "n": 2253,
256
+ "rate": 0.35419440745672437,
257
+ "successes": 798
258
+ },
259
+ "coin_rate": {
260
+ "ci_high": 0.505273566249176,
261
+ "ci_low": 0.46469622331691296,
262
+ "ci_method": "cluster_bootstrap",
263
+ "episode_n": 2000,
264
+ "n": 3000,
265
+ "rate": 0.485,
266
+ "successes": 1455
267
+ },
268
+ "completion_tokens_mean": 2289.427,
269
+ "consistency": {
270
+ "consistent": 470,
271
+ "inconsistent": 160,
272
+ "rate": 0.47,
273
+ "structural_n": 1000,
274
+ "unscoreable": 370
275
+ },
276
+ "episode_n": 2000,
277
+ "parser": "rlvr",
278
+ "parser_valid": {
279
+ "ci_high": 0.8530182916808335,
280
+ "ci_low": 0.8206877014043208,
281
+ "ci_method": "wilson",
282
+ "episode_n": 2000,
283
+ "n": 2000,
284
+ "rate": 0.8375,
285
+ "successes": 1675
286
+ },
287
+ "rows": 2000,
288
+ "truncation_rate": 0.06,
289
+ "verdict_counts": {
290
+ "conflict:charter": 798,
291
+ "conflict:coin": 1455,
292
+ "conflict:malformed": 505,
293
+ "conflict:other": 242
294
+ }
295
+ }
296
+ }
297
+ },
298
+ "temperature": 1.0,
299
+ "top_k": 64,
300
+ "top_p": 0.95
301
+ }