rorylshanks commited on
Commit
27c0c79
·
verified ·
1 Parent(s): c754d67

Jeeves v2: SFT on short, finished reasoning chains, public data only, two temperatures

Browse files

Replaces the weights, head and export metadata with v2. The drafters are v1's (v2 drafters to follow). v1 is tagged `v1`.

export.json CHANGED
@@ -1,12 +1,13 @@
1
  {
2
- "source_checkpoint": "runs/best_step402",
3
  "base_model": "Qwen/Qwen3.5-9B",
4
  "dtype": "bfloat16",
5
  "format": "markers-v3-plainchains",
6
  "head_dim": 256,
7
- "temperature": 1.8589280843734741,
8
- "stage": "cispo",
9
- "step": 402,
 
10
  "sources": {
11
  "banking77": 0,
12
  "boolq": 1,
@@ -33,79 +34,199 @@
33
  "buried": 22,
34
  "unknowable": 23,
35
  "unknowable_control": 24,
36
- "night2_assertion": 25,
37
- "night2_dates": 26,
38
- "night2_unknowable": 27,
39
- "night2_unknowable_control": 28
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
40
  },
41
- "metrics": null,
42
  "training_config": {
43
- "stage": "cispo",
44
- "run_dir": "runs/cispo_rep",
45
- "data_dir": "data",
46
  "model": "Qwen/Qwen3.5-9B",
47
- "init": "runs/sft_v4/final",
48
- "resume": null,
49
  "lora_r": 16,
50
  "lora_alpha": 32.0,
51
- "lora_dropout": 0.0,
52
  "head_dim": 256,
 
 
 
 
 
 
 
 
 
 
 
 
 
53
  "lr": 5e-05,
54
  "head_lr": 0.0001,
55
- "weight_decay": 0.0,
56
  "warmup": 20,
57
- "epochs": 4,
58
- "max_steps": 0,
 
59
  "batch_size": 8,
60
  "grad_clip": 1.0,
61
  "max_len": 4096,
62
  "max_seq_len": 8192,
63
  "train_limit": 0,
64
  "dev_limit": 0,
65
- "eval_every": 50,
66
- "eval_batch_size": 16,
67
  "eval_think": true,
68
- "eval_think_limit": 256,
69
  "log_every": 10,
70
- "seed": 0,
71
  "gradient_checkpointing": true,
72
  "prompts_per_step": 8,
73
  "group": 8,
74
- "nothink_per_group": 0,
75
- "nothink_start_epoch": 1,
76
- "max_think": 2560,
77
- "temperature": 1.0,
78
- "top_p": 1.0,
79
  "eps_high": 0.2,
80
- "updates": 1,
81
- "train_mix_frac": 0.0,
82
  "rollout_ce_weight": 0.5,
83
  "anchor_ce_weight": 0.5,
84
  "anchor_batch": 8,
85
- "gen_batch_size": 64,
86
  "logprob_chunk": 128,
87
  "micro_groups": 2,
88
  "length_hinge": 2048,
89
  "length_hinge_end": 1024,
90
- "length_ramp": 0,
91
- "length_penalty_weight": 1.0,
92
  "length_penalty_cap": 0.1,
93
- "length_penalty_correct_only": true,
94
- "length_hinge2": 0,
95
- "length_hinge2_end": 1792,
96
- "length_penalty2_weight": 0.0,
97
- "closure_reward": 0.0,
98
- "pad_multiple": 128,
99
- "num_workers": 2,
100
- "compile": true,
101
- "graph_decode": true,
102
- "async_rollouts": true,
103
  "ckpt_minutes": 45.0,
104
  "log_rollouts": 3,
105
  "log_chain_chars": 1200,
106
- "sft_think_frac": 0.5,
107
  "chain_cache": "data/chains",
108
- "filter_active": false,
109
- "filter_min_spread": 0.02
 
 
 
 
 
110
  }
111
  }
 
1
  {
2
+ "source_checkpoint": "runs/sft/final",
3
  "base_model": "Qwen/Qwen3.5-9B",
4
  "dtype": "bfloat16",
5
  "format": "markers-v3-plainchains",
6
  "head_dim": 256,
7
+ "temperature": 1.522,
8
+ "think_temperature": 5.4195,
9
+ "stage": "sft",
10
+ "step": 16867,
11
  "sources": {
12
  "banking77": 0,
13
  "boolq": 1,
 
34
  "buried": 22,
35
  "unknowable": 23,
36
  "unknowable_control": 24,
37
+ "aqua_rat": 25,
38
+ "cosmos_qa": 26,
39
+ "esci": 27,
40
+ "glaive_fc": 28,
41
+ "gsm8k": 29,
42
+ "hellaswag": 30,
43
+ "newyorker": 31,
44
+ "openbookqa": 32,
45
+ "ragtruth": 33,
46
+ "ruletaker": 34,
47
+ "sgd": 35,
48
+ "swag": 36,
49
+ "toolace": 37,
50
+ "when2call": 38,
51
+ "winogrande": 39,
52
+ "decider_qasc": 40,
53
+ "decider_massive_intent_upstream": 41,
54
+ "night2_dates": 42,
55
+ "kev/documents-v1": 43,
56
+ "decider_boolq": 44,
57
+ "decider/routing_messages": 45,
58
+ "decider/commands": 46,
59
+ "decider_go_emotions": 47,
60
+ "decider_banking77": 48,
61
+ "typed_decisions": 49,
62
+ "decider_commonsense_qa": 50,
63
+ "pi_llmail": 51,
64
+ "decider_helpsteer3_pref": 52,
65
+ "decider_medqa": 53,
66
+ "decider_prosocial_safety": 54,
67
+ "openjev/v1/phone-extraction-control-v1": 55,
68
+ "kev/hard-v1": 56,
69
+ "decider_dbpedia": 57,
70
+ "decider_hh_rlhf": 58,
71
+ "decider_bias_in_bios": 59,
72
+ "openjev/v1/mailroom-control-v1": 60,
73
+ "decider_json_state": 61,
74
+ "decider_rules": 62,
75
+ "pi_slabs": 63,
76
+ "openjev/v1/silent-failure-control-v1": 64,
77
+ "decider_medmcqa": 65,
78
+ "decider_mind2web": 66,
79
+ "pi_neuralchemy_v1": 67,
80
+ "openjev/v1/amount-extraction-control-v1": 68,
81
+ "decider/routing_terse": 69,
82
+ "openjev/v1/browser-drone-expansion-v1-redistributable": 70,
83
+ "openjev/v1/ir-control-v1": 71,
84
+ "pi_gandalf": 72,
85
+ "decider_snli": 73,
86
+ "decider_arc": 74,
87
+ "pi_harelix": 75,
88
+ "openjev/v1/email-selection-control-v1": 76,
89
+ "decider_clinc_oos": 77,
90
+ "openjev/v1.1/community-hard-mix-v2-redistributable": 78,
91
+ "pi_hackaprompt": 79,
92
+ "pi_mixP_benign": 80,
93
+ "pi_injecagent": 81,
94
+ "night2_assertion": 82,
95
+ "decider/situations": 83,
96
+ "kev/devtools-v1": 84,
97
+ "decider_civil_comments": 85,
98
+ "pi_alespalla": 86,
99
+ "decider_hate_speech_scales": 87,
100
+ "decider_liar2": 88,
101
+ "pi_yanismiraoui": 89,
102
+ "decider_isolated": 90,
103
+ "decider_helpsteer2": 91,
104
+ "decider/custom_questions": 92,
105
+ "openjev/v1/entity-alignment-control-v1": 93,
106
+ "pi_harmbench_benign": 94,
107
+ "spark": 95,
108
+ "night2_unknowable": 96,
109
+ "pi_jailbreakbench": 97,
110
+ "night2_unknowable_control": 98,
111
+ "openjev/v1/context-retention-control-v1": 99,
112
+ "openjev/v1/citation-control-v1": 100,
113
+ "pi_neuralchemy_original": 101,
114
+ "pi_llmail_fp": 102,
115
+ "openjev/v1/sponsor-segment-control-v1": 103
116
+ },
117
+ "metrics": {
118
+ "dev_acc": 0.8311879367414491,
119
+ "dev_nll": 0.4963217283180857,
120
+ "dev_n": 5438,
121
+ "per_source": {
122
+ "banking77": 0.7759,
123
+ "boolq": 0.95,
124
+ "agnews": 0.9567,
125
+ "mnli": 0.8017,
126
+ "sst5": 0.6625,
127
+ "yelp": 0.7688,
128
+ "trec": 0.8448,
129
+ "dbpedia14": 0.9741,
130
+ "amazon": 0.65,
131
+ "imdb": 0.975,
132
+ "mmlu": 0.7328,
133
+ "emotion": 0.5948,
134
+ "tweet_offensive": 0.75,
135
+ "qnli": 0.9375,
136
+ "paws": 0.825,
137
+ "sciq": 0.9655,
138
+ "contrastive": 0.9694,
139
+ "compositional": 0.9688,
140
+ "composition_holdout": 0.9583,
141
+ "semif": 0.8889,
142
+ "wanli": 0.7073,
143
+ "mmlu_pro": 0.6164,
144
+ "buried": 0.7414,
145
+ "unknowable": 0.4128,
146
+ "unknowable_control": 0.9795,
147
+ "aqua_rat": 0.6181,
148
+ "cosmos_qa": 0.9,
149
+ "esci": 0.5933,
150
+ "glaive_fc": 0.9933,
151
+ "gsm8k": 0.755,
152
+ "hellaswag": 0.9536,
153
+ "newyorker": 0.7415,
154
+ "openbookqa": 0.8933,
155
+ "ragtruth": 0.7933,
156
+ "ruletaker": 0.94,
157
+ "sgd": 0.9733,
158
+ "swag": 0.8267,
159
+ "toolace": 0.9533,
160
+ "when2call": 0.9467,
161
+ "winogrande": 0.92
162
+ },
163
+ "unknowable": {
164
+ "n": 109,
165
+ "mean_max_p": 0.5008,
166
+ "share_at_0_9": 0.0183,
167
+ "control_acc": 0.9795,
168
+ "control_mean_max_p": 0.9638,
169
+ "control_share_at_0_9": 0.8699
170
+ }
171
  },
 
172
  "training_config": {
 
 
 
173
  "model": "Qwen/Qwen3.5-9B",
174
+ "init": null,
 
175
  "lora_r": 16,
176
  "lora_alpha": 32.0,
 
177
  "head_dim": 256,
178
+ "eval_batch_size": 16,
179
+ "eval_think_limit": 256,
180
+ "seed": 0,
181
+ "max_think": 2560,
182
+ "temperature": 1.0,
183
+ "gen_batch_size": 64,
184
+ "pad_multiple": 256,
185
+ "num_workers": 2,
186
+ "compile": true,
187
+ "stage": "sft",
188
+ "run_dir": "runs/sft",
189
+ "data_dir": "data",
190
+ "resume": null,
191
  "lr": 5e-05,
192
  "head_lr": 0.0001,
 
193
  "warmup": 20,
194
+ "sft_epochs": 1,
195
+ "cispo_epochs": 4,
196
+ "stop_step": 402,
197
  "batch_size": 8,
198
  "grad_clip": 1.0,
199
  "max_len": 4096,
200
  "max_seq_len": 8192,
201
  "train_limit": 0,
202
  "dev_limit": 0,
203
+ "eval_every": 2000,
 
204
  "eval_think": true,
 
205
  "log_every": 10,
 
206
  "gradient_checkpointing": true,
207
  "prompts_per_step": 8,
208
  "group": 8,
 
 
 
 
 
209
  "eps_high": 0.2,
 
 
210
  "rollout_ce_weight": 0.5,
211
  "anchor_ce_weight": 0.5,
212
  "anchor_batch": 8,
213
+ "anchor_limit": 0,
214
  "logprob_chunk": 128,
215
  "micro_groups": 2,
216
  "length_hinge": 2048,
217
  "length_hinge_end": 1024,
 
 
218
  "length_penalty_cap": 0.1,
 
 
 
 
 
 
 
 
 
 
219
  "ckpt_minutes": 45.0,
220
  "log_rollouts": 3,
221
  "log_chain_chars": 1200,
222
+ "sft_think_frac": 0.15,
223
  "chain_cache": "data/chains",
224
+ "bucket": true,
225
+ "chain_max_len": 1536,
226
+ "unclosed_reward": 1.0,
227
+ "loop_penalty": 0.0,
228
+ "greedy_rollouts": 0,
229
+ "chain_lm_weight": 0.5,
230
+ "sft_truncate_frac": 0.2
231
  }
232
  }
head.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:2ab935dea9eecf263086d9ed6dfe97fd5558193663ecae102fd3c50ae6878d60
3
  size 8392767
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:228601cd90a4ca047711847b573c47a790ba72b2dfce310a6333b5c78cc8ce44
3
  size 8392767
model-00001-of-00005.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:c8d38178b51c845d6befca9e2c8a579889055cecf510fdbc85dbdf7bb867a0d6
3
  size 4268059288
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aab4cd72ce197c122c314a75380b5580a2c5a6f4089f7326dc2c49b3cedb592e
3
  size 4268059288
model-00002-of-00005.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:a25e1a608a5be9cadef95e3f40fdca5c17b6c21e86481186adfd7a8a1f5b8f40
3
  size 4266315424
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8d5a3ab2fe0235336a5c26a2782003b2467945b3b32ad85798cb29d01742e67a
3
  size 4266315424
model-00003-of-00005.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ab693e0b5940052ad04450fccfd12c87a49ce9ad23984bdfc07ee20d3ab654fe
3
  size 4215378072
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5809ff16230bf34923cb3f7c65f03a5f2bf9fbf3c02e1132fc941724c4c36aee
3
  size 4215378072
model-00004-of-00005.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9942b7369222a4403809a05f1da21f53b006f2d5cf72651483dda48c4931ed9a
3
  size 3123665648
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:799ca2358f6801d5a5d204ea5864efc82a57bce57f9fd0d82633e318e18b0b00
3
  size 3123665648