Hugging Face's logo

AlephFunk
/
jev-qwen

PEFT
Safetensors
lora
typed-decisions
looped-transformer
ai-control
benchmark
Model card Files
xet
 Community
AlephFunk commited on
Commit
fa2de73
·
verified ·
1 Parent(s): 7eddd6a

Publish adapters/multi_loop_s0

Browse files
Files changed (3) hide show
  1. adapters/multi_loop_s0/adapter_config.json +57 -0
  2. adapters/multi_loop_s0/adapter_model.safetensors +3 -0
  3. adapters/multi_loop_s0/train_record.json +269 -0
adapters/multi_loop_s0/adapter_config.json ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": {
6
+ "base_model_class": "Qwen3_5ForCausalLM",
7
+ "parent_library": "transformers.models.qwen3_5.modeling_qwen3_5"
8
+ },
9
+ "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base",
10
+ "bias": "none",
11
+ "corda_config": null,
12
+ "ensure_weight_tying": false,
13
+ "eva_config": null,
14
+ "exclude_modules": null,
15
+ "fan_in_fan_out": false,
16
+ "inference_mode": true,
17
+ "init_lora_weights": true,
18
+ "kasa_config": null,
19
+ "layer_replication": null,
20
+ "layers_pattern": null,
21
+ "layers_to_transform": null,
22
+ "loftq_config": {},
23
+ "lora_alpha": 32,
24
+ "lora_bias": false,
25
+ "lora_dropout": 0.05,
26
+ "lora_ga_config": null,
27
+ "megatron_config": null,
28
+ "megatron_core": "megatron.core",
29
+ "modules_to_save": null,
30
+ "monteclora_config": null,
31
+ "peft_type": "LORA",
32
+ "peft_version": "0.21.1",
33
+ "qalora_group_size": 16,
34
+ "r": 16,
35
+ "rank_pattern": {},
36
+ "revision": null,
37
+ "target_modules": [
38
+ "up_proj",
39
+ "k_proj",
40
+ "down_proj",
41
+ "v_proj",
42
+ "gate_proj",
43
+ "in_proj_qkv",
44
+ "in_proj_z",
45
+ "out_proj",
46
+ "o_proj",
47
+ "q_proj"
48
+ ],
49
+ "target_parameters": null,
50
+ "task_type": null,
51
+ "trainable_token_indices": null,
52
+ "use_bdlora": null,
53
+ "use_dora": false,
54
+ "use_qalora": false,
55
+ "use_rslora": false,
56
+ "velora_config": null
57
+ }
adapters/multi_loop_s0/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:da6d4d47d0d45f0162f6f2954a170853f58a1aa89c4aff896f011f041727d200
3
+ size 121951008
adapters/multi_loop_s0/train_record.json ADDED
@@ -0,0 +1,269 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "multi_loop",
3
+ "seed": 0,
4
+ "created_utc": "2026-10-06T07:04:46Z",
5
+ "git": {
6
+ "commit": "ee9010336048a723521bad97699702c3e32a8a03",
7
+ "dirty": false
8
+ },
9
+ "config": {
10
+ "name": "multi_loop",
11
+ "variant": "configs/variants/v2_loop_mid_r2.yaml",
12
+ "seed": 0,
13
+ "data": {
14
+ "prompt_suite": "evals/suites/dev.yaml",
15
+ "exclude_suites": [
16
+ "evals/suites/smoke.yaml",
17
+ "evals/suites/dev.yaml"
18
+ ],
19
+ "tasks": null,
20
+ "difficulties": [
21
+ 1,
22
+ 2,
23
+ 3
24
+ ],
25
+ "n_per_difficulty": 100,
26
+ "n_val_per_difficulty": 4,
27
+ "formats": [
28
+ "choice",
29
+ "generate",
30
+ "generate_cot"
31
+ ],
32
+ "source": "multi",
33
+ "sources": [
34
+ {
35
+ "kind": "dev",
36
+ "prompt_suite": "evals/suites/dev.yaml",
37
+ "exclude_suites": [
38
+ "evals/suites/smoke.yaml",
39
+ "evals/suites/dev.yaml"
40
+ ],
41
+ "tasks": null,
42
+ "difficulties": [
43
+ 1,
44
+ 2,
45
+ 3
46
+ ],
47
+ "n_per_difficulty": 100,
48
+ "n_val_per_difficulty": 4,
49
+ "formats": [
50
+ "choice",
51
+ "generate",
52
+ "generate_cot"
53
+ ]
54
+ },
55
+ {
56
+ "kind": "rmp",
57
+ "families": [
58
+ "pointer_chase",
59
+ "gated_pointer_chase",
60
+ "masked_pointer_chase",
61
+ "reachability",
62
+ "sudoku4"
63
+ ],
64
+ "n_per_family": 480,
65
+ "n_val_per_family": 20,
66
+ "index_offset": 1000
67
+ },
68
+ {
69
+ "kind": "s2",
70
+ "upsample": 3
71
+ },
72
+ {
73
+ "kind": "s7",
74
+ "upsample": 4
75
+ }
76
+ ]
77
+ },
78
+ "lora": {
79
+ "r": 16,
80
+ "alpha": 32,
81
+ "dropout": 0.05,
82
+ "target_modules": [
83
+ "q_proj",
84
+ "k_proj",
85
+ "v_proj",
86
+ "o_proj",
87
+ "in_proj_qkv",
88
+ "in_proj_z",
89
+ "out_proj",
90
+ "gate_proj",
91
+ "up_proj",
92
+ "down_proj"
93
+ ],
94
+ "layers_to_transform": null
95
+ },
96
+ "loss": {
97
+ "choice": "ce"
98
+ },
99
+ "optim": {
100
+ "lr": 0.0001,
101
+ "warmup_steps": 20,
102
+ "min_lr_ratio": 0.1,
103
+ "grad_accum": 8,
104
+ "epochs": 1,
105
+ "max_grad_norm": 1.0,
106
+ "weight_decay": 0.0
107
+ },
108
+ "gradient_checkpointing": true,
109
+ "eval_every": 100,
110
+ "save_every": 25,
111
+ "max_steps": null
112
+ },
113
+ "config_sha256": "21356fa5a536151a7e0813f9394832315e4059bcf171984f7913f33a9a76ee94",
114
+ "variant": {
115
+ "name": "v2_loop_mid_r2",
116
+ "driver": "schedule",
117
+ "loop": {
118
+ "start": 12,
119
+ "end": 16,
120
+ "n_iters": 2
121
+ },
122
+ "halting": null,
123
+ "adapter": null
124
+ },
125
+ "model": {
126
+ "id": "Qwen/Qwen3.5-4B-Base",
127
+ "revision_requested": "1001bb4d826a52d1f399e183466143f4da7b741b",
128
+ "revision_resolved": null,
129
+ "class": "Qwen3_5ForCausalLM",
130
+ "dtype": "bfloat16",
131
+ "load_in_4bit": false,
132
+ "cuda_memory_fraction": 0.7,
133
+ "unexpected_keys_dropped": 0,
134
+ "n_params_text_stack": 4236225536,
135
+ "num_layers": 32,
136
+ "layer_types": [
137
+ "linear_attention",
138
+ "linear_attention",
139
+ "linear_attention",
140
+ "full_attention",
141
+ "linear_attention",
142
+ "linear_attention",
143
+ "linear_attention",
144
+ "full_attention",
145
+ "linear_attention",
146
+ "linear_attention",
147
+ "linear_attention",
148
+ "full_attention",
149
+ "linear_attention",
150
+ "linear_attention",
151
+ "linear_attention",
152
+ "full_attention",
153
+ "linear_attention",
154
+ "linear_attention",
155
+ "linear_attention",
156
+ "full_attention",
157
+ "linear_attention",
158
+ "linear_attention",
159
+ "linear_attention",
160
+ "full_attention",
161
+ "linear_attention",
162
+ "linear_attention",
163
+ "linear_attention",
164
+ "full_attention",
165
+ "linear_attention",
166
+ "linear_attention",
167
+ "linear_attention",
168
+ "full_attention"
169
+ ],
170
+ "trainable_params": 30474240
171
+ },
172
+ "data": {
173
+ "source": "multi (SPEC-U1)",
174
+ "parts": {
175
+ "dev": {
176
+ "n_train": 2400,
177
+ "n_val": 96,
178
+ "upsample": 1,
179
+ "formats": {
180
+ "choice": 800,
181
+ "generate": 800,
182
+ "generate_cot": 800
183
+ }
184
+ },
185
+ "rmp": {
186
+ "n_train": 2400,
187
+ "n_val": 100,
188
+ "upsample": 1,
189
+ "families": [
190
+ "pointer_chase",
191
+ "gated_pointer_chase",
192
+ "masked_pointer_chase",
193
+ "reachability",
194
+ "sudoku4"
195
+ ],
196
+ "rows_sha256": "55b96105bdbdf6dd12216026a984b4aa9c303ca3d9cbb5626500365db92c786c",
197
+ "formats": {
198
+ "choice": 2400
199
+ }
200
+ },
201
+ "s2": {
202
+ "n_train": 702,
203
+ "n_val": 50,
204
+ "upsample": 3,
205
+ "n_rows": 234,
206
+ "rows_sha256": "1f237e0a932e1c898faec8edb8cfc43c7376db7d6fc9371e68a07a392794d719"
207
+ },
208
+ "s7": {
209
+ "n_train": 216,
210
+ "n_val": 0,
211
+ "upsample": 4,
212
+ "n_rows": 54,
213
+ "rows_sha256": "f49e45f0efb5fd5f548b4d2b38db7add192b8c8e0c33c7e775f25b0f475c8f4d"
214
+ }
215
+ },
216
+ "n_train": 5718,
217
+ "n_val": 246,
218
+ "formats": {
219
+ "choice": 4118,
220
+ "generate": 800,
221
+ "generate_cot": 800
222
+ },
223
+ "max_train_tokens": 1078
224
+ },
225
+ "steps": 714,
226
+ "final": {
227
+ "step": 714,
228
+ "lr": 1.0000461065451529e-05,
229
+ "grad_norm": 4.645827770233154,
230
+ "seconds": 7.537588600000163,
231
+ "loss": 0.04994113209795614,
232
+ "loss_choice": 0.038554409009520896,
233
+ "loss_generate": 0.12964819371700287,
234
+ "train_choice_accuracy": 1.0,
235
+ "peak_vram_mb": 9340.74267578125,
236
+ "val_ce_choice": 0.35926737235499373,
237
+ "val_ce_generate": 0.07934580545980907,
238
+ "val_ce_generate_cot": 0.031363994781941074,
239
+ "val_choice_accuracy": 0.8681318681318682
240
+ },
241
+ "hardware": {
242
+ "platform": "Windows-11-10.0.26200-SP0",
243
+ "python": "3.12.10",
244
+ "cpu": "AMD64 Family 26 Model 36 Stepping 0, AuthenticAMD",
245
+ "cpu_count": 20,
246
+ "packages": {
247
+ "torch": "2.11.0+cu128",
248
+ "transformers": "5.17.0",
249
+ "peft": "0.21.1",
250
+ "accelerate": "1.15.0",
251
+ "bitsandbytes": "0.50.2",
252
+ "safetensors": "0.8.0",
253
+ "tokenizers": "0.23.2"
254
+ },
255
+ "gpu": {
256
+ "name": "NVIDIA GeForce RTX 5080 Laptop GPU",
257
+ "memory_total_mib": 16303,
258
+ "memory_used_mib": 10742,
259
+ "utilization_pct": 5,
260
+ "driver_version": "572.76"
261
+ },
262
+ "ram_total_gb": 31.1,
263
+ "ram_available_gb": 14.3,
264
+ "torch_cuda_available": true,
265
+ "torch_cuda_version": "12.8"
266
+ },
267
+ "wall_time_s": 6014.2382328,
268
+ "adapter": "adapter"
269
+ }