Hugging Face's logo

tiny-random
/
d3-turbo

Feature Extraction
Transformers
Safetensors
qwen3_5
custom_code
Model card Files
xet
 Community
yujiepan commited on
Commit
fbf5df9
·
verified ·
1 Parent(s): 38f1b61

Upload folder using huggingface_hub

Browse files
Files changed (5) hide show
  1. README.md +9 -527
  2. __pycache__/d3_fast.cpython-312.pyc +0 -0
  3. config.json +2 -58
  4. model.safetensors +2 -2
  5. readout.safetensors +1 -1
README.md CHANGED
@@ -13,11 +13,11 @@ Structural comparison with `vllm-sr/d3-turbo`:
13
  | Task / public interface | D3 System One decision | same `system_one()` interface |
14
  | Input modalities | text, image, video | same |
15
  | Vocabulary rows / embedding dtype | 248,320 / BF16 | same |
16
- | Text layers and cadence | 64; 3 linear + 1 full | same |
17
  | Text width / attention head dimension | 5,120 / 256 | 32 / 32 |
18
  | Linear-attention K/V heads and dimensions | 16 / 48; 128 / 128 | 1 / 1; 32 / 32 |
19
  | Text MLP width | 17,408 | 64 |
20
- | Vision depth / width / output width | 27 / 1,152 / 5,120 | 27 / 32 / 32 |
21
  | Decision readout | 255-way FP32 | same |
22
  | Stored precision | BF16 backbone + FP32 readout | same |
23
 
@@ -28,7 +28,7 @@ decision configuration, tokenizer, image processor, and video processor.
28
 
29
  | File path | Size |
30
  |------|------|
31
- | model.safetensors | 18.3MB |
32
  | readout.safetensors | 0.0MB |
33
 
34
 
@@ -42,7 +42,7 @@ import torch
42
  from PIL import Image
43
  from transformers import AutoModel
44
 
45
- model_id = "tiny-random/d3-qwen3.5-vl"
46
  # The pinned D3 runtime calls torch.cuda.set_device() during construction,
47
  # which requires an indexed CUDA device rather than the shorthand "cuda".
48
  device = "cuda:0" if torch.cuda.is_available() else "cpu"
@@ -84,7 +84,7 @@ from safetensors.torch import save_file
84
  from transformers import AutoConfig, Qwen3_5Model, set_seed
85
 
86
  source_model_id = "vllm-sr/d3-turbo"
87
- save_folder = "/tmp/tiny-random/d3-qwen35-vl"
88
  source_revision = "a0f3e9e59ebd1202fe1d4de1c4c0f9a986758cbb"
89
  set_seed(42)
90
  folder = Path(save_folder)
@@ -105,8 +105,8 @@ text.update(
105
  linear_value_head_dim=32,
106
  linear_num_key_heads=1,
107
  linear_num_value_heads=1,
108
- num_hidden_layers=64,
109
- layer_types=["linear_attention", "linear_attention", "linear_attention", "full_attention"] * 16,
110
  )
111
  text["rope_parameters"]["mrope_section"] = [1, 1, 2]
112
  raw["vision_config"].update(
@@ -114,7 +114,7 @@ raw["vision_config"].update(
114
  intermediate_size=64,
115
  num_heads=1,
116
  out_hidden_size=32,
117
- depth=27,
118
  )
119
  (folder / "config.json").write_text(json.dumps(raw, indent=2), encoding="utf-8")
120
  config = AutoConfig.from_pretrained(folder)
@@ -186,7 +186,7 @@ Module(
186
  (pos_embed): Embedding(2304, 32)
187
  (rotary_pos_emb): Qwen3_5VisionRotaryEmbedding()
188
  (blocks): ModuleList(
189
- (0-26): 27 x Qwen3_5VisionBlock(
190
  (norm1): LayerNorm((32,), eps=1e-06, elementwise_affine=True, bias=True)
191
  (norm2): LayerNorm((32,), eps=1e-06, elementwise_affine=True, bias=True)
192
  (attn): Qwen3_5VisionAttention(
@@ -284,524 +284,6 @@ Module(
284
  (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
285
  (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
286
  )
287
- (8-10): 3 x Qwen3_5DecoderLayer(
288
- (linear_attn): Qwen3_5GatedDeltaNet(
289
- (conv1d): Conv1d(96, 96, kernel_size=(4,), stride=(1,), padding=(3,), groups=96, bias=False)
290
- (norm): Qwen3_5RMSNormGated()
291
- (out_proj): Linear(in_features=32, out_features=32, bias=False)
292
- (in_proj_qkv): Linear(in_features=32, out_features=96, bias=False)
293
- (in_proj_z): Linear(in_features=32, out_features=32, bias=False)
294
- (in_proj_b): Linear(in_features=32, out_features=1, bias=False)
295
- (in_proj_a): Linear(in_features=32, out_features=1, bias=False)
296
- )
297
- (mlp): Qwen3_5MLP(
298
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
299
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
300
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
301
- (act_fn): SiLUActivation()
302
- )
303
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
304
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
305
- )
306
- (11): Qwen3_5DecoderLayer(
307
- (self_attn): Qwen3_5Attention(
308
- (q_proj): Linear(in_features=32, out_features=64, bias=False)
309
- (k_proj): Linear(in_features=32, out_features=32, bias=False)
310
- (v_proj): Linear(in_features=32, out_features=32, bias=False)
311
- (o_proj): Linear(in_features=32, out_features=32, bias=False)
312
- (q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
313
- (k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
314
- )
315
- (mlp): Qwen3_5MLP(
316
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
317
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
318
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
319
- (act_fn): SiLUActivation()
320
- )
321
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
322
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
323
- )
324
- (12-14): 3 x Qwen3_5DecoderLayer(
325
- (linear_attn): Qwen3_5GatedDeltaNet(
326
- (conv1d): Conv1d(96, 96, kernel_size=(4,), stride=(1,), padding=(3,), groups=96, bias=False)
327
- (norm): Qwen3_5RMSNormGated()
328
- (out_proj): Linear(in_features=32, out_features=32, bias=False)
329
- (in_proj_qkv): Linear(in_features=32, out_features=96, bias=False)
330
- (in_proj_z): Linear(in_features=32, out_features=32, bias=False)
331
- (in_proj_b): Linear(in_features=32, out_features=1, bias=False)
332
- (in_proj_a): Linear(in_features=32, out_features=1, bias=False)
333
- )
334
- (mlp): Qwen3_5MLP(
335
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
336
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
337
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
338
- (act_fn): SiLUActivation()
339
- )
340
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
341
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
342
- )
343
- (15): Qwen3_5DecoderLayer(
344
- (self_attn): Qwen3_5Attention(
345
- (q_proj): Linear(in_features=32, out_features=64, bias=False)
346
- (k_proj): Linear(in_features=32, out_features=32, bias=False)
347
- (v_proj): Linear(in_features=32, out_features=32, bias=False)
348
- (o_proj): Linear(in_features=32, out_features=32, bias=False)
349
- (q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
350
- (k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
351
- )
352
- (mlp): Qwen3_5MLP(
353
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
354
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
355
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
356
- (act_fn): SiLUActivation()
357
- )
358
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
359
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
360
- )
361
- (16-18): 3 x Qwen3_5DecoderLayer(
362
- (linear_attn): Qwen3_5GatedDeltaNet(
363
- (conv1d): Conv1d(96, 96, kernel_size=(4,), stride=(1,), padding=(3,), groups=96, bias=False)
364
- (norm): Qwen3_5RMSNormGated()
365
- (out_proj): Linear(in_features=32, out_features=32, bias=False)
366
- (in_proj_qkv): Linear(in_features=32, out_features=96, bias=False)
367
- (in_proj_z): Linear(in_features=32, out_features=32, bias=False)
368
- (in_proj_b): Linear(in_features=32, out_features=1, bias=False)
369
- (in_proj_a): Linear(in_features=32, out_features=1, bias=False)
370
- )
371
- (mlp): Qwen3_5MLP(
372
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
373
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
374
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
375
- (act_fn): SiLUActivation()
376
- )
377
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
378
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
379
- )
380
- (19): Qwen3_5DecoderLayer(
381
- (self_attn): Qwen3_5Attention(
382
- (q_proj): Linear(in_features=32, out_features=64, bias=False)
383
- (k_proj): Linear(in_features=32, out_features=32, bias=False)
384
- (v_proj): Linear(in_features=32, out_features=32, bias=False)
385
- (o_proj): Linear(in_features=32, out_features=32, bias=False)
386
- (q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
387
- (k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
388
- )
389
- (mlp): Qwen3_5MLP(
390
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
391
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
392
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
393
- (act_fn): SiLUActivation()
394
- )
395
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
396
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
397
- )
398
- (20-22): 3 x Qwen3_5DecoderLayer(
399
- (linear_attn): Qwen3_5GatedDeltaNet(
400
- (conv1d): Conv1d(96, 96, kernel_size=(4,), stride=(1,), padding=(3,), groups=96, bias=False)
401
- (norm): Qwen3_5RMSNormGated()
402
- (out_proj): Linear(in_features=32, out_features=32, bias=False)
403
- (in_proj_qkv): Linear(in_features=32, out_features=96, bias=False)
404
- (in_proj_z): Linear(in_features=32, out_features=32, bias=False)
405
- (in_proj_b): Linear(in_features=32, out_features=1, bias=False)
406
- (in_proj_a): Linear(in_features=32, out_features=1, bias=False)
407
- )
408
- (mlp): Qwen3_5MLP(
409
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
410
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
411
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
412
- (act_fn): SiLUActivation()
413
- )
414
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
415
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
416
- )
417
- (23): Qwen3_5DecoderLayer(
418
- (self_attn): Qwen3_5Attention(
419
- (q_proj): Linear(in_features=32, out_features=64, bias=False)
420
- (k_proj): Linear(in_features=32, out_features=32, bias=False)
421
- (v_proj): Linear(in_features=32, out_features=32, bias=False)
422
- (o_proj): Linear(in_features=32, out_features=32, bias=False)
423
- (q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
424
- (k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
425
- )
426
- (mlp): Qwen3_5MLP(
427
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
428
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
429
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
430
- (act_fn): SiLUActivation()
431
- )
432
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
433
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
434
- )
435
- (24-26): 3 x Qwen3_5DecoderLayer(
436
- (linear_attn): Qwen3_5GatedDeltaNet(
437
- (conv1d): Conv1d(96, 96, kernel_size=(4,), stride=(1,), padding=(3,), groups=96, bias=False)
438
- (norm): Qwen3_5RMSNormGated()
439
- (out_proj): Linear(in_features=32, out_features=32, bias=False)
440
- (in_proj_qkv): Linear(in_features=32, out_features=96, bias=False)
441
- (in_proj_z): Linear(in_features=32, out_features=32, bias=False)
442
- (in_proj_b): Linear(in_features=32, out_features=1, bias=False)
443
- (in_proj_a): Linear(in_features=32, out_features=1, bias=False)
444
- )
445
- (mlp): Qwen3_5MLP(
446
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
447
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
448
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
449
- (act_fn): SiLUActivation()
450
- )
451
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
452
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
453
- )
454
- (27): Qwen3_5DecoderLayer(
455
- (self_attn): Qwen3_5Attention(
456
- (q_proj): Linear(in_features=32, out_features=64, bias=False)
457
- (k_proj): Linear(in_features=32, out_features=32, bias=False)
458
- (v_proj): Linear(in_features=32, out_features=32, bias=False)
459
- (o_proj): Linear(in_features=32, out_features=32, bias=False)
460
- (q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
461
- (k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
462
- )
463
- (mlp): Qwen3_5MLP(
464
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
465
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
466
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
467
- (act_fn): SiLUActivation()
468
- )
469
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
470
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
471
- )
472
- (28-30): 3 x Qwen3_5DecoderLayer(
473
- (linear_attn): Qwen3_5GatedDeltaNet(
474
- (conv1d): Conv1d(96, 96, kernel_size=(4,), stride=(1,), padding=(3,), groups=96, bias=False)
475
- (norm): Qwen3_5RMSNormGated()
476
- (out_proj): Linear(in_features=32, out_features=32, bias=False)
477
- (in_proj_qkv): Linear(in_features=32, out_features=96, bias=False)
478
- (in_proj_z): Linear(in_features=32, out_features=32, bias=False)
479
- (in_proj_b): Linear(in_features=32, out_features=1, bias=False)
480
- (in_proj_a): Linear(in_features=32, out_features=1, bias=False)
481
- )
482
- (mlp): Qwen3_5MLP(
483
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
484
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
485
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
486
- (act_fn): SiLUActivation()
487
- )
488
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
489
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
490
- )
491
- (31): Qwen3_5DecoderLayer(
492
- (self_attn): Qwen3_5Attention(
493
- (q_proj): Linear(in_features=32, out_features=64, bias=False)
494
- (k_proj): Linear(in_features=32, out_features=32, bias=False)
495
- (v_proj): Linear(in_features=32, out_features=32, bias=False)
496
- (o_proj): Linear(in_features=32, out_features=32, bias=False)
497
- (q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
498
- (k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
499
- )
500
- (mlp): Qwen3_5MLP(
501
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
502
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
503
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
504
- (act_fn): SiLUActivation()
505
- )
506
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
507
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
508
- )
509
- (32-34): 3 x Qwen3_5DecoderLayer(
510
- (linear_attn): Qwen3_5GatedDeltaNet(
511
- (conv1d): Conv1d(96, 96, kernel_size=(4,), stride=(1,), padding=(3,), groups=96, bias=False)
512
- (norm): Qwen3_5RMSNormGated()
513
- (out_proj): Linear(in_features=32, out_features=32, bias=False)
514
- (in_proj_qkv): Linear(in_features=32, out_features=96, bias=False)
515
- (in_proj_z): Linear(in_features=32, out_features=32, bias=False)
516
- (in_proj_b): Linear(in_features=32, out_features=1, bias=False)
517
- (in_proj_a): Linear(in_features=32, out_features=1, bias=False)
518
- )
519
- (mlp): Qwen3_5MLP(
520
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
521
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
522
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
523
- (act_fn): SiLUActivation()
524
- )
525
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
526
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
527
- )
528
- (35): Qwen3_5DecoderLayer(
529
- (self_attn): Qwen3_5Attention(
530
- (q_proj): Linear(in_features=32, out_features=64, bias=False)
531
- (k_proj): Linear(in_features=32, out_features=32, bias=False)
532
- (v_proj): Linear(in_features=32, out_features=32, bias=False)
533
- (o_proj): Linear(in_features=32, out_features=32, bias=False)
534
- (q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
535
- (k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
536
- )
537
- (mlp): Qwen3_5MLP(
538
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
539
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
540
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
541
- (act_fn): SiLUActivation()
542
- )
543
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
544
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
545
- )
546
- (36-38): 3 x Qwen3_5DecoderLayer(
547
- (linear_attn): Qwen3_5GatedDeltaNet(
548
- (conv1d): Conv1d(96, 96, kernel_size=(4,), stride=(1,), padding=(3,), groups=96, bias=False)
549
- (norm): Qwen3_5RMSNormGated()
550
- (out_proj): Linear(in_features=32, out_features=32, bias=False)
551
- (in_proj_qkv): Linear(in_features=32, out_features=96, bias=False)
552
- (in_proj_z): Linear(in_features=32, out_features=32, bias=False)
553
- (in_proj_b): Linear(in_features=32, out_features=1, bias=False)
554
- (in_proj_a): Linear(in_features=32, out_features=1, bias=False)
555
- )
556
- (mlp): Qwen3_5MLP(
557
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
558
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
559
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
560
- (act_fn): SiLUActivation()
561
- )
562
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
563
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
564
- )
565
- (39): Qwen3_5DecoderLayer(
566
- (self_attn): Qwen3_5Attention(
567
- (q_proj): Linear(in_features=32, out_features=64, bias=False)
568
- (k_proj): Linear(in_features=32, out_features=32, bias=False)
569
- (v_proj): Linear(in_features=32, out_features=32, bias=False)
570
- (o_proj): Linear(in_features=32, out_features=32, bias=False)
571
- (q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
572
- (k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
573
- )
574
- (mlp): Qwen3_5MLP(
575
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
576
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
577
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
578
- (act_fn): SiLUActivation()
579
- )
580
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
581
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
582
- )
583
- (40-42): 3 x Qwen3_5DecoderLayer(
584
- (linear_attn): Qwen3_5GatedDeltaNet(
585
- (conv1d): Conv1d(96, 96, kernel_size=(4,), stride=(1,), padding=(3,), groups=96, bias=False)
586
- (norm): Qwen3_5RMSNormGated()
587
- (out_proj): Linear(in_features=32, out_features=32, bias=False)
588
- (in_proj_qkv): Linear(in_features=32, out_features=96, bias=False)
589
- (in_proj_z): Linear(in_features=32, out_features=32, bias=False)
590
- (in_proj_b): Linear(in_features=32, out_features=1, bias=False)
591
- (in_proj_a): Linear(in_features=32, out_features=1, bias=False)
592
- )
593
- (mlp): Qwen3_5MLP(
594
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
595
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
596
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
597
- (act_fn): SiLUActivation()
598
- )
599
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
600
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
601
- )
602
- (43): Qwen3_5DecoderLayer(
603
- (self_attn): Qwen3_5Attention(
604
- (q_proj): Linear(in_features=32, out_features=64, bias=False)
605
- (k_proj): Linear(in_features=32, out_features=32, bias=False)
606
- (v_proj): Linear(in_features=32, out_features=32, bias=False)
607
- (o_proj): Linear(in_features=32, out_features=32, bias=False)
608
- (q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
609
- (k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
610
- )
611
- (mlp): Qwen3_5MLP(
612
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
613
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
614
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
615
- (act_fn): SiLUActivation()
616
- )
617
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
618
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
619
- )
620
- (44-46): 3 x Qwen3_5DecoderLayer(
621
- (linear_attn): Qwen3_5GatedDeltaNet(
622
- (conv1d): Conv1d(96, 96, kernel_size=(4,), stride=(1,), padding=(3,), groups=96, bias=False)
623
- (norm): Qwen3_5RMSNormGated()
624
- (out_proj): Linear(in_features=32, out_features=32, bias=False)
625
- (in_proj_qkv): Linear(in_features=32, out_features=96, bias=False)
626
- (in_proj_z): Linear(in_features=32, out_features=32, bias=False)
627
- (in_proj_b): Linear(in_features=32, out_features=1, bias=False)
628
- (in_proj_a): Linear(in_features=32, out_features=1, bias=False)
629
- )
630
- (mlp): Qwen3_5MLP(
631
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
632
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
633
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
634
- (act_fn): SiLUActivation()
635
- )
636
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
637
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
638
- )
639
- (47): Qwen3_5DecoderLayer(
640
- (self_attn): Qwen3_5Attention(
641
- (q_proj): Linear(in_features=32, out_features=64, bias=False)
642
- (k_proj): Linear(in_features=32, out_features=32, bias=False)
643
- (v_proj): Linear(in_features=32, out_features=32, bias=False)
644
- (o_proj): Linear(in_features=32, out_features=32, bias=False)
645
- (q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
646
- (k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
647
- )
648
- (mlp): Qwen3_5MLP(
649
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
650
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
651
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
652
- (act_fn): SiLUActivation()
653
- )
654
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
655
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
656
- )
657
- (48-50): 3 x Qwen3_5DecoderLayer(
658
- (linear_attn): Qwen3_5GatedDeltaNet(
659
- (conv1d): Conv1d(96, 96, kernel_size=(4,), stride=(1,), padding=(3,), groups=96, bias=False)
660
- (norm): Qwen3_5RMSNormGated()
661
- (out_proj): Linear(in_features=32, out_features=32, bias=False)
662
- (in_proj_qkv): Linear(in_features=32, out_features=96, bias=False)
663
- (in_proj_z): Linear(in_features=32, out_features=32, bias=False)
664
- (in_proj_b): Linear(in_features=32, out_features=1, bias=False)
665
- (in_proj_a): Linear(in_features=32, out_features=1, bias=False)
666
- )
667
- (mlp): Qwen3_5MLP(
668
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
669
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
670
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
671
- (act_fn): SiLUActivation()
672
- )
673
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
674
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
675
- )
676
- (51): Qwen3_5DecoderLayer(
677
- (self_attn): Qwen3_5Attention(
678
- (q_proj): Linear(in_features=32, out_features=64, bias=False)
679
- (k_proj): Linear(in_features=32, out_features=32, bias=False)
680
- (v_proj): Linear(in_features=32, out_features=32, bias=False)
681
- (o_proj): Linear(in_features=32, out_features=32, bias=False)
682
- (q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
683
- (k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
684
- )
685
- (mlp): Qwen3_5MLP(
686
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
687
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
688
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
689
- (act_fn): SiLUActivation()
690
- )
691
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
692
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
693
- )
694
- (52-54): 3 x Qwen3_5DecoderLayer(
695
- (linear_attn): Qwen3_5GatedDeltaNet(
696
- (conv1d): Conv1d(96, 96, kernel_size=(4,), stride=(1,), padding=(3,), groups=96, bias=False)
697
- (norm): Qwen3_5RMSNormGated()
698
- (out_proj): Linear(in_features=32, out_features=32, bias=False)
699
- (in_proj_qkv): Linear(in_features=32, out_features=96, bias=False)
700
- (in_proj_z): Linear(in_features=32, out_features=32, bias=False)
701
- (in_proj_b): Linear(in_features=32, out_features=1, bias=False)
702
- (in_proj_a): Linear(in_features=32, out_features=1, bias=False)
703
- )
704
- (mlp): Qwen3_5MLP(
705
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
706
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
707
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
708
- (act_fn): SiLUActivation()
709
- )
710
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
711
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
712
- )
713
- (55): Qwen3_5DecoderLayer(
714
- (self_attn): Qwen3_5Attention(
715
- (q_proj): Linear(in_features=32, out_features=64, bias=False)
716
- (k_proj): Linear(in_features=32, out_features=32, bias=False)
717
- (v_proj): Linear(in_features=32, out_features=32, bias=False)
718
- (o_proj): Linear(in_features=32, out_features=32, bias=False)
719
- (q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
720
- (k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
721
- )
722
- (mlp): Qwen3_5MLP(
723
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
724
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
725
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
726
- (act_fn): SiLUActivation()
727
- )
728
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
729
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
730
- )
731
- (56-58): 3 x Qwen3_5DecoderLayer(
732
- (linear_attn): Qwen3_5GatedDeltaNet(
733
- (conv1d): Conv1d(96, 96, kernel_size=(4,), stride=(1,), padding=(3,), groups=96, bias=False)
734
- (norm): Qwen3_5RMSNormGated()
735
- (out_proj): Linear(in_features=32, out_features=32, bias=False)
736
- (in_proj_qkv): Linear(in_features=32, out_features=96, bias=False)
737
- (in_proj_z): Linear(in_features=32, out_features=32, bias=False)
738
- (in_proj_b): Linear(in_features=32, out_features=1, bias=False)
739
- (in_proj_a): Linear(in_features=32, out_features=1, bias=False)
740
- )
741
- (mlp): Qwen3_5MLP(
742
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
743
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
744
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
745
- (act_fn): SiLUActivation()
746
- )
747
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
748
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
749
- )
750
- (59): Qwen3_5DecoderLayer(
751
- (self_attn): Qwen3_5Attention(
752
- (q_proj): Linear(in_features=32, out_features=64, bias=False)
753
- (k_proj): Linear(in_features=32, out_features=32, bias=False)
754
- (v_proj): Linear(in_features=32, out_features=32, bias=False)
755
- (o_proj): Linear(in_features=32, out_features=32, bias=False)
756
- (q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
757
- (k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
758
- )
759
- (mlp): Qwen3_5MLP(
760
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
761
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
762
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
763
- (act_fn): SiLUActivation()
764
- )
765
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
766
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
767
- )
768
- (60-62): 3 x Qwen3_5DecoderLayer(
769
- (linear_attn): Qwen3_5GatedDeltaNet(
770
- (conv1d): Conv1d(96, 96, kernel_size=(4,), stride=(1,), padding=(3,), groups=96, bias=False)
771
- (norm): Qwen3_5RMSNormGated()
772
- (out_proj): Linear(in_features=32, out_features=32, bias=False)
773
- (in_proj_qkv): Linear(in_features=32, out_features=96, bias=False)
774
- (in_proj_z): Linear(in_features=32, out_features=32, bias=False)
775
- (in_proj_b): Linear(in_features=32, out_features=1, bias=False)
776
- (in_proj_a): Linear(in_features=32, out_features=1, bias=False)
777
- )
778
- (mlp): Qwen3_5MLP(
779
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
780
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
781
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
782
- (act_fn): SiLUActivation()
783
- )
784
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
785
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
786
- )
787
- (63): Qwen3_5DecoderLayer(
788
- (self_attn): Qwen3_5Attention(
789
- (q_proj): Linear(in_features=32, out_features=64, bias=False)
790
- (k_proj): Linear(in_features=32, out_features=32, bias=False)
791
- (v_proj): Linear(in_features=32, out_features=32, bias=False)
792
- (o_proj): Linear(in_features=32, out_features=32, bias=False)
793
- (q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
794
- (k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
795
- )
796
- (mlp): Qwen3_5MLP(
797
- (gate_proj): Linear(in_features=32, out_features=64, bias=False)
798
- (up_proj): Linear(in_features=32, out_features=64, bias=False)
799
- (down_proj): Linear(in_features=64, out_features=32, bias=False)
800
- (act_fn): SiLUActivation()
801
- )
802
- (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
803
- (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
804
- )
805
  )
806
  (norm): Qwen3_5RMSNorm((32,), eps=1e-06)
807
  (rotary_emb): Qwen3_5TextRotaryEmbedding()
 
13
  | Task / public interface | D3 System One decision | same `system_one()` interface |
14
  | Input modalities | text, image, video | same |
15
  | Vocabulary rows / embedding dtype | 248,320 / BF16 | same |
16
+ | Text layers and cadence | 64; 3 linear + 1 full | 8; two cycles |
17
  | Text width / attention head dimension | 5,120 / 256 | 32 / 32 |
18
  | Linear-attention K/V heads and dimensions | 16 / 48; 128 / 128 | 1 / 1; 32 / 32 |
19
  | Text MLP width | 17,408 | 64 |
20
+ | Vision depth / width / output width | 27 / 1,152 / 5,120 | 2 / 32 / 32 |
21
  | Decision readout | 255-way FP32 | same |
22
  | Stored precision | BF16 backbone + FP32 readout | same |
23
 
 
28
 
29
  | File path | Size |
30
  |------|------|
31
+ | model.safetensors | 16.4MB |
32
  | readout.safetensors | 0.0MB |
33
 
34
 
 
42
  from PIL import Image
43
  from transformers import AutoModel
44
 
45
+ model_id = "tiny-random/d3-turbo"
46
  # The pinned D3 runtime calls torch.cuda.set_device() during construction,
47
  # which requires an indexed CUDA device rather than the shorthand "cuda".
48
  device = "cuda:0" if torch.cuda.is_available() else "cpu"
 
84
  from transformers import AutoConfig, Qwen3_5Model, set_seed
85
 
86
  source_model_id = "vllm-sr/d3-turbo"
87
+ save_folder = "/tmp/tiny-random/d3-turbo"
88
  source_revision = "a0f3e9e59ebd1202fe1d4de1c4c0f9a986758cbb"
89
  set_seed(42)
90
  folder = Path(save_folder)
 
105
  linear_value_head_dim=32,
106
  linear_num_key_heads=1,
107
  linear_num_value_heads=1,
108
+ num_hidden_layers=8,
109
+ layer_types=["linear_attention", "linear_attention", "linear_attention", "full_attention"] * 2,
110
  )
111
  text["rope_parameters"]["mrope_section"] = [1, 1, 2]
112
  raw["vision_config"].update(
 
114
  intermediate_size=64,
115
  num_heads=1,
116
  out_hidden_size=32,
117
+ depth=2,
118
  )
119
  (folder / "config.json").write_text(json.dumps(raw, indent=2), encoding="utf-8")
120
  config = AutoConfig.from_pretrained(folder)
 
186
  (pos_embed): Embedding(2304, 32)
187
  (rotary_pos_emb): Qwen3_5VisionRotaryEmbedding()
188
  (blocks): ModuleList(
189
+ (0-1): 2 x Qwen3_5VisionBlock(
190
  (norm1): LayerNorm((32,), eps=1e-06, elementwise_affine=True, bias=True)
191
  (norm2): LayerNorm((32,), eps=1e-06, elementwise_affine=True, bias=True)
192
  (attn): Qwen3_5VisionAttention(
 
284
  (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
285
  (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
286
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
287
  )
288
  (norm): Qwen3_5RMSNorm((32,), eps=1e-06)
289
  (rotary_emb): Qwen3_5TextRotaryEmbedding()
__pycache__/d3_fast.cpython-312.pyc ADDED
Binary file (34.3 kB). View file
 
config.json CHANGED
@@ -32,62 +32,6 @@
32
  "initializer_range": 0.02,
33
  "intermediate_size": 64,
34
  "layer_types": [
35
- "linear_attention",
36
- "linear_attention",
37
- "linear_attention",
38
- "full_attention",
39
- "linear_attention",
40
- "linear_attention",
41
- "linear_attention",
42
- "full_attention",
43
- "linear_attention",
44
- "linear_attention",
45
- "linear_attention",
46
- "full_attention",
47
- "linear_attention",
48
- "linear_attention",
49
- "linear_attention",
50
- "full_attention",
51
- "linear_attention",
52
- "linear_attention",
53
- "linear_attention",
54
- "full_attention",
55
- "linear_attention",
56
- "linear_attention",
57
- "linear_attention",
58
- "full_attention",
59
- "linear_attention",
60
- "linear_attention",
61
- "linear_attention",
62
- "full_attention",
63
- "linear_attention",
64
- "linear_attention",
65
- "linear_attention",
66
- "full_attention",
67
- "linear_attention",
68
- "linear_attention",
69
- "linear_attention",
70
- "full_attention",
71
- "linear_attention",
72
- "linear_attention",
73
- "linear_attention",
74
- "full_attention",
75
- "linear_attention",
76
- "linear_attention",
77
- "linear_attention",
78
- "full_attention",
79
- "linear_attention",
80
- "linear_attention",
81
- "linear_attention",
82
- "full_attention",
83
- "linear_attention",
84
- "linear_attention",
85
- "linear_attention",
86
- "full_attention",
87
- "linear_attention",
88
- "linear_attention",
89
- "linear_attention",
90
- "full_attention",
91
  "linear_attention",
92
  "linear_attention",
93
  "linear_attention",
@@ -108,7 +52,7 @@
108
  "mtp_num_hidden_layers": 1,
109
  "mtp_use_dedicated_embeddings": false,
110
  "num_attention_heads": 1,
111
- "num_hidden_layers": 64,
112
  "num_key_value_heads": 1,
113
  "output_gate_type": "swish",
114
  "pad_token_id": null,
@@ -134,7 +78,7 @@
134
  "video_token_id": 248057,
135
  "vision_config": {
136
  "deepstack_visual_indexes": [],
137
- "depth": 27,
138
  "hidden_act": "gelu_pytorch_tanh",
139
  "hidden_size": 32,
140
  "in_channels": 3,
 
32
  "initializer_range": 0.02,
33
  "intermediate_size": 64,
34
  "layer_types": [
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
35
  "linear_attention",
36
  "linear_attention",
37
  "linear_attention",
 
52
  "mtp_num_hidden_layers": 1,
53
  "mtp_use_dedicated_embeddings": false,
54
  "num_attention_heads": 1,
55
+ "num_hidden_layers": 8,
56
  "num_key_value_heads": 1,
57
  "output_gate_type": "swish",
58
  "pad_token_id": null,
 
78
  "video_token_id": 248057,
79
  "vision_config": {
80
  "deepstack_visual_indexes": [],
81
+ "depth": 2,
82
  "hidden_act": "gelu_pytorch_tanh",
83
  "hidden_size": 32,
84
  "in_channels": 3,
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:677c046ef1ec2a8e524b76206eb03c8dfdc93a43f27780e70a5560f37341eed2
3
- size 18273568
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3d811ceba2264b767198673f23a481b26e0f59ea61014e575628b4e65df7c797
3
+ size 16417224
readout.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9f344540d624b9a35c82a74c39004414201e4cfe2f54217f7c2bae39d1f43351
3
  size 32720
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f9f7b5611b1d3d2a29fd332972cf32631d3aab0cdd421191fa4c873818201c78
3
  size 32720