---
library_name: transformers
base_model:
- vllm-sr/d3-turbo
---
This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from [vllm-sr/d3-turbo](https://huggingface.co/vllm-sr/d3-turbo).
Structural comparison with `vllm-sr/d3-turbo`:
| Feature | Source | Tiny |
|---|---:|---:|
| Task / public interface | D3 System One decision | same `system_one()` interface |
| Input modalities | text, image, video | same |
| Vocabulary rows / embedding dtype | 248,320 / BF16 | same |
| Text layers and cadence | 64; 3 linear + 1 full | 8; two cycles |
| Text width / attention head dimension | 5,120 / 256 | 32 / 32 |
| Linear-attention K/V heads and dimensions | 16 / 48; 128 / 128 | 1 / 1; 32 / 32 |
| Text MLP width | 17,408 | 64 |
| Vision depth / width / output width | 27 / 1,152 / 5,120 | 2 / 32 / 32 |
| Decision readout | 255-way FP32 | same |
| Stored precision | BF16 backbone + FP32 readout | same |
This is a D3 decision checkpoint, not a text-generation checkpoint. Its
source runtime supplies prefix-bidirectional full attention on top of the
causal Gated DeltaNet layers, and the tiny package preserves its raw runtime,
decision configuration, tokenizer, image processor, and video processor.
| File path | Size |
|------|------|
| model.safetensors | 16.4MB |
| readout.safetensors | 0.0MB |
### Example usage:
- Transformers (✅ verified)
```python
import numpy as np
import torch
from PIL import Image
from transformers import AutoModel
model_id = "tiny-random/d3-turbo"
# The pinned D3 runtime calls torch.cuda.set_device() during construction,
# which requires an indexed CUDA device rather than the shorthand "cuda".
device = "cuda:0" if torch.cuda.is_available() else "cpu"
model = AutoModel.from_pretrained(
model_id, trust_remote_code=True, device=device, verify="none",
).eval()
image = Image.new("RGB", (64, 64), "red")
video = np.stack([
np.asarray(Image.new("RGB", (64, 64), color))
for color in ("red", "green", "blue", "yellow")
])
result = model.system_one(
state="A small red square changes color over time.",
questions={
"route": {
"type": "choice",
"instructions": "Choose the better description.",
"criteria": {"still": "a still red image", "motion": "a changing video"},
}
},
images=[image],
videos=[video],
)
print(result)
```
### Codes to create this repo (✅ verified):
Click to expand
```python
import json
from pathlib import Path
from shutil import copyfile
import torch
from huggingface_hub import hf_hub_download, list_repo_files
from safetensors.torch import save_file
from transformers import AutoConfig, Qwen3_5Model, set_seed
source_model_id = "vllm-sr/d3-turbo"
save_folder = "/tmp/tiny-random/d3-turbo"
source_revision = "a0f3e9e59ebd1202fe1d4de1c4c0f9a986758cbb"
set_seed(42)
folder = Path(save_folder)
folder.mkdir(parents=True, exist_ok=True)
raw = json.loads(Path(hf_hub_download(
source_model_id, "config.json", revision=source_revision,
)).read_text(encoding="utf-8"))
text = raw["text_config"]
# D3 uses the stock Qwen3.5 backbone directly: retain every one of its
# 64 layers and its three linear-attention then one full-attention cadence.
text.update(
hidden_size=32,
head_dim=32,
num_attention_heads=1,
num_key_value_heads=1,
intermediate_size=64,
linear_key_head_dim=32,
linear_value_head_dim=32,
linear_num_key_heads=1,
linear_num_value_heads=1,
num_hidden_layers=8,
layer_types=["linear_attention", "linear_attention", "linear_attention", "full_attention"] * 2,
)
text["rope_parameters"]["mrope_section"] = [1, 1, 2]
raw["vision_config"].update(
hidden_size=32,
intermediate_size=64,
num_heads=1,
out_hidden_size=32,
depth=2,
)
(folder / "config.json").write_text(json.dumps(raw, indent=2), encoding="utf-8")
config = AutoConfig.from_pretrained(folder)
model = Qwen3_5Model(config).to(torch.bfloat16)
with torch.no_grad():
for name, parameter in sorted(model.named_parameters()):
if not parameter.is_floating_point():
continue
# Qwen3.5 RMSNorm implements (1 + weight); its source checkpoint
# stores these parameters in BF16 around the zero-centered form.
if name.startswith("language_model") and (
name.endswith("input_layernorm.weight")
or name.endswith("post_attention_layernorm.weight")
or name.endswith(".linear_attn.norm.weight")
or name.endswith("language_model.norm.weight")
or name.endswith(".self_attn.q_norm.weight")
or name.endswith(".self_attn.k_norm.weight")
):
parameter.zero_()
elif name.endswith((".linear_attn.A_log", ".linear_attn.dt_bias")):
# Keep the native DeltaNet initialization: it is not an
# ordinary projection weight and controls recurrent decay.
continue
elif ".norm" in name and name.endswith("weight"):
parameter.fill_(1)
else:
torch.nn.init.normal_(parameter, mean=0.0, std=0.02)
# The D3 package stores its decision head separately from the backbone.
# Build a real module tree solely for the README's faithful model printout,
# then serialize the exact source layout below.
package = torch.nn.Module()
package.backbone = model
package.readout = torch.nn.Linear(text["hidden_size"], 255, bias=False, dtype=torch.float32)
torch.nn.init.normal_(package.readout.weight, mean=0.0, std=0.02)
model.save_pretrained(folder, safe_serialization=True)
save_file({"weight": package.readout.weight.detach().cpu().contiguous()}, folder / "readout.safetensors")
# The pinned D3 runtime, tokenizer, chat template, and image/video
# processor are part of the executable source interface. Copy them as raw
# bytes after saving so none of the source assets is reserialized.
assets = {
"chat_template.jinja", "d3_compose.py", "d3_engine.py", "d3_fast.py",
"d3_format.py", "d3_kernels.py", "d3_runtime.py", "d3_server.py",
"decision_config.json", "merges.txt", "modeling_d3.py", "pipeline_d3.py",
"preprocessor_config.json", "requirements.txt", "tokenizer.json",
"tokenizer_config.json", "video_preprocessor_config.json", "vocab.json",
}
for filename in sorted(set(list_repo_files(source_model_id, revision=source_revision)) & assets):
copyfile(
hf_hub_download(source_model_id, filename, revision=source_revision),
folder / filename,
)
```
Click to expand
```text
Module(
(backbone): Qwen3_5Model(
(visual): Qwen3_5VisionModel(
(patch_embed): Qwen3_5VisionPatchEmbed(
(proj): Conv3d(3, 32, kernel_size=(2, 16, 16), stride=(2, 16, 16))
)
(pos_embed): Embedding(2304, 32)
(rotary_pos_emb): Qwen3_5VisionRotaryEmbedding()
(blocks): ModuleList(
(0-1): 2 x Qwen3_5VisionBlock(
(norm1): LayerNorm((32,), eps=1e-06, elementwise_affine=True, bias=True)
(norm2): LayerNorm((32,), eps=1e-06, elementwise_affine=True, bias=True)
(attn): Qwen3_5VisionAttention(
(qkv): Linear(in_features=32, out_features=96, bias=True)
(proj): Linear(in_features=32, out_features=32, bias=True)
)
(mlp): Qwen3_5VisionMLP(
(linear_fc1): Linear(in_features=32, out_features=64, bias=True)
(linear_fc2): Linear(in_features=64, out_features=32, bias=True)
(act_fn): GELUTanh()
)
)
)
(merger): Qwen3_5VisionPatchMerger(
(norm): LayerNorm((32,), eps=1e-06, elementwise_affine=True, bias=True)
(linear_fc1): Linear(in_features=128, out_features=128, bias=True)
(act_fn): GELU(approximate='none')
(linear_fc2): Linear(in_features=128, out_features=32, bias=True)
)
)
(language_model): Qwen3_5TextModel(
(embed_tokens): Embedding(248320, 32)
(layers): ModuleList(
(0-2): 3 x Qwen3_5DecoderLayer(
(linear_attn): Qwen3_5GatedDeltaNet(
(conv1d): Conv1d(96, 96, kernel_size=(4,), stride=(1,), padding=(3,), groups=96, bias=False)
(norm): Qwen3_5RMSNormGated()
(out_proj): Linear(in_features=32, out_features=32, bias=False)
(in_proj_qkv): Linear(in_features=32, out_features=96, bias=False)
(in_proj_z): Linear(in_features=32, out_features=32, bias=False)
(in_proj_b): Linear(in_features=32, out_features=1, bias=False)
(in_proj_a): Linear(in_features=32, out_features=1, bias=False)
)
(mlp): Qwen3_5MLP(
(gate_proj): Linear(in_features=32, out_features=64, bias=False)
(up_proj): Linear(in_features=32, out_features=64, bias=False)
(down_proj): Linear(in_features=64, out_features=32, bias=False)
(act_fn): SiLUActivation()
)
(input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
(post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
)
(3): Qwen3_5DecoderLayer(
(self_attn): Qwen3_5Attention(
(q_proj): Linear(in_features=32, out_features=64, bias=False)
(k_proj): Linear(in_features=32, out_features=32, bias=False)
(v_proj): Linear(in_features=32, out_features=32, bias=False)
(o_proj): Linear(in_features=32, out_features=32, bias=False)
(q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
(k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
)
(mlp): Qwen3_5MLP(
(gate_proj): Linear(in_features=32, out_features=64, bias=False)
(up_proj): Linear(in_features=32, out_features=64, bias=False)
(down_proj): Linear(in_features=64, out_features=32, bias=False)
(act_fn): SiLUActivation()
)
(input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
(post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
)
(4-6): 3 x Qwen3_5DecoderLayer(
(linear_attn): Qwen3_5GatedDeltaNet(
(conv1d): Conv1d(96, 96, kernel_size=(4,), stride=(1,), padding=(3,), groups=96, bias=False)
(norm): Qwen3_5RMSNormGated()
(out_proj): Linear(in_features=32, out_features=32, bias=False)
(in_proj_qkv): Linear(in_features=32, out_features=96, bias=False)
(in_proj_z): Linear(in_features=32, out_features=32, bias=False)
(in_proj_b): Linear(in_features=32, out_features=1, bias=False)
(in_proj_a): Linear(in_features=32, out_features=1, bias=False)
)
(mlp): Qwen3_5MLP(
(gate_proj): Linear(in_features=32, out_features=64, bias=False)
(up_proj): Linear(in_features=32, out_features=64, bias=False)
(down_proj): Linear(in_features=64, out_features=32, bias=False)
(act_fn): SiLUActivation()
)
(input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
(post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
)
(7): Qwen3_5DecoderLayer(
(self_attn): Qwen3_5Attention(
(q_proj): Linear(in_features=32, out_features=64, bias=False)
(k_proj): Linear(in_features=32, out_features=32, bias=False)
(v_proj): Linear(in_features=32, out_features=32, bias=False)
(o_proj): Linear(in_features=32, out_features=32, bias=False)
(q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
(k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
)
(mlp): Qwen3_5MLP(
(gate_proj): Linear(in_features=32, out_features=64, bias=False)
(up_proj): Linear(in_features=32, out_features=64, bias=False)
(down_proj): Linear(in_features=64, out_features=32, bias=False)
(act_fn): SiLUActivation()
)
(input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
(post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
)
)
(norm): Qwen3_5RMSNorm((32,), eps=1e-06)
(rotary_emb): Qwen3_5TextRotaryEmbedding()
)
)
(readout): Linear(in_features=32, out_features=255, bias=False)
)
```