---
library_name: transformers
base_model:
- KRAFTON/Raon-1-mini-preview
---
This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from [KRAFTON/Raon-1-mini-preview](https://huggingface.co/KRAFTON/Raon-1-mini-preview).
Structural comparison with `KRAFTON/Raon-1-mini-preview`:
| Feature | Source | Tiny |
|---|---:|---:|
| Vocabulary / tied embeddings | 259,840 / untied | same |
| Layers | 25 | 8; two 3-RDA + 1-MLA cycles |
| RDA / NoPE MLA layer cadence | 18 / 7 | 6 / 2 |
| Raon mHC streams | 4 | same |
| Dense prefix / MoE layers | layer 0 / 24 layers | layer 0 / 7 layers |
| Routed experts / active experts | 128 / 6 | 64 / 6 |
| Hidden width / attention head dimension | 2,560 / 128 | 32 / 32 |
| RDA, MLA and LoRA dimensions | source widths | 32-wide compatible projections |
| Routed-expert storage | MXFP8 E4M3 + E8M0 1x32 scales | same |
| Router / shared expert / output gate | sigmoid top-k / present / present | same |
Transformers constructs Raon's architecture, then the definition rewrites only
routed-expert tensors to the source MXFP8 storage representation. This is
necessary because random CPU initialization of `float8_e4m3fn` parameters is
not implemented by PyTorch; it does not change the serialized precision.
| File path | Size |
|------|------|
| model.safetensors | 35.3MB |
### Example usage:
- Transformers (✅ verified)
```python
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
model_id = "tiny-random/raon-1-mini"
device = "cuda" if torch.cuda.is_available() else "cpu"
model = AutoModelForCausalLM.from_pretrained(model_id, trust_remote_code=True).to(device).eval()
tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
inputs = tokenizer("Hello", return_tensors="pt").to(device)
output = model.generate(**inputs, do_sample=False, min_new_tokens=10, max_new_tokens=10)
print(tokenizer.decode(output[0, inputs["input_ids"].shape[1]:], skip_special_tokens=True))
```
### Codes to create this repo (✅ verified):
Click to expand
```python
import json
from pathlib import Path
from shutil import copyfile
import torch
from huggingface_hub import hf_hub_download, list_repo_files
from safetensors.torch import save_file
from transformers import AutoConfig, AutoModelForCausalLM, set_seed
source_model_id = "KRAFTON/Raon-1-mini-preview"
save_folder = "/tmp/tiny-random/raon-1-mini"
source_revision = "7bf87c45fc7b658d3c9532305f3aaed106bbac4f"
set_seed(42)
folder = Path(save_folder)
folder.mkdir(parents=True, exist_ok=True)
raw = json.loads(Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text())
raw.update(
hidden_size=32, head_dim=32, num_attention_heads=1, num_hidden_layers=8, intermediate_size=128,
num_experts=64, num_experts_per_token=6, moe_intermediate_size=32,
q_lora_rank=32, kv_lora_rank=32, qk_nope_head_dim=32,
qk_rope_head_dim=32, v_head_dim=32, rda_decay_rank=32,
max_position_embeddings=4096,
)
raw["rda_config"].update(
num_heads=1, head_dim=32,
rda_layers=[1, 2, 3, 5, 6, 7], mla_layers=[4, 8],
)
# Build the public remote-code graph in BF16 first. The source's MXFP8
# modules cannot initialize random float8 parameters directly, then the
# routed experts are converted back to the source's E4M3 + E8M0 format.
construction_raw = json.loads(json.dumps(raw))
construction_raw["quantization_config"]["dequantize"] = True
(folder / "config.json").write_text(json.dumps(construction_raw, indent=2) + "\n", encoding="utf-8")
for filename in ("configuration_raon1_mini.py", "modeling_raon1_mini.py"):
copyfile(hf_hub_download(source_model_id, filename, revision=source_revision), folder / filename)
config = AutoConfig.from_pretrained(folder, trust_remote_code=True)
model = AutoModelForCausalLM.from_config(config, trust_remote_code=True).eval()
with torch.no_grad():
for name, parameter in sorted(model.named_parameters()):
if name.endswith(("weight", "phi_pre", "phi_post", "phi_z")) and not name.endswith("norm.weight"):
torch.nn.init.normal_(parameter, mean=0, std=0.02)
elif name.endswith("norm.weight") or name.endswith("layernorm.weight"):
parameter.fill_(1)
def quantize_mxfp8(weight: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
rows, columns = weight.shape
assert columns % 32 == 0, f"MXFP8 columns must divide into 32-value blocks, got {columns}"
blocks = weight.float().reshape(rows, columns // 32, 32)
bits = (blocks.abs().amax(-1) / 448.0).view(torch.int32)
exponent, mantissa = (bits >> 23) & 0xFF, bits & 0x7FFFFF
round_up = (mantissa > 0) & (exponent != 0xFE) & ~((exponent == 0) & (mantissa <= 0x400000))
scale = (exponent + round_up.int()).clamp(max=254)
reciprocal = ((254 - scale) << 23).view(torch.float32)
reciprocal = torch.where(scale == 254, torch.full_like(reciprocal, 2.0**-127), reciprocal)
values = (blocks * reciprocal.unsqueeze(-1)).clamp(-448.0, 448.0)
return values.to(torch.float8_e4m3fn).reshape(rows, columns), scale.to(torch.uint8)
state = model.state_dict()
for name in sorted(tuple(state)):
if ".moe.experts." in name and name.endswith(".weight"):
values, scales = quantize_mxfp8(state[name])
state[name] = values
state[name.removesuffix(".weight") + ".weight_scale_inv"] = scales
raw["quantization_config"]["dequantize"] = False
(folder / "config.json").write_text(json.dumps(raw, indent=2) + "\n", encoding="utf-8")
save_file(state, folder / "model.safetensors", metadata={"format": "pt"})
assets = {"LICENSE", "chat_template.jinja", "generation_config.json", "tokenizer.json", "tokenizer_config.json"}
for filename in sorted(set(list_repo_files(source_model_id, revision=source_revision)) & assets):
copyfile(hf_hub_download(source_model_id, filename, revision=source_revision), folder / filename)
```
Click to expand
```text
Raon1MiniForCausalLM(
(model): Raon1MiniModel(
(embed_tokens): Embedding(259840, 32, padding_idx=259200)
(layers): ModuleList(
(0): Raon1MiniDecoderLayer(
(input_layernorm): Raon1MiniRMSNorm((32,), eps=1e-05)
(post_attention_layernorm): Raon1MiniRMSNorm((32,), eps=1e-05)
(self_attn): Raon1MiniRDA(
(q_proj): Linear(in_features=32, out_features=32, bias=False)
(k_proj): Linear(in_features=32, out_features=32, bias=False)
(v_proj): Linear(in_features=32, out_features=32, bias=False)
(q_conv1d): Raon1MiniShortConv()
(k_conv1d): Raon1MiniShortConv()
(v_conv1d): Raon1MiniShortConv()
(f_a_proj): Linear(in_features=32, out_features=32, bias=False)
(f_b_proj): Linear(in_features=32, out_features=32, bias=False)
(b_proj): Linear(in_features=32, out_features=1, bias=False)
(rda_w_proj): Linear(in_features=32, out_features=1, bias=False)
(g_proj): Linear(in_features=32, out_features=32, bias=False)
(o_norm): Raon1MiniRMSNorm((32,), eps=1e-05)
(o_proj): Linear(in_features=32, out_features=32, bias=False)
)
(self_attn_raon_mhc): RaonMHC(
(res): RaonMHCStreamMix()
)
(mlp_raon_mhc): RaonMHC(
(res): RaonMHCStreamMix()
)
(mlp): Raon1MiniMLP(
(gate_proj): Linear(in_features=32, out_features=128, bias=False)
(up_proj): Linear(in_features=32, out_features=128, bias=False)
(down_proj): Linear(in_features=128, out_features=32, bias=False)
)
)
(1-2): 2 x Raon1MiniDecoderLayer(
(input_layernorm): Raon1MiniRMSNorm((32,), eps=1e-05)
(post_attention_layernorm): Raon1MiniRMSNorm((32,), eps=1e-05)
(self_attn): Raon1MiniRDA(
(q_proj): Linear(in_features=32, out_features=32, bias=False)
(k_proj): Linear(in_features=32, out_features=32, bias=False)
(v_proj): Linear(in_features=32, out_features=32, bias=False)
(q_conv1d): Raon1MiniShortConv()
(k_conv1d): Raon1MiniShortConv()
(v_conv1d): Raon1MiniShortConv()
(f_a_proj): Linear(in_features=32, out_features=32, bias=False)
(f_b_proj): Linear(in_features=32, out_features=32, bias=False)
(b_proj): Linear(in_features=32, out_features=1, bias=False)
(rda_w_proj): Linear(in_features=32, out_features=1, bias=False)
(g_proj): Linear(in_features=32, out_features=32, bias=False)
(o_norm): Raon1MiniRMSNorm((32,), eps=1e-05)
(o_proj): Linear(in_features=32, out_features=32, bias=False)
)
(self_attn_raon_mhc): RaonMHC(
(res): RaonMHCStreamMix()
)
(mlp_raon_mhc): RaonMHC(
(res): RaonMHCStreamMix()
)
(moe): Raon1MiniMoE(
(gate): Raon1MiniRouter()
(experts): ModuleList(
(0-63): 64 x Raon1MiniExpert(
(gate_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
(up_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
(down_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
)
)
(shared_experts): Raon1MiniMLP(
(gate_proj): Linear(in_features=32, out_features=32, bias=False)
(up_proj): Linear(in_features=32, out_features=32, bias=False)
(down_proj): Linear(in_features=32, out_features=32, bias=False)
)
)
)
(3): Raon1MiniDecoderLayer(
(input_layernorm): Raon1MiniRMSNorm((32,), eps=1e-05)
(post_attention_layernorm): Raon1MiniRMSNorm((32,), eps=1e-05)
(self_attn): Raon1MiniMLA(
(q_a_proj): Linear(in_features=32, out_features=32, bias=False)
(q_a_layernorm): Raon1MiniRMSNorm((32,), eps=1e-06)
(q_b_proj): Linear(in_features=32, out_features=64, bias=False)
(kv_a_proj_with_mqa): Linear(in_features=32, out_features=64, bias=False)
(kv_a_layernorm): Raon1MiniRMSNorm((32,), eps=1e-06)
(kv_b_proj): Linear(in_features=32, out_features=64, bias=False)
(g_proj): Linear(in_features=32, out_features=32, bias=False)
(o_proj): Linear(in_features=32, out_features=32, bias=False)
)
(self_attn_raon_mhc): RaonMHC(
(res): RaonMHCStreamMix()
)
(mlp_raon_mhc): RaonMHC(
(res): RaonMHCStreamMix()
)
(moe): Raon1MiniMoE(
(gate): Raon1MiniRouter()
(experts): ModuleList(
(0-63): 64 x Raon1MiniExpert(
(gate_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
(up_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
(down_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
)
)
(shared_experts): Raon1MiniMLP(
(gate_proj): Linear(in_features=32, out_features=32, bias=False)
(up_proj): Linear(in_features=32, out_features=32, bias=False)
(down_proj): Linear(in_features=32, out_features=32, bias=False)
)
)
)
(4-6): 3 x Raon1MiniDecoderLayer(
(input_layernorm): Raon1MiniRMSNorm((32,), eps=1e-05)
(post_attention_layernorm): Raon1MiniRMSNorm((32,), eps=1e-05)
(self_attn): Raon1MiniRDA(
(q_proj): Linear(in_features=32, out_features=32, bias=False)
(k_proj): Linear(in_features=32, out_features=32, bias=False)
(v_proj): Linear(in_features=32, out_features=32, bias=False)
(q_conv1d): Raon1MiniShortConv()
(k_conv1d): Raon1MiniShortConv()
(v_conv1d): Raon1MiniShortConv()
(f_a_proj): Linear(in_features=32, out_features=32, bias=False)
(f_b_proj): Linear(in_features=32, out_features=32, bias=False)
(b_proj): Linear(in_features=32, out_features=1, bias=False)
(rda_w_proj): Linear(in_features=32, out_features=1, bias=False)
(g_proj): Linear(in_features=32, out_features=32, bias=False)
(o_norm): Raon1MiniRMSNorm((32,), eps=1e-05)
(o_proj): Linear(in_features=32, out_features=32, bias=False)
)
(self_attn_raon_mhc): RaonMHC(
(res): RaonMHCStreamMix()
)
(mlp_raon_mhc): RaonMHC(
(res): RaonMHCStreamMix()
)
(moe): Raon1MiniMoE(
(gate): Raon1MiniRouter()
(experts): ModuleList(
(0-63): 64 x Raon1MiniExpert(
(gate_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
(up_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
(down_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
)
)
(shared_experts): Raon1MiniMLP(
(gate_proj): Linear(in_features=32, out_features=32, bias=False)
(up_proj): Linear(in_features=32, out_features=32, bias=False)
(down_proj): Linear(in_features=32, out_features=32, bias=False)
)
)
)
(7): Raon1MiniDecoderLayer(
(input_layernorm): Raon1MiniRMSNorm((32,), eps=1e-05)
(post_attention_layernorm): Raon1MiniRMSNorm((32,), eps=1e-05)
(self_attn): Raon1MiniMLA(
(q_a_proj): Linear(in_features=32, out_features=32, bias=False)
(q_a_layernorm): Raon1MiniRMSNorm((32,), eps=1e-06)
(q_b_proj): Linear(in_features=32, out_features=64, bias=False)
(kv_a_proj_with_mqa): Linear(in_features=32, out_features=64, bias=False)
(kv_a_layernorm): Raon1MiniRMSNorm((32,), eps=1e-06)
(kv_b_proj): Linear(in_features=32, out_features=64, bias=False)
(g_proj): Linear(in_features=32, out_features=32, bias=False)
(o_proj): Linear(in_features=32, out_features=32, bias=False)
)
(self_attn_raon_mhc): RaonMHC(
(res): RaonMHCStreamMix()
)
(mlp_raon_mhc): RaonMHC(
(res): RaonMHCStreamMix()
)
(moe): Raon1MiniMoE(
(gate): Raon1MiniRouter()
(experts): ModuleList(
(0-63): 64 x Raon1MiniExpert(
(gate_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
(up_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
(down_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
)
)
(shared_experts): Raon1MiniMLP(
(gate_proj): Linear(in_features=32, out_features=32, bias=False)
(up_proj): Linear(in_features=32, out_features=32, bias=False)
(down_proj): Linear(in_features=32, out_features=32, bias=False)
)
)
)
)
(norm): Raon1MiniRMSNorm((32,), eps=1e-05)
)
(lm_head): Linear(in_features=32, out_features=259840, bias=False)
)
```