---
base_model:
- meta-models/Muse-Spark-Open-816B-A42B
---
This is a randomly initialized tiny model based on [meta-models/Muse-Spark-Open-816B-A42B](https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B) for testing and debugging.
| Feature | Source | Tiny |
|---|---|---|
| Checkpoint tensor elements | 821,906,490,368 BF16, 130,039,296 F32 | 3,580,264 BF16, 2,304 F32 |
| Safetensors weight files | 1,644,333,393,864 bytes (65 files) | 7,211,288 bytes (model.safetensors) |
| Text layers / attention cycle | 62 / sliding, full, sliding, sliding | 8 / two source cycles |
| Residual / expert pool / expert intermediate width | 8192 / 4096 / 4096 | 8 / 4 / 16 |
| Q heads / KV heads / head dimension | 128 / 16 / 64 | 8 / 1 / 32 |
| Experts / activated experts | 256 / 8 | 32 / 8 |
| Vocabulary / tying | 202048 / untied | Same |
| Vision layers / width / temporal patch / merge | 50 / 1536 / 2 / 2 | 8 / 32 / 2 / 2 |
| Audio stages / block depths / sample rate | 4 / [2, 2, 2, 2] / 24 kHz | Same |
| Checkpoint precision | BF16; router weights/biases FP32 | Same |
| Modalities | Text + image + video + audio → text | Same |
| File path | Size |
|------|------|
| model.safetensors | 7.2MB |
### Example usage:
- Transformers (✅ verified)
This example uses text, an image, predecoded video frames and a 24 kHz waveform. SDPA remains enabled; batched expert GEMMs support the tiny expert pool without grouped-GEMM alignment padding.
```python
def run_transformers(model_id: str):
import numpy as np
import torch
from PIL import Image
from transformers import AutoProcessor, MuseSparkForConditionalGeneration
from transformers.video_utils import VideoMetadata
processor = AutoProcessor.from_pretrained(model_id)
model = MuseSparkForConditionalGeneration.from_pretrained(
model_id,
dtype=torch.bfloat16,
attn_implementation="sdpa",
experts_implementation="batched_mm",
device_map="cuda",
)
image = Image.new("RGB", (112, 112), "blue")
video = np.stack([np.full((112, 112, 3), value, dtype=np.uint8) for value in (40, 80, 120, 160)])
audio = (0.1 * np.sin(2 * np.pi * 440 * np.arange(24000) / 24000)).astype(np.float32)
messages = [
{
"role": "user",
"content": [
{"type": "image"},
{"type": "video"},
{"type": "audio"},
{"type": "text", "text": "Describe the image, video and sound."},
],
}
]
prompt = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, reasoning_effort="low")
inputs = processor(
text=[prompt],
images=[image],
videos=[video],
audio=[audio],
sampling_rate=24000,
video_metadata=[VideoMetadata(total_num_frames=4, fps=2, frames_indices=[0, 1, 2, 3])],
do_sample_frames=False,
return_tensors="pt",
).to(model.device, dtype=model.dtype)
outputs = model.generate(**inputs, max_new_tokens=10, do_sample=False)
print(model)
print(processor.batch_decode(outputs[:, inputs.input_ids.shape[1] :], skip_special_tokens=True))
if __name__ == "__main__":
run_transformers(model_id="tiny-random/muse-spark-open")
```
- SGLang (✅ verified)
Muse Spark's BF16 experts require EP=TP, so this single-node launch uses TP=4 and EP=4. Install FFmpeg shared libraries for video decoding.
```bash
sglang serve \
--model-path tiny-random/muse-spark-open \
--tp-size 4 \
--ep-size 4 \
--attention-backend triton \
--context-length 4096 \
--max-running-requests 4 \
--max-total-tokens 4096 \
--mem-fraction-static 0.2 \
--disable-custom-all-reduce
```
### Codes to create this repo:
Click to expand
```python
def create_model(source_model_id: str, source_revision: str, save_folder: str) -> str:
import json
from pathlib import Path
import torch
from huggingface_hub import hf_hub_download
from transformers import MuseSparkConfig, MuseSparkForConditionalGeneration, set_seed
folder = Path(save_folder)
folder.mkdir(parents=True, exist_ok=True)
config_json = json.loads(
Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text()
)
text = config_json["text_config"]
text.update(
hidden_size=8,
num_attention_heads=8,
num_key_value_heads=1,
head_dim=32,
num_hidden_layers=8,
intermediate_size=32,
moe_hidden_size=4,
moe_intermediate_size=16,
num_local_experts=32,
)
for name in ("layer_types", "mlp_layer_types", "layer_rope_theta"):
text[name] = text[name][:8]
vision = config_json["vision_config"]
vision.update(hidden_size=32, intermediate_size=64, num_attention_heads=4, num_hidden_layers=8)
vision["layer_types"] = vision["layer_types"][:8]
config_json["audio_config"].update(hidden_sizes=[8, 16, 32, 64], out_hidden_size=8)
config_json.update(out_hidden_size=128, projector_hidden_size=16)
config = MuseSparkConfig.from_dict(config_json)
set_seed(42)
model = MuseSparkForConditionalGeneration._from_config(config, dtype=torch.bfloat16, attn_implementation="sdpa")
# The source checkpoint uses FP32 only for routed-expert router weights and
# correction biases; residual gates and all other parameters remain BF16.
for module in model.modules():
if module.__class__.__name__ == "MuseSparkTopkRouter":
module.float()
model.save_pretrained(folder, safe_serialization=True)
# Restore the deliberately edited raw config and immutable shared assets.
(folder / "config.json").write_text(json.dumps(config_json, indent=2) + "\n")
for name in (
"tokenizer.json",
"tokenizer_config.json",
"special_tokens_map.json",
"chat_template.jinja",
"processor_config.json",
"generation_config.json",
"l4_200k_base",
"LICENSE.md",
"USE_POLICY.md",
):
(folder / name).write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes())
return str(model)
create_model(
source_model_id="meta-models/Muse-Spark-Open-816B-A42B",
source_revision="0fb02478b02565b2ce7fc57e6051c91172774ac9",
save_folder="/tmp/muse-spark-open",
)
```
Click to expand
```text
MuseSparkForConditionalGeneration(
(model): MuseSparkModel(
(vision_tower): MuseSparkVisionModel(
(patch_embedder): MuseSparkVisionPatchEmbedder(
(patch_embedding): Linear(in_features=1176, out_features=32, bias=False)
(position_embedding_table): Embedding(1024, 32)
)
(rotary_emb): MuseSparkVisionRotaryEmbedding()
(ln_pre): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True)
(layers): ModuleList(
(0-7): 8 x MuseSparkVisionEncoderLayer(
(norm1): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True)
(norm2): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True)
(attn): MuseSparkVisionAttention(
(proj): Linear(in_features=32, out_features=32, bias=True)
(q_proj): Linear(in_features=32, out_features=32, bias=True)
(k_proj): Linear(in_features=32, out_features=32, bias=True)
(v_proj): Linear(in_features=32, out_features=32, bias=True)
)
(mlp): MuseSparkVisionMLP(
(fc1): Linear(in_features=32, out_features=64, bias=True)
(act): GELUActivation()
(fc2): Linear(in_features=64, out_features=32, bias=True)
)
)
)
(ln_post): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True)
)
(language_model): MuseSparkTextModel(
(embed_tokens): MuseSparkTextNormedEmbedding(
202048, 8, padding_idx=200018
(embed_norm): MuseSparkRMSNorm()
)
(layers): ModuleList(
(0-7): 8 x MuseSparkTextDecoderLayer(
(self_attn): MuseSparkTextAttention(
(q_proj): Linear(in_features=8, out_features=256, bias=False)
(k_proj): Linear(in_features=8, out_features=32, bias=False)
(v_proj): Linear(in_features=8, out_features=32, bias=False)
(o_proj): Linear(in_features=256, out_features=8, bias=False)
(gate_proj): Linear(in_features=8, out_features=256, bias=False)
(qk_norm): MuseSparkRMSNorm()
(o_norm): MuseSparkRMSNorm()
)
(mlp): MuseSparkMoE(
(gate): MuseSparkTopkRouter(
(scoring_function): Sigmoid()
)
(pre_expert_proj): Linear(in_features=8, out_features=4, bias=False)
(pre_expert_norm): MuseSparkTextCenteredRMSNorm((4,), eps=1e-05)
(experts): MuseSparkExperts(
(act_fn): SiLUActivation()
(post_expert_norm): MuseSparkTextInputScaledRMSNorm((4,), eps=1e-08)
)
(post_expert_proj): Linear(in_features=4, out_features=8, bias=False)
)
(input_layernorm): MuseSparkTextCenteredRMSNorm((8,), eps=1e-05)
(post_attention_layernorm): MuseSparkRMSNorm()
(pre_feedforward_layernorm): MuseSparkTextCenteredRMSNorm((8,), eps=1e-05)
(post_feedforward_layernorm): MuseSparkTextInputScaledRMSNorm((8,), eps=1e-08)
(post_attention_residual_gate): MuseSparkResidualGate()
(post_feedforward_residual_gate): MuseSparkResidualGate()
)
)
(norm): MuseSparkRMSNorm()
(rotary_emb): MuseSparkTextRotaryEmbedding()
)
(vision_adapter): MuseSparkVisionAdapter(
(fc1): Linear(in_features=128, out_features=16, bias=False)
(act): GELUActivation()
(fc2): Linear(in_features=16, out_features=16, bias=False)
)
(vision_projection): Linear(in_features=16, out_features=8, bias=False)
(perception_emb_norm): MuseSparkRMSNorm()
(audio_tower): MuseSparkAudioModel(
(layers): ModuleList(
(0): MuseSparkAudioLayer(
(patch_embedder): MuseSparkAudioPatchEmbedding(
(proj): Linear(in_features=16, out_features=8, bias=True)
(norm): MuseSparkRMSNorm()
)
(conv_blocks): ModuleList(
(0-1): 2 x MuseSparkAudioConvBlock(
(conv): MuseSparkAudioCausalConv1d(8, 8, kernel_size=(7,), stride=(1,), groups=8)
(norm): MuseSparkRMSNorm()
(mlp): MuseSparkAudioMLP(
(fc1): Linear(in_features=8, out_features=32, bias=True)
(fc2): Linear(in_features=32, out_features=8, bias=True)
(activation_fn): GELUActivation()
)
)
)
)
(1): MuseSparkAudioLayer(
(patch_embedder): MuseSparkAudioPatchEmbedding(
(proj): Linear(in_features=48, out_features=16, bias=True)
(norm): MuseSparkRMSNorm()
)
(conv_blocks): ModuleList(
(0-1): 2 x MuseSparkAudioConvBlock(
(conv): MuseSparkAudioCausalConv1d(16, 16, kernel_size=(7,), stride=(1,), groups=16)
(norm): MuseSparkRMSNorm()
(mlp): MuseSparkAudioMLP(
(fc1): Linear(in_features=16, out_features=64, bias=True)
(fc2): Linear(in_features=64, out_features=16, bias=True)
(activation_fn): GELUActivation()
)
)
)
)
(2): MuseSparkAudioLayer(
(patch_embedder): MuseSparkAudioPatchEmbedding(
(proj): Linear(in_features=80, out_features=32, bias=True)
(norm): MuseSparkRMSNorm()
)
(conv_blocks): ModuleList(
(0-1): 2 x MuseSparkAudioConvBlock(
(conv): MuseSparkAudioCausalConv1d(32, 32, kernel_size=(7,), stride=(1,), groups=32)
(norm): MuseSparkRMSNorm()
(mlp): MuseSparkAudioMLP(
(fc1): Linear(in_features=32, out_features=128, bias=True)
(fc2): Linear(in_features=128, out_features=32, bias=True)
(activation_fn): GELUActivation()
)
)
)
)
(3): MuseSparkAudioLayer(
(patch_embedder): MuseSparkAudioPatchEmbedding(
(proj): Linear(in_features=128, out_features=64, bias=True)
(norm): MuseSparkRMSNorm()
)
(conv_blocks): ModuleList(
(0-1): 2 x MuseSparkAudioConvBlock(
(conv): MuseSparkAudioCausalConv1d(64, 64, kernel_size=(7,), stride=(1,), groups=64)
(norm): MuseSparkRMSNorm()
(mlp): MuseSparkAudioMLP(
(fc1): Linear(in_features=64, out_features=256, bias=True)
(fc2): Linear(in_features=256, out_features=64, bias=True)
(activation_fn): GELUActivation()
)
)
)
)
)
(norm): MuseSparkRMSNorm()
(proj): Linear(in_features=64, out_features=8, bias=True)
(post_proj_norm): MuseSparkRMSNorm()
)
)
(lm_head): Linear(in_features=8, out_features=202048, bias=False)
)
```