| --- |
| base_model: |
| - meta-models/Muse-Spark-Open-816B-A42B |
| --- |
| |
| This is a randomly initialized tiny model based on [meta-models/Muse-Spark-Open-816B-A42B](https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B) for testing and debugging. |
|
|
| | Feature | Source | Tiny | |
| |---|---|---| |
| | Checkpoint tensor elements | 821,906,490,368 BF16, 130,039,296 F32 | 3,580,264 BF16, 2,304 F32 | |
| | Safetensors weight files | 1,644,333,393,864 bytes (65 files) | 7,211,288 bytes (model.safetensors) | |
| | Text layers / attention cycle | 62 / sliding, full, sliding, sliding | 8 / two source cycles | |
| | Residual / expert pool / expert intermediate width | 8192 / 4096 / 4096 | 8 / 4 / 16 | |
| | Q heads / KV heads / head dimension | 128 / 16 / 64 | 8 / 1 / 32 | |
| | Experts / activated experts | 256 / 8 | 32 / 8 | |
| | Vocabulary / tying | 202048 / untied | Same | |
| | Vision layers / width / temporal patch / merge | 50 / 1536 / 2 / 2 | 8 / 32 / 2 / 2 | |
| | Audio stages / block depths / sample rate | 4 / [2, 2, 2, 2] / 24 kHz | Same | |
| | Checkpoint precision | BF16; router weights/biases FP32 | Same | |
| | Modalities | Text + image + video + audio → text | Same | |
|
|
| | File path | Size | |
| |------|------| |
| | model.safetensors | 7.2MB | |
|
|
| ### Example usage: |
|
|
| - Transformers (✅ verified) |
|
|
| This example uses text, an image, predecoded video frames and a 24 kHz waveform. SDPA remains enabled; batched expert GEMMs support the tiny expert pool without grouped-GEMM alignment padding. |
|
|
| ```python |
| def run_transformers(model_id: str): |
| import numpy as np |
| import torch |
| from PIL import Image |
| from transformers import AutoProcessor, MuseSparkForConditionalGeneration |
| from transformers.video_utils import VideoMetadata |
| |
| processor = AutoProcessor.from_pretrained(model_id) |
| model = MuseSparkForConditionalGeneration.from_pretrained( |
| model_id, |
| dtype=torch.bfloat16, |
| attn_implementation="sdpa", |
| experts_implementation="batched_mm", |
| device_map="cuda", |
| ) |
| image = Image.new("RGB", (112, 112), "blue") |
| video = np.stack([np.full((112, 112, 3), value, dtype=np.uint8) for value in (40, 80, 120, 160)]) |
| audio = (0.1 * np.sin(2 * np.pi * 440 * np.arange(24000) / 24000)).astype(np.float32) |
| messages = [ |
| { |
| "role": "user", |
| "content": [ |
| {"type": "image"}, |
| {"type": "video"}, |
| {"type": "audio"}, |
| {"type": "text", "text": "Describe the image, video and sound."}, |
| ], |
| } |
| ] |
| prompt = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, reasoning_effort="low") |
| inputs = processor( |
| text=[prompt], |
| images=[image], |
| videos=[video], |
| audio=[audio], |
| sampling_rate=24000, |
| video_metadata=[VideoMetadata(total_num_frames=4, fps=2, frames_indices=[0, 1, 2, 3])], |
| do_sample_frames=False, |
| return_tensors="pt", |
| ).to(model.device, dtype=model.dtype) |
| outputs = model.generate(**inputs, max_new_tokens=10, do_sample=False) |
| print(model) |
| print(processor.batch_decode(outputs[:, inputs.input_ids.shape[1] :], skip_special_tokens=True)) |
| |
| |
| if __name__ == "__main__": |
| run_transformers(model_id="tiny-random/muse-spark-open") |
| ``` |
|
|
| - SGLang (✅ verified) |
|
|
| Muse Spark's BF16 experts require EP=TP, so this single-node launch uses TP=4 and EP=4. Install FFmpeg shared libraries for video decoding. |
|
|
| ```bash |
| sglang serve \ |
| --model-path tiny-random/muse-spark-open \ |
| --tp-size 4 \ |
| --ep-size 4 \ |
| --attention-backend triton \ |
| --context-length 4096 \ |
| --max-running-requests 4 \ |
| --max-total-tokens 4096 \ |
| --mem-fraction-static 0.2 \ |
| --disable-custom-all-reduce |
| ``` |
|
|
| ### Codes to create this repo: |
|
|
| <details><summary>Click to expand</summary> |
|
|
| ```python |
| def create_model(source_model_id: str, source_revision: str, save_folder: str) -> str: |
| import json |
| from pathlib import Path |
| import torch |
| from huggingface_hub import hf_hub_download |
| from transformers import MuseSparkConfig, MuseSparkForConditionalGeneration, set_seed |
| |
| folder = Path(save_folder) |
| folder.mkdir(parents=True, exist_ok=True) |
| config_json = json.loads( |
| Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text() |
| ) |
| text = config_json["text_config"] |
| text.update( |
| hidden_size=8, |
| num_attention_heads=8, |
| num_key_value_heads=1, |
| head_dim=32, |
| num_hidden_layers=8, |
| intermediate_size=32, |
| moe_hidden_size=4, |
| moe_intermediate_size=16, |
| num_local_experts=32, |
| ) |
| for name in ("layer_types", "mlp_layer_types", "layer_rope_theta"): |
| text[name] = text[name][:8] |
| vision = config_json["vision_config"] |
| vision.update(hidden_size=32, intermediate_size=64, num_attention_heads=4, num_hidden_layers=8) |
| vision["layer_types"] = vision["layer_types"][:8] |
| config_json["audio_config"].update(hidden_sizes=[8, 16, 32, 64], out_hidden_size=8) |
| config_json.update(out_hidden_size=128, projector_hidden_size=16) |
| config = MuseSparkConfig.from_dict(config_json) |
| set_seed(42) |
| model = MuseSparkForConditionalGeneration._from_config(config, dtype=torch.bfloat16, attn_implementation="sdpa") |
| # The source checkpoint uses FP32 only for routed-expert router weights and |
| # correction biases; residual gates and all other parameters remain BF16. |
| for module in model.modules(): |
| if module.__class__.__name__ == "MuseSparkTopkRouter": |
| module.float() |
| model.save_pretrained(folder, safe_serialization=True) |
| # Restore the deliberately edited raw config and immutable shared assets. |
| (folder / "config.json").write_text(json.dumps(config_json, indent=2) + "\n") |
| for name in ( |
| "tokenizer.json", |
| "tokenizer_config.json", |
| "special_tokens_map.json", |
| "chat_template.jinja", |
| "processor_config.json", |
| "generation_config.json", |
| "l4_200k_base", |
| "LICENSE.md", |
| "USE_POLICY.md", |
| ): |
| (folder / name).write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes()) |
| return str(model) |
| |
| |
| create_model( |
| source_model_id="meta-models/Muse-Spark-Open-816B-A42B", |
| source_revision="0fb02478b02565b2ce7fc57e6051c91172774ac9", |
| save_folder="/tmp/muse-spark-open", |
| ) |
| ``` |
|
|
| </details> |
|
|
| ### Printing the model: |
|
|
| <details><summary>Click to expand</summary> |
|
|
| ```text |
| MuseSparkForConditionalGeneration( |
| (model): MuseSparkModel( |
| (vision_tower): MuseSparkVisionModel( |
| (patch_embedder): MuseSparkVisionPatchEmbedder( |
| (patch_embedding): Linear(in_features=1176, out_features=32, bias=False) |
| (position_embedding_table): Embedding(1024, 32) |
| ) |
| (rotary_emb): MuseSparkVisionRotaryEmbedding() |
| (ln_pre): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True) |
| (layers): ModuleList( |
| (0-7): 8 x MuseSparkVisionEncoderLayer( |
| (norm1): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True) |
| (norm2): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True) |
| (attn): MuseSparkVisionAttention( |
| (proj): Linear(in_features=32, out_features=32, bias=True) |
| (q_proj): Linear(in_features=32, out_features=32, bias=True) |
| (k_proj): Linear(in_features=32, out_features=32, bias=True) |
| (v_proj): Linear(in_features=32, out_features=32, bias=True) |
| ) |
| (mlp): MuseSparkVisionMLP( |
| (fc1): Linear(in_features=32, out_features=64, bias=True) |
| (act): GELUActivation() |
| (fc2): Linear(in_features=64, out_features=32, bias=True) |
| ) |
| ) |
| ) |
| (ln_post): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True) |
| ) |
| (language_model): MuseSparkTextModel( |
| (embed_tokens): MuseSparkTextNormedEmbedding( |
| 202048, 8, padding_idx=200018 |
| (embed_norm): MuseSparkRMSNorm() |
| ) |
| (layers): ModuleList( |
| (0-7): 8 x MuseSparkTextDecoderLayer( |
| (self_attn): MuseSparkTextAttention( |
| (q_proj): Linear(in_features=8, out_features=256, bias=False) |
| (k_proj): Linear(in_features=8, out_features=32, bias=False) |
| (v_proj): Linear(in_features=8, out_features=32, bias=False) |
| (o_proj): Linear(in_features=256, out_features=8, bias=False) |
| (gate_proj): Linear(in_features=8, out_features=256, bias=False) |
| (qk_norm): MuseSparkRMSNorm() |
| (o_norm): MuseSparkRMSNorm() |
| ) |
| (mlp): MuseSparkMoE( |
| (gate): MuseSparkTopkRouter( |
| (scoring_function): Sigmoid() |
| ) |
| (pre_expert_proj): Linear(in_features=8, out_features=4, bias=False) |
| (pre_expert_norm): MuseSparkTextCenteredRMSNorm((4,), eps=1e-05) |
| (experts): MuseSparkExperts( |
| (act_fn): SiLUActivation() |
| (post_expert_norm): MuseSparkTextInputScaledRMSNorm((4,), eps=1e-08) |
| ) |
| (post_expert_proj): Linear(in_features=4, out_features=8, bias=False) |
| ) |
| (input_layernorm): MuseSparkTextCenteredRMSNorm((8,), eps=1e-05) |
| (post_attention_layernorm): MuseSparkRMSNorm() |
| (pre_feedforward_layernorm): MuseSparkTextCenteredRMSNorm((8,), eps=1e-05) |
| (post_feedforward_layernorm): MuseSparkTextInputScaledRMSNorm((8,), eps=1e-08) |
| (post_attention_residual_gate): MuseSparkResidualGate() |
| (post_feedforward_residual_gate): MuseSparkResidualGate() |
| ) |
| ) |
| (norm): MuseSparkRMSNorm() |
| (rotary_emb): MuseSparkTextRotaryEmbedding() |
| ) |
| (vision_adapter): MuseSparkVisionAdapter( |
| (fc1): Linear(in_features=128, out_features=16, bias=False) |
| (act): GELUActivation() |
| (fc2): Linear(in_features=16, out_features=16, bias=False) |
| ) |
| (vision_projection): Linear(in_features=16, out_features=8, bias=False) |
| (perception_emb_norm): MuseSparkRMSNorm() |
| (audio_tower): MuseSparkAudioModel( |
| (layers): ModuleList( |
| (0): MuseSparkAudioLayer( |
| (patch_embedder): MuseSparkAudioPatchEmbedding( |
| (proj): Linear(in_features=16, out_features=8, bias=True) |
| (norm): MuseSparkRMSNorm() |
| ) |
| (conv_blocks): ModuleList( |
| (0-1): 2 x MuseSparkAudioConvBlock( |
| (conv): MuseSparkAudioCausalConv1d(8, 8, kernel_size=(7,), stride=(1,), groups=8) |
| (norm): MuseSparkRMSNorm() |
| (mlp): MuseSparkAudioMLP( |
| (fc1): Linear(in_features=8, out_features=32, bias=True) |
| (fc2): Linear(in_features=32, out_features=8, bias=True) |
| (activation_fn): GELUActivation() |
| ) |
| ) |
| ) |
| ) |
| (1): MuseSparkAudioLayer( |
| (patch_embedder): MuseSparkAudioPatchEmbedding( |
| (proj): Linear(in_features=48, out_features=16, bias=True) |
| (norm): MuseSparkRMSNorm() |
| ) |
| (conv_blocks): ModuleList( |
| (0-1): 2 x MuseSparkAudioConvBlock( |
| (conv): MuseSparkAudioCausalConv1d(16, 16, kernel_size=(7,), stride=(1,), groups=16) |
| (norm): MuseSparkRMSNorm() |
| (mlp): MuseSparkAudioMLP( |
| (fc1): Linear(in_features=16, out_features=64, bias=True) |
| (fc2): Linear(in_features=64, out_features=16, bias=True) |
| (activation_fn): GELUActivation() |
| ) |
| ) |
| ) |
| ) |
| (2): MuseSparkAudioLayer( |
| (patch_embedder): MuseSparkAudioPatchEmbedding( |
| (proj): Linear(in_features=80, out_features=32, bias=True) |
| (norm): MuseSparkRMSNorm() |
| ) |
| (conv_blocks): ModuleList( |
| (0-1): 2 x MuseSparkAudioConvBlock( |
| (conv): MuseSparkAudioCausalConv1d(32, 32, kernel_size=(7,), stride=(1,), groups=32) |
| (norm): MuseSparkRMSNorm() |
| (mlp): MuseSparkAudioMLP( |
| (fc1): Linear(in_features=32, out_features=128, bias=True) |
| (fc2): Linear(in_features=128, out_features=32, bias=True) |
| (activation_fn): GELUActivation() |
| ) |
| ) |
| ) |
| ) |
| (3): MuseSparkAudioLayer( |
| (patch_embedder): MuseSparkAudioPatchEmbedding( |
| (proj): Linear(in_features=128, out_features=64, bias=True) |
| (norm): MuseSparkRMSNorm() |
| ) |
| (conv_blocks): ModuleList( |
| (0-1): 2 x MuseSparkAudioConvBlock( |
| (conv): MuseSparkAudioCausalConv1d(64, 64, kernel_size=(7,), stride=(1,), groups=64) |
| (norm): MuseSparkRMSNorm() |
| (mlp): MuseSparkAudioMLP( |
| (fc1): Linear(in_features=64, out_features=256, bias=True) |
| (fc2): Linear(in_features=256, out_features=64, bias=True) |
| (activation_fn): GELUActivation() |
| ) |
| ) |
| ) |
| ) |
| ) |
| (norm): MuseSparkRMSNorm() |
| (proj): Linear(in_features=64, out_features=8, bias=True) |
| (post_proj_norm): MuseSparkRMSNorm() |
| ) |
| ) |
| (lm_head): Linear(in_features=8, out_features=202048, bias=False) |
| ) |
| ``` |
|
|
| </details> |
|
|
| Environment: [.meta.json](.meta.json) |
|
|
| ### Change log: |
|
|
| - 2026-10-26: Created a tiny Muse Spark Open checkpoint preserving text, image, video and audio inputs. |
|
|
| This repo is generated by GPT-6.1-Sol |
