Instructions to use tiny-random/qwen3.8-moe with libraries, inference providers, notebooks, and local apps. Follow these links to get started.

Libraries

How to use tiny-random/qwen3.8-moe with Transformers:

# Use a pipeline as a high-level helper
from transformers import pipeline

pipe = pipeline("text-generation", model="tiny-random/qwen3.8-moe")
messages = [
    {"role": "user", "content": "Who are you?"},
]
pipe(messages)

# pip install -U transformers accelerate
# Load model directly
from transformers import AutoTokenizer, AutoModelForCausalLM

tokenizer = AutoTokenizer.from_pretrained("tiny-random/qwen3.8-moe")
model = AutoModelForCausalLM.from_pretrained("tiny-random/qwen3.8-moe", device_map="auto")
messages = [
    {"role": "user", "content": "Who are you?"},
]
inputs = tokenizer.apply_chat_template(
	messages,
	add_generation_prompt=True,
	tokenize=True,
	return_dict=True,
	return_tensors="pt",
).to(model.device)

outputs = model.generate(**inputs, max_new_tokens=256)
print(tokenizer.decode(outputs[0][inputs["input_ids"].shape[-1]:]))

Notebooks
Google Colab
Kaggle
Local Apps Settings

vLLM

How to use tiny-random/qwen3.8-moe with vLLM:

Install from pip and serve model

# Install vLLM from pip:
pip install vllm
# Start the vLLM server:
vllm serve "tiny-random/qwen3.8-moe"
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:8000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "tiny-random/qwen3.8-moe",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Use Docker

docker model run hf.co/tiny-random/qwen3.8-moe

SGLang

How to use tiny-random/qwen3.8-moe with SGLang:

Install from pip and serve model

# Install SGLang from pip:
pip install sglang
# Start the SGLang server:
python3 -m sglang.launch_server \
    --model-path "tiny-random/qwen3.8-moe" \
    --host 0.0.0.0 \
    --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "tiny-random/qwen3.8-moe",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Use Docker images

docker run --gpus all \
    --shm-size 32g \
    -p 30000:30000 \
    -v ~/.cache/huggingface:/root/.cache/huggingface \
    --env "HF_TOKEN=<secret>" \
    --ipc=host \
    lmsysorg/sglang:latest \
    python3 -m sglang.launch_server \
        --model-path "tiny-random/qwen3.8-moe" \
        --host 0.0.0.0 \
        --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "tiny-random/qwen3.8-moe",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Docker Model Runner
How to use tiny-random/qwen3.8-moe with Docker Model Runner:
```
docker model run hf.co/tiny-random/qwen3.8-moe
```

yujiepan commited on 11 days ago

Commit

ed0e498

verified ·

1 Parent(s): 9432ad1

Make creation examples standalone with public dependencies

Browse files

Files changed (2) hide show

.meta.json +1 -2
README.md +40 -11

.meta.json CHANGED Viewed

@@ -5,7 +5,6 @@
   "kernels": "0.17.0",
   "safetensors": "0.8.0",
   "sglang": "0.5.23.dev384+g9ab74b851",
-  "tiny-random": "0.2.0",
   "torch": "2.14.1",
   "transformers": "5.19.0"
-}

   "kernels": "0.17.0",
   "safetensors": "0.8.0",
   "sglang": "0.5.23.dev384+g9ab74b851",
   "torch": "2.14.1",
   "transformers": "5.19.0"
+}

README.md CHANGED Viewed

@@ -79,13 +79,14 @@ print(tokenizer.decode(output[0, inputs["input_ids"].shape[1]:], skip_special_to
 ```python
 import json
 from pathlib import Path
 import torch
-from huggingface_hub import hf_hub_download
-from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer, GenerationConfig, set_seed
-# Creation helpers: pip install git+https://github.com/yujiepan/tiny-random.git
-from tiny_random.qwen_utils import add_qwen_mtp, save_random_qwen_checkpoint, compact_qwen_tokenizer, check_qwen_repo_size
 source_model_id = "Qwen/Qwen3.8-2.4T-A95B"
 save_folder = "/tmp/tiny-random/qwen38-moe"
@@ -105,12 +106,40 @@ raw.update(
 (folder / "config.json").write_text(json.dumps(raw, indent=2))
 config = AutoConfig.from_pretrained(folder)
 model = AutoModelForCausalLM.from_config(config, dtype=torch.bfloat16)
-add_qwen_mtp(model, model.model)
-save_random_qwen_checkpoint(model, folder)
-AutoTokenizer.from_pretrained(source_model_id, revision=source_revision).save_pretrained(folder)
-GenerationConfig.from_pretrained(source_model_id, revision=source_revision).save_pretrained(folder)
-compact_qwen_tokenizer(folder)
-check_qwen_repo_size(folder)
 ```
 </details>
@@ -133,7 +162,6 @@ Qwen3.8 MoE: 8 backbone layers + 1 MTP layer; hidden_size=16
 - kernels: 0.17.0
 - safetensors: 0.8.0
 - sglang: 0.5.23.dev384+g9ab74b851
-- tiny-random: 0.2.0
 - torch: 2.14.1
 - transformers: 5.19.0
@@ -141,5 +169,6 @@ Qwen3.8 MoE: 8 backbone layers + 1 MTP layer; hidden_size=16
 - 2026-10-23: Initial version.
 - 2026-10-23: Reduced all repo files below 50 MB; GPU-validated TP=2, MTP and CUDA Graph.
 This repo is generated by GPT-6.1-Sol

 ```python
 import json
+from copy import deepcopy
 from pathlib import Path
+from shutil import copyfile
 import torch
+from huggingface_hub import hf_hub_download, list_repo_files
+from safetensors.torch import save_file
+from transformers import AutoConfig, AutoModelForCausalLM, set_seed
 source_model_id = "Qwen/Qwen3.8-2.4T-A95B"
 save_folder = "/tmp/tiny-random/qwen38-moe"
 (folder / "config.json").write_text(json.dumps(raw, indent=2))
 config = AutoConfig.from_pretrained(folder)
 model = AutoModelForCausalLM.from_config(config, dtype=torch.bfloat16)
+backbone = model.model
+text = backbone.config
+def norm():
+    return type(backbone.norm)(text.hidden_size, text.rms_norm_eps).to(torch.bfloat16)
+model.mtp = torch.nn.ModuleDict({
+    "pre_fc_norm_embedding": norm(),
+    "pre_fc_norm_hidden": norm(),
+    "fc": torch.nn.Linear(2 * text.hidden_size, text.hidden_size, bias=False, dtype=torch.bfloat16),
+    "layers": torch.nn.ModuleList([deepcopy(backbone.layers[-1])]),
+    "norm": norm(),
+})
+with torch.no_grad():
+    for name, param in sorted(model.named_parameters()):
+        if name.endswith("A_log"):
+            param.zero_()
+        elif name.endswith("dt_bias"):
+            param.fill_(1)
+        elif "norm" in name and name.endswith("weight"):
+            param.fill_(1 if "visual" in name or ".linear_attn.norm." in name else 0)
+        elif name.endswith("bias"):
+            param.zero_()
+        else:
+            torch.nn.init.normal_(param, mean=0, std=0.02)
+state = {name: value.detach().cpu().contiguous() for name, value in model.state_dict().items()}
+save_file(state, folder / "model.safetensors", metadata={"format": "pt"})
+# Preserve tokenizer, processor and generation assets exactly as published.
+assets = {
+    "tokenizer.json", "tokenizer_config.json", "special_tokens_map.json",
+    "vocab.json", "merges.txt", "added_tokens.json", "chat_template.jinja",
+    "chat_template.json", "preprocessor_config.json", "processor_config.json",
+    "video_preprocessor_config.json", "generation_config.json",
+}
+for name in sorted(set(list_repo_files(source_model_id, revision=source_revision)) & assets):
+    copyfile(hf_hub_download(source_model_id, name, revision=source_revision), folder / name)
 ```
 </details>
 - kernels: 0.17.0
 - safetensors: 0.8.0
 - sglang: 0.5.23.dev384+g9ab74b851
 - torch: 2.14.1
 - transformers: 5.19.0
 - 2026-10-23: Initial version.
 - 2026-10-23: Reduced all repo files below 50 MB; GPU-validated TP=2, MTP and CUDA Graph.
+- 2026-10-23: Creation examples use only public dependencies; source assets are copied unchanged.
 This repo is generated by GPT-6.1-Sol