{ "mic": "0.3", "name": "SmolLM3 causal LM", "usage": "AutoTokenizer + AutoModelForCausalLM, batched with left padding, logits probes, one seeded SGD step", "architecture": "SmolLM3ForCausalLM", "checkpoints": { "HuggingFaceTB/SmolLM3-3B": { "fixture": "huggingface-mic/SmolLM3-3B-mic" } }, "fixtures": { "huggingface-mic/SmolLM3-3B-mic": { "built_from": "HuggingFaceTB/SmolLM3-3B", "covers": [ "HuggingFaceTB/SmolLM3-3B" ], "why_separate": "Slim random-weight fixture keeps the real SmolLM3 config (36 layers, 2048 hidden, 128256 vocab, NoPE every 4th layer) so CPU runs stay small while the checkpoint runs on GPU.", "build": { "seed": 17, "model_class": "SmolLM3ForCausalLM", "config_overrides": { "fields": { "num_hidden_layers": 4, "hidden_size": 256, "intermediate_size": 512, "num_attention_heads": 4, "num_key_value_heads": 2, "max_position_embeddings": 512, "no_rope_layers": [ 1, 1, 1, 0 ], "layer_types": [ "full_attention", "full_attention", "full_attention", "full_attention" ], "max_window_layers": 3, "tie_word_embeddings": true }, "reason": "Shrink depth/width for a CPU-sized fixture while keeping the NoPE-every-4th-layer pattern; the vocabulary stays at 128256 because the copied tokenizer emits ids up to 128k." } } } }, "integrations": { "transformers": { "entry_point": "integrations/transformers/usage.md", "format": "runnable-markdown", "requires": [ "transformers>=5", "torch" ], "targets": [ "fixtures", "checkpoints" ], "variants": { "default": {}, "eager": { "attn_implementation": "eager" }, "bfloat16": { "dtype": "bfloat16" } }, "training": true, "resources": { "fixture": { "memory_gb": 2, "download_mb": 200 }, "HuggingFaceTB/SmolLM3-3B": { "devices": [ "cuda" ], "gpu_memory_gb": 16, "memory_gb": 16, "download_mb": 6200, "variants": { "bfloat16": { "gpu_memory_gb": 10, "memory_gb": 12 }, "default": { "devices": [] }, "eager": { "devices": [] } } } } } }, "outputs": { "runtime": { "compare": "exact" }, "next_token_ids": { "compare": "exact" }, "next_token_logits": { "compare": "allclose", "atol": 0.0001, "rtol": 0.0001 }, "training.loss": { "compare": "allclose", "atol": 0.0001, "rtol": 0.0001 }, "training.gradients": { "compare": "all_true" }, "training.updated": { "compare": "true" } } }