tarekziade HF Staff
Publish MIC usage, slim fixture and reviewed Transformers expectations
a56aa27 verified Download build.json from huggingface-mic/flint-mic: direct link, hf CLI and curl.
- Browser
- Download file 4.67 kB
-
https://huggingface.co/huggingface-mic/flint-mic/resolve/main/build.json
- Command line
-
hf download hf://huggingface-mic/flint-mic/build.json
-
curl -L -o build.json https://huggingface.co/huggingface-mic/flint-mic/resolve/main/build.json
4.67 kB
| { | |
| "fixture": "huggingface-mic/flint-mic", | |
| "key": "45ae0da829158f4cbb0428319bf4920f3ade363002b7a3a8ea00190a62e2f5eb", | |
| "key_fields": { | |
| "architectures": [ | |
| "FlintForCausalLM" | |
| ], | |
| "output_hidden_states": false, | |
| "return_dict": true, | |
| "chunk_size_feed_forward": 0, | |
| "is_encoder_decoder": false, | |
| "id2label": { | |
| "0": "LABEL_0", | |
| "1": "LABEL_1" | |
| }, | |
| "label2id": { | |
| "LABEL_0": 0, | |
| "LABEL_1": 1 | |
| }, | |
| "problem_type": null, | |
| "vocab_size": 150144, | |
| "hidden_size": 64, | |
| "intermediate_size": 128, | |
| "num_hidden_layers": 3, | |
| "num_attention_heads": 4, | |
| "num_key_value_heads": 2, | |
| "hidden_act": "silu", | |
| "max_position_embeddings": 262144, | |
| "initializer_range": 0.02, | |
| "rms_norm_eps": 1e-06, | |
| "use_cache": true, | |
| "tie_word_embeddings": false, | |
| "rope_parameters": { | |
| "rope_theta": 10000.0, | |
| "rope_type": "default" | |
| }, | |
| "sliding_window": 513, | |
| "attention_dropout": 0.0, | |
| "decoder_sparse_step": 1, | |
| "moe_intermediate_size": 32, | |
| "shared_expert_intermediate_size": 128, | |
| "num_experts_per_tok": 2, | |
| "num_experts": 4, | |
| "norm_topk_prob": true, | |
| "output_router_logits": false, | |
| "router_aux_loss_coef": 0.0005, | |
| "mlp_only_layers": [ | |
| 0 | |
| ], | |
| "layer_types": [ | |
| "full_attention", | |
| "sliding_attention", | |
| "full_attention" | |
| ], | |
| "pad_token_id": 150001, | |
| "bos_token_id": 150000, | |
| "eos_token_id": [ | |
| 150001, | |
| 150003, | |
| 150005, | |
| 150018 | |
| ], | |
| "head_dim": 64, | |
| "attention_bias": false, | |
| "model_type": "flint", | |
| "no_rope_layer_indices": [ | |
| 0, | |
| 1, | |
| 5, | |
| 9, | |
| 13, | |
| 17, | |
| 21, | |
| 25, | |
| 29, | |
| 33, | |
| 37, | |
| 41, | |
| 45, | |
| 48 | |
| ], | |
| "router_score_function": "sigmoid", | |
| "softmax_type": "learnable", | |
| "use_sliding_window": true, | |
| "output_attentions": false, | |
| "dtype": "bfloat16" | |
| }, | |
| "covers": { | |
| "RecursiveSI/flint": "2a247be0c0aec895ccb0ac7ca92229ec6df37288" | |
| }, | |
| "built_from": { | |
| "repo": "RecursiveSI/flint", | |
| "revision": "2a247be0c0aec895ccb0ac7ca92229ec6df37288" | |
| }, | |
| "seed": 17, | |
| "dtype": "bfloat16", | |
| "builder": "example/build_fixtures.py", | |
| "builder_revision": "4140f5de707b81845dc190394fa962c91c66e343", | |
| "recipe": { | |
| "seed": 17, | |
| "model_class": "FlintForCausalLM" | |
| }, | |
| "shrink": "rule", | |
| "model_class": "FlintForCausalLM", | |
| "parameters": 19490124, | |
| "missing_parameters": [], | |
| "serialization": { | |
| "file": "model.safetensors", | |
| "source_file": "model.safetensors.index.json", | |
| "reverted_conversions": [ | |
| "WeightConverter: ['mlp.experts.*.gate_proj.weight', 'mlp.experts.*.up_proj.weight'] -> ['mlp.experts.gate_up_proj']", | |
| "WeightConverter: ['mlp.experts.*.down_proj.weight'] -> ['mlp.experts.down_proj']", | |
| "WeightRenaming: ['self_attn.softmax_offset'] -> ['self_attn.sinks']" | |
| ] | |
| }, | |
| "checkpoint_inspection": "parameter headers/indexes; bounded binary fallback", | |
| "numeric_health_scope": "CPU float32/bfloat16/float16; autocast for low-precision losses", | |
| "captured": { | |
| "call": "training step", | |
| "inputs": [ | |
| "attention_mask", | |
| "input_ids", | |
| "labels" | |
| ], | |
| "class": "FlintForCausalLM" | |
| }, | |
| "weights": "random", | |
| "gate": { | |
| "random": { | |
| "pass": true, | |
| "accepted_insensitive": {}, | |
| "threshold": 0.001, | |
| "deltas": { | |
| "input_ids:roll_batch": 0.76953125, | |
| "input_ids:reverse": 1.08984375, | |
| "attention_mask:all_ones": 0.192626953125 | |
| }, | |
| "weakest": "attention_mask:all_ones", | |
| "no_perturbation_defined": [], | |
| "not_a_forward_parameter": [] | |
| } | |
| }, | |
| "numeric_health": { | |
| "float32": { | |
| "outputs_finite": true, | |
| "loss_finite": true, | |
| "gradients_finite": true, | |
| "finite": true | |
| }, | |
| "bfloat16": { | |
| "outputs_finite": true, | |
| "loss_finite": true, | |
| "gradients_finite": true, | |
| "finite": true | |
| }, | |
| "float16": { | |
| "outputs_finite": true, | |
| "loss_finite": true, | |
| "unscaled_gradients_finite": true, | |
| "finite": true | |
| } | |
| }, | |
| "copied_unchanged": [ | |
| "LICENSE", | |
| "chat_template.jinja", | |
| "generation_config.json", | |
| "tokenizer.json", | |
| "tokenizer_config.json" | |
| ], | |
| "environment": { | |
| "python": "3.11.14", | |
| "machine": "arm64", | |
| "torch": "2.15.0", | |
| "transformers": "5.24.0.dev0", | |
| "tokenizers": "0.23.3" | |
| }, | |
| "sha256": { | |
| "LICENSE": "43070e2d4e532684de521b885f385d0841030efa2b1a20bafb76133a5e1379c1", | |
| "chat_template.jinja": "7940f4ffa3275c36caa4a582f845aaad79da11018b3961e8fbcad6ae29733466", | |
| "config.json": "a0ff6842e967e11385eeac4f2ab026061b81f1f367e5256ce762d99cae4a94d3", | |
| "generation_config.json": "a499c71895e008abd572558a5ad5904feaed92cf2a3b6e2cb0e9cabf03b96ed3", | |
| "model.safetensors": "cd27ff248726c86e40651b57aa6a9a906498eeeffb0903a6ef83dd9095e4652e", | |
| "tokenizer.json": "a263cce0b7e2b03f7ee07ce809783ef4b47f160c3f268a4ea1524f638da45b68", | |
| "tokenizer_config.json": "159ac88217e90ffd0a4a294726048bb302829e82e1958dc85bb203aa79138fbf" | |
| } | |
| } | |