{ "fixture": "huggingface-mic/flint-mic", "key": "45ae0da829158f4cbb0428319bf4920f3ade363002b7a3a8ea00190a62e2f5eb", "key_fields": { "architectures": [ "FlintForCausalLM" ], "output_hidden_states": false, "return_dict": true, "chunk_size_feed_forward": 0, "is_encoder_decoder": false, "id2label": { "0": "LABEL_0", "1": "LABEL_1" }, "label2id": { "LABEL_0": 0, "LABEL_1": 1 }, "problem_type": null, "vocab_size": 150144, "hidden_size": 64, "intermediate_size": 128, "num_hidden_layers": 3, "num_attention_heads": 4, "num_key_value_heads": 2, "hidden_act": "silu", "max_position_embeddings": 262144, "initializer_range": 0.02, "rms_norm_eps": 1e-06, "use_cache": true, "tie_word_embeddings": false, "rope_parameters": { "rope_theta": 10000.0, "rope_type": "default" }, "sliding_window": 513, "attention_dropout": 0.0, "decoder_sparse_step": 1, "moe_intermediate_size": 32, "shared_expert_intermediate_size": 128, "num_experts_per_tok": 2, "num_experts": 4, "norm_topk_prob": true, "output_router_logits": false, "router_aux_loss_coef": 0.0005, "mlp_only_layers": [ 0 ], "layer_types": [ "full_attention", "sliding_attention", "full_attention" ], "pad_token_id": 150001, "bos_token_id": 150000, "eos_token_id": [ 150001, 150003, 150005, 150018 ], "head_dim": 64, "attention_bias": false, "model_type": "flint", "no_rope_layer_indices": [ 0, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 48 ], "router_score_function": "sigmoid", "softmax_type": "learnable", "use_sliding_window": true, "output_attentions": false, "dtype": "bfloat16" }, "covers": { "RecursiveSI/flint": "2a247be0c0aec895ccb0ac7ca92229ec6df37288" }, "built_from": { "repo": "RecursiveSI/flint", "revision": "2a247be0c0aec895ccb0ac7ca92229ec6df37288" }, "seed": 17, "dtype": "bfloat16", "builder": "example/build_fixtures.py", "builder_revision": "4140f5de707b81845dc190394fa962c91c66e343", "recipe": { "seed": 17, "model_class": "FlintForCausalLM" }, "shrink": "rule", "model_class": "FlintForCausalLM", "parameters": 19490124, "missing_parameters": [], "serialization": { "file": "model.safetensors", "source_file": "model.safetensors.index.json", "reverted_conversions": [ "WeightConverter: ['mlp.experts.*.gate_proj.weight', 'mlp.experts.*.up_proj.weight'] -> ['mlp.experts.gate_up_proj']", "WeightConverter: ['mlp.experts.*.down_proj.weight'] -> ['mlp.experts.down_proj']", "WeightRenaming: ['self_attn.softmax_offset'] -> ['self_attn.sinks']" ] }, "checkpoint_inspection": "parameter headers/indexes; bounded binary fallback", "numeric_health_scope": "CPU float32/bfloat16/float16; autocast for low-precision losses", "captured": { "call": "training step", "inputs": [ "attention_mask", "input_ids", "labels" ], "class": "FlintForCausalLM" }, "weights": "random", "gate": { "random": { "pass": true, "accepted_insensitive": {}, "threshold": 0.001, "deltas": { "input_ids:roll_batch": 0.76953125, "input_ids:reverse": 1.08984375, "attention_mask:all_ones": 0.192626953125 }, "weakest": "attention_mask:all_ones", "no_perturbation_defined": [], "not_a_forward_parameter": [] } }, "numeric_health": { "float32": { "outputs_finite": true, "loss_finite": true, "gradients_finite": true, "finite": true }, "bfloat16": { "outputs_finite": true, "loss_finite": true, "gradients_finite": true, "finite": true }, "float16": { "outputs_finite": true, "loss_finite": true, "unscaled_gradients_finite": true, "finite": true } }, "copied_unchanged": [ "LICENSE", "chat_template.jinja", "generation_config.json", "tokenizer.json", "tokenizer_config.json" ], "environment": { "python": "3.11.14", "machine": "arm64", "torch": "2.15.0", "transformers": "5.24.0.dev0", "tokenizers": "0.23.3" }, "sha256": { "LICENSE": "43070e2d4e532684de521b885f385d0841030efa2b1a20bafb76133a5e1379c1", "chat_template.jinja": "7940f4ffa3275c36caa4a582f845aaad79da11018b3961e8fbcad6ae29733466", "config.json": "a0ff6842e967e11385eeac4f2ab026061b81f1f367e5256ce762d99cae4a94d3", "generation_config.json": "a499c71895e008abd572558a5ad5904feaed92cf2a3b6e2cb0e9cabf03b96ed3", "model.safetensors": "cd27ff248726c86e40651b57aa6a9a906498eeeffb0903a6ef83dd9095e4652e", "tokenizer.json": "a263cce0b7e2b03f7ee07ce809783ef4b47f160c3f268a4ea1524f638da45b68", "tokenizer_config.json": "159ac88217e90ffd0a4a294726048bb302829e82e1958dc85bb203aa79138fbf" } }