flint-mic / build.json
tarekziade's picture
tarekziade HF Staff
Publish MIC usage, slim fixture and reviewed Transformers expectations
a56aa27 verified
Raw History Blame Contribute Delete
4.67 kB
{
"fixture": "huggingface-mic/flint-mic",
"key": "45ae0da829158f4cbb0428319bf4920f3ade363002b7a3a8ea00190a62e2f5eb",
"key_fields": {
"architectures": [
"FlintForCausalLM"
],
"output_hidden_states": false,
"return_dict": true,
"chunk_size_feed_forward": 0,
"is_encoder_decoder": false,
"id2label": {
"0": "LABEL_0",
"1": "LABEL_1"
},
"label2id": {
"LABEL_0": 0,
"LABEL_1": 1
},
"problem_type": null,
"vocab_size": 150144,
"hidden_size": 64,
"intermediate_size": 128,
"num_hidden_layers": 3,
"num_attention_heads": 4,
"num_key_value_heads": 2,
"hidden_act": "silu",
"max_position_embeddings": 262144,
"initializer_range": 0.02,
"rms_norm_eps": 1e-06,
"use_cache": true,
"tie_word_embeddings": false,
"rope_parameters": {
"rope_theta": 10000.0,
"rope_type": "default"
},
"sliding_window": 513,
"attention_dropout": 0.0,
"decoder_sparse_step": 1,
"moe_intermediate_size": 32,
"shared_expert_intermediate_size": 128,
"num_experts_per_tok": 2,
"num_experts": 4,
"norm_topk_prob": true,
"output_router_logits": false,
"router_aux_loss_coef": 0.0005,
"mlp_only_layers": [
0
],
"layer_types": [
"full_attention",
"sliding_attention",
"full_attention"
],
"pad_token_id": 150001,
"bos_token_id": 150000,
"eos_token_id": [
150001,
150003,
150005,
150018
],
"head_dim": 64,
"attention_bias": false,
"model_type": "flint",
"no_rope_layer_indices": [
0,
1,
5,
9,
13,
17,
21,
25,
29,
33,
37,
41,
45,
48
],
"router_score_function": "sigmoid",
"softmax_type": "learnable",
"use_sliding_window": true,
"output_attentions": false,
"dtype": "bfloat16"
},
"covers": {
"RecursiveSI/flint": "2a247be0c0aec895ccb0ac7ca92229ec6df37288"
},
"built_from": {
"repo": "RecursiveSI/flint",
"revision": "2a247be0c0aec895ccb0ac7ca92229ec6df37288"
},
"seed": 17,
"dtype": "bfloat16",
"builder": "example/build_fixtures.py",
"builder_revision": "4140f5de707b81845dc190394fa962c91c66e343",
"recipe": {
"seed": 17,
"model_class": "FlintForCausalLM"
},
"shrink": "rule",
"model_class": "FlintForCausalLM",
"parameters": 19490124,
"missing_parameters": [],
"serialization": {
"file": "model.safetensors",
"source_file": "model.safetensors.index.json",
"reverted_conversions": [
"WeightConverter: ['mlp.experts.*.gate_proj.weight', 'mlp.experts.*.up_proj.weight'] -> ['mlp.experts.gate_up_proj']",
"WeightConverter: ['mlp.experts.*.down_proj.weight'] -> ['mlp.experts.down_proj']",
"WeightRenaming: ['self_attn.softmax_offset'] -> ['self_attn.sinks']"
]
},
"checkpoint_inspection": "parameter headers/indexes; bounded binary fallback",
"numeric_health_scope": "CPU float32/bfloat16/float16; autocast for low-precision losses",
"captured": {
"call": "training step",
"inputs": [
"attention_mask",
"input_ids",
"labels"
],
"class": "FlintForCausalLM"
},
"weights": "random",
"gate": {
"random": {
"pass": true,
"accepted_insensitive": {},
"threshold": 0.001,
"deltas": {
"input_ids:roll_batch": 0.76953125,
"input_ids:reverse": 1.08984375,
"attention_mask:all_ones": 0.192626953125
},
"weakest": "attention_mask:all_ones",
"no_perturbation_defined": [],
"not_a_forward_parameter": []
}
},
"numeric_health": {
"float32": {
"outputs_finite": true,
"loss_finite": true,
"gradients_finite": true,
"finite": true
},
"bfloat16": {
"outputs_finite": true,
"loss_finite": true,
"gradients_finite": true,
"finite": true
},
"float16": {
"outputs_finite": true,
"loss_finite": true,
"unscaled_gradients_finite": true,
"finite": true
}
},
"copied_unchanged": [
"LICENSE",
"chat_template.jinja",
"generation_config.json",
"tokenizer.json",
"tokenizer_config.json"
],
"environment": {
"python": "3.11.14",
"machine": "arm64",
"torch": "2.15.0",
"transformers": "5.24.0.dev0",
"tokenizers": "0.23.3"
},
"sha256": {
"LICENSE": "43070e2d4e532684de521b885f385d0841030efa2b1a20bafb76133a5e1379c1",
"chat_template.jinja": "7940f4ffa3275c36caa4a582f845aaad79da11018b3961e8fbcad6ae29733466",
"config.json": "a0ff6842e967e11385eeac4f2ab026061b81f1f367e5256ce762d99cae4a94d3",
"generation_config.json": "a499c71895e008abd572558a5ad5904feaed92cf2a3b6e2cb0e9cabf03b96ed3",
"model.safetensors": "cd27ff248726c86e40651b57aa6a9a906498eeeffb0903a6ef83dd9095e4652e",
"tokenizer.json": "a263cce0b7e2b03f7ee07ce809783ef4b47f160c3f268a4ea1524f638da45b68",
"tokenizer_config.json": "159ac88217e90ffd0a4a294726048bb302829e82e1958dc85bb203aa79138fbf"
}
}