| { |
| "name": "Ines-1", |
| "codename": "mini-v41 / mini-v41-Decisions", |
| "model_type": "mini-v41", |
| "architectures": [ |
| "MiniV41" |
| ], |
| "description": "1.59B-parameter MoE causal encoder-decoder (405M active per token) with Engram n-gram memory, fine-tuned to answer typed decisions (choice / score / noul)", |
| "parameters_total": 1590066240, |
| "parameters_active_per_token": 405052224, |
| "model": { |
| "attention_dropout": 0.0, |
| "attn_sink": false, |
| "ced_enabled": true, |
| "decoder_kv_source": "shared", |
| "decoder_layers": 8, |
| "decoder_window_size": 128, |
| "deterministic_init": true, |
| "dtype": "bf16", |
| "encoder_layers": 8, |
| "encoder_window_size": 0, |
| "engram": { |
| "backend": "token_id", |
| "embedding_dim": 64, |
| "enabled": true, |
| "hash_seed": 10007, |
| "layer_ids": [ |
| 1, |
| 5 |
| ], |
| "max_ngram": 4, |
| "n_heads": 2, |
| "num_tables": 2, |
| "pad_token_id": 0, |
| "table_rows": 1000000 |
| }, |
| "ffn_intermediate_size": 2816, |
| "head_dim": 64, |
| "hidden_size": 1024, |
| "init_seed": 0, |
| "initializer_range": 0.02, |
| "loop_count": 1, |
| "max_sequence_length": 2048, |
| "moe": { |
| "ablate_routed_experts_to_zero": false, |
| "aux_loss_coef": 0.01, |
| "bias_update_speed": 0.001, |
| "capacity_factor": 0.0, |
| "dense_layer_ids": [], |
| "enabled": true, |
| "expert_intermediate_size": 1536, |
| "experts_backend": "grouped", |
| "gate_temp": 1.0, |
| "norm_topk_prob": true, |
| "num_routed_experts": 16, |
| "num_shared_experts": 1, |
| "route_scale": 1.0, |
| "router_fp32": true, |
| "score_func": "sqrtsoftplus", |
| "shared_expert_intermediate_size": 1536, |
| "top_k": 2, |
| "use_bias_balancing": false, |
| "z_loss_coef": 0.001 |
| }, |
| "num_attention_heads": 16, |
| "num_key_value_heads": 4, |
| "recurrent_residual_scale": null, |
| "recurrent_span": null, |
| "rms_norm_eps": 1e-06, |
| "rope_theta": 10000.0, |
| "swiglu_limit": 0.0, |
| "tie_word_embeddings": false, |
| "vocab_size": 65536 |
| }, |
| "decision_reader": { |
| "options": "letters A-Z read at the first assistant position", |
| "max_options": 26, |
| "languages": [ |
| "en", |
| "es" |
| ], |
| "max_prompt_tokens": 2048, |
| "questions_per_prompt": 1 |
| }, |
| "provenance": { |
| "run_id": "decisions-v3-mix-from-v2", |
| "parent_run_id": "decisions-v2-e8", |
| "parent_weights_sha256": "d5c76527a9b03c8315df8af7abb55acbfa9ab811c68f55bbd057bf2b796c8fbf", |
| "source_weights_sha256_fp32_pt": "5657d3ee7bf9934a52a34f270aa4ebc78ae69ef788ad49c0b07cbd1fe8ebe8fb", |
| "selected_epoch": 2, |
| "val_score": 0.6444873254407529, |
| "training_recipe_hash": "e4b355103247ef81", |
| "dataset_subset_id": "train_helmo+train_jev+train_n4ze3m+train_tasksource+train_typed_en+train_typed_es+train_typed_es_telepatia", |
| "tokenizer_sha256": "859e5c63f13924ebed8dd9a3dd6690438fd084771728c1da3a220017a5e74469", |
| "engram_backend_identity": "token_id:65536:65536:197f7a314b356f70", |
| "training_code_commit": "0b5e63e3da05c1a51eb9f0ea1758f8d53d200a80-dirty", |
| "bundle_model_code_commit": "5d1ad97", |
| "bundle_release_code_commit": "329aac4", |
| "torch_version_at_training": "2.14.0+cu130" |
| } |
| } |
|
|
