{ "version": 1, "model": { "variant": "middle14", "context_policy": { "policy": "global_linear_local_native", "original_max_positions": 8192, "max_positions": 65536, "input_upper_limit": 65536, "global_factor": 8.0, "rope_parameters": { "full_attention": { "rope_type": "linear", "rope_theta": 160000.0, "factor": 8.0 }, "sliding_attention": { "rope_type": "default", "rope_theta": 10000.0 } } }, "model_repository": "convaiinnovations/laya", "declared_model_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "upstream_revision": "c7527708f9f5220c669d8aa385077cd28d04708a", "checkpoint": "/root/.cache/huggingface/hub/models--convaiinnovations--laya/snapshots/1c5edc17a7acd8701df6fc341c0d179f1c62c982", "checkpoint_sha256": { "model.safetensors": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "rl_agent_config.json": "ae287b56bbcf5f8c4f4541ae9dfd00c914c4c48b940b8398c3058af37ba92bbd", "encoder/config.json": "bf3ab80598fdccf414855a2ce80f22859e4492d06ca8a62ddd1cfb63972f8979", "tokenizer/tokenizer.json": "6c8aaa9a542084f2457eab775d4eeb51f92a70c0fd9de28d5edb0ddec3c08d30", "tokenizer/tokenizer_config.json": "50044de60daaa73df97d262e15a40d4faf0160e7d742df64b377877a1320dd12" }, "native_common_sha256": "f231d42fcec84da203222fcaa89c083b22776e00341e66e118183d754e1dcabf", "native_config": { "encoder": "answerdotai/ModernBERT-large", "head_layers": 2, "max_len": 512, "head_max_len": 192, "max_prefixes": 6, "act_costs": { "escalate": 0.5 }, "cost_wrong_act": 3.0, "amp_dtype": "bf16", "model_name": "rl-agent", "temperature": [ 1.6369030475616455, 1.2514300346374512, 1.983399510383606 ], "temperature_by_options": { "choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793 }, "training": { "updates": 7313, "epochs_completed": 1, "hours": 1.96, "world_size": 1, "fine_tuned_from_checkpoint": true } }, "encoder_config": { "transformers_version": "5.16.1", "architectures": [ "ModernBertForMaskedLM" ], "output_hidden_states": false, "return_dict": true, "dtype": "float32", "chunk_size_feed_forward": 0, "is_encoder_decoder": false, "id2label": { "0": "LABEL_0", "1": "LABEL_1" }, "label2id": { "LABEL_0": 0, "LABEL_1": 1 }, "problem_type": null, "vocab_size": 50368, "hidden_size": 1024, "intermediate_size": 2624, "num_hidden_layers": 42, "num_attention_heads": 16, "hidden_activation": "gelu", "max_position_embeddings": 65536, "initializer_range": 0.02, "initializer_cutoff_factor": 2.0, "norm_eps": 1e-05, "norm_bias": false, "pad_token_id": 50283, "eos_token_id": 50282, "bos_token_id": 50281, "cls_token_id": 50281, "sep_token_id": 50282, "attention_bias": false, "attention_dropout": 0.0, "layer_types": [ "full_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "full_attention" ], "rope_parameters": { "full_attention": { "rope_type": "linear", "rope_theta": 160000.0, "factor": 8.0 }, "sliding_attention": { "rope_type": "default", "rope_theta": 10000.0 } }, "local_attention": 128, "embedding_dropout": 0.0, "mlp_bias": false, "mlp_dropout": 0.0, "decoder_bias": true, "classifier_pooling": "mean", "classifier_dropout": 0.0, "classifier_bias": false, "classifier_activation": "gelu", "deterministic_flash_attn": false, "sparse_prediction": false, "sparse_pred_ignore_index": -100, "tie_word_embeddings": true, "_name_or_path": "/root/.cache/huggingface/hub/models--convaiinnovations--laya/snapshots/1c5edc17a7acd8701df6fc341c0d179f1c62c982/encoder", "global_attn_every_n_layers": 3, "gradient_checkpointing": false, "layer_norm_eps": 1e-05, "model_type": "modernbert", "position_embedding_type": "absolute", "repad_logits_with_grad": false, "laya_layer_origins": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27 ], "laya_variant": "middle14", "output_attentions": false }, "total_parameters": 592895747, "transformers_version": "5.16.1", "torch_version": "2.12.0+rocm7.14.0", "gradient_checkpointing_supported": true, "revision_verification": "Declared download revision; actual local file content hashes recorded independently" }, "index_sha256": "e276de114156f75bc135df2a2f84003b11743c106ffd98dc7e2bc5e676073414", "corpus_inventory": { "mapping_sha256": "8fbf15db76b54c8a91b8768eafb3204a77ab3ae985ef691c461adaf1062c8483", "included_text_decisions": 2095926, "included_by_source": { "additional_dataset": 26469, "belebele-related": 147097, "codecomplex": 8820, "contractnli": 7191, "kormedmcqa": 1890, "lexglue": 89256, "original-mmlu": 99842, "passage-rerank-csn-python": 412178, "passage-rerank-fiqa": 14131, "passage-rerank-miracl-fr": 11426, "passage-rerank-nfcorpus": 110575, "passage-rerank-nq": 480163, "passage-rerank-scifact": 919, "paws-x": 98802, "security-code": 4120, "toolret-training-20w": 480162, "wanli": 102885 }, "excluded_vision_decisions": 304074, "excluded_vision_by_source": { "additional_dataset": 1076, "vision-cifar10": 50000, "vision-oxford-pets": 3680, "vision-tallyqa": 249318 }, "language_filter": null, "language_note": "All source languages retained; English identifies model variant, not data filtering." }, "gold_decisions": 2095926, "epochs": 1, "world_size": 8, "global_batch": 1024, "per_device_batch": 128, "microbatch_max_tokens": 262144, "microbatch_max_attention_elements": 1073741824, "batch_policy": "padded_real_examples_with_token_budget_singleton_fallback", "seed": 42, "learning_rate": 0.0001, "warmup_steps": 0, "planned_optimizer_steps": 2047, "max_length": 65536, "head_max_length": 8192, "overflow_policy": "error", "format": "laya_native", "encoder_gradient_checkpointing": true, "head_gradient_checkpointing_layers": 2, "loss": "gold_decision_cross_entropy", "scope": { "trainable_parameter_names": [ "encoder.embeddings.norm.weight", "encoder.layers.0.attn.Wqkv.weight", "encoder.layers.0.attn.Wo.weight", "encoder.layers.0.mlp_norm.weight", "encoder.layers.0.mlp.Wi.weight", "encoder.layers.0.mlp.Wo.weight", "encoder.layers.1.attn_norm.weight", "encoder.layers.1.attn.Wqkv.weight", "encoder.layers.1.attn.Wo.weight", "encoder.layers.1.mlp_norm.weight", "encoder.layers.1.mlp.Wi.weight", "encoder.layers.1.mlp.Wo.weight", "encoder.layers.2.attn_norm.weight", "encoder.layers.2.attn.Wqkv.weight", "encoder.layers.2.attn.Wo.weight", "encoder.layers.2.mlp_norm.weight", "encoder.layers.2.mlp.Wi.weight", "encoder.layers.2.mlp.Wo.weight", "encoder.layers.3.attn_norm.weight", "encoder.layers.3.attn.Wqkv.weight", "encoder.layers.3.attn.Wo.weight", "encoder.layers.3.mlp_norm.weight", "encoder.layers.3.mlp.Wi.weight", "encoder.layers.3.mlp.Wo.weight", "encoder.layers.4.attn_norm.weight", "encoder.layers.4.attn.Wqkv.weight", "encoder.layers.4.attn.Wo.weight", "encoder.layers.4.mlp_norm.weight", "encoder.layers.4.mlp.Wi.weight", "encoder.layers.4.mlp.Wo.weight", "encoder.layers.5.attn_norm.weight", "encoder.layers.5.attn.Wqkv.weight", "encoder.layers.5.attn.Wo.weight", "encoder.layers.5.mlp_norm.weight", "encoder.layers.5.mlp.Wi.weight", "encoder.layers.5.mlp.Wo.weight", "encoder.layers.6.attn_norm.weight", "encoder.layers.6.attn.Wqkv.weight", "encoder.layers.6.attn.Wo.weight", "encoder.layers.6.mlp_norm.weight", "encoder.layers.6.mlp.Wi.weight", "encoder.layers.6.mlp.Wo.weight", "encoder.layers.7.attn_norm.weight", "encoder.layers.7.attn.Wqkv.weight", "encoder.layers.7.attn.Wo.weight", "encoder.layers.7.mlp_norm.weight", "encoder.layers.7.mlp.Wi.weight", "encoder.layers.7.mlp.Wo.weight", "encoder.layers.8.attn_norm.weight", "encoder.layers.8.attn.Wqkv.weight", "encoder.layers.8.attn.Wo.weight", "encoder.layers.8.mlp_norm.weight", "encoder.layers.8.mlp.Wi.weight", "encoder.layers.8.mlp.Wo.weight", "encoder.layers.9.attn_norm.weight", "encoder.layers.9.attn.Wqkv.weight", "encoder.layers.9.attn.Wo.weight", "encoder.layers.9.mlp_norm.weight", "encoder.layers.9.mlp.Wi.weight", "encoder.layers.9.mlp.Wo.weight", "encoder.layers.10.attn_norm.weight", "encoder.layers.10.attn.Wqkv.weight", "encoder.layers.10.attn.Wo.weight", "encoder.layers.10.mlp_norm.weight", "encoder.layers.10.mlp.Wi.weight", "encoder.layers.10.mlp.Wo.weight", "encoder.layers.11.attn_norm.weight", "encoder.layers.11.attn.Wqkv.weight", "encoder.layers.11.attn.Wo.weight", "encoder.layers.11.mlp_norm.weight", "encoder.layers.11.mlp.Wi.weight", "encoder.layers.11.mlp.Wo.weight", "encoder.layers.12.attn_norm.weight", "encoder.layers.12.attn.Wqkv.weight", "encoder.layers.12.attn.Wo.weight", "encoder.layers.12.mlp_norm.weight", "encoder.layers.12.mlp.Wi.weight", "encoder.layers.12.mlp.Wo.weight", "encoder.layers.13.attn_norm.weight", "encoder.layers.13.attn.Wqkv.weight", "encoder.layers.13.attn.Wo.weight", "encoder.layers.13.mlp_norm.weight", "encoder.layers.13.mlp.Wi.weight", "encoder.layers.13.mlp.Wo.weight", "encoder.layers.14.attn_norm.weight", "encoder.layers.14.attn.Wqkv.weight", "encoder.layers.14.attn.Wo.weight", "encoder.layers.14.mlp_norm.weight", "encoder.layers.14.mlp.Wi.weight", "encoder.layers.14.mlp.Wo.weight", "encoder.layers.15.attn_norm.weight", "encoder.layers.15.attn.Wqkv.weight", "encoder.layers.15.attn.Wo.weight", "encoder.layers.15.mlp_norm.weight", "encoder.layers.15.mlp.Wi.weight", "encoder.layers.15.mlp.Wo.weight", "encoder.layers.16.attn_norm.weight", "encoder.layers.16.attn.Wqkv.weight", "encoder.layers.16.attn.Wo.weight", "encoder.layers.16.mlp_norm.weight", "encoder.layers.16.mlp.Wi.weight", "encoder.layers.16.mlp.Wo.weight", "encoder.layers.17.attn_norm.weight", "encoder.layers.17.attn.Wqkv.weight", "encoder.layers.17.attn.Wo.weight", "encoder.layers.17.mlp_norm.weight", "encoder.layers.17.mlp.Wi.weight", "encoder.layers.17.mlp.Wo.weight", "encoder.layers.18.attn_norm.weight", "encoder.layers.18.attn.Wqkv.weight", "encoder.layers.18.attn.Wo.weight", "encoder.layers.18.mlp_norm.weight", "encoder.layers.18.mlp.Wi.weight", "encoder.layers.18.mlp.Wo.weight", "encoder.layers.19.attn_norm.weight", "encoder.layers.19.attn.Wqkv.weight", "encoder.layers.19.attn.Wo.weight", "encoder.layers.19.mlp_norm.weight", "encoder.layers.19.mlp.Wi.weight", "encoder.layers.19.mlp.Wo.weight", "encoder.layers.20.attn_norm.weight", "encoder.layers.20.attn.Wqkv.weight", "encoder.layers.20.attn.Wo.weight", "encoder.layers.20.mlp_norm.weight", "encoder.layers.20.mlp.Wi.weight", "encoder.layers.20.mlp.Wo.weight", "encoder.layers.21.attn_norm.weight", "encoder.layers.21.attn.Wqkv.weight", "encoder.layers.21.attn.Wo.weight", "encoder.layers.21.mlp_norm.weight", "encoder.layers.21.mlp.Wi.weight", "encoder.layers.21.mlp.Wo.weight", "encoder.layers.22.attn_norm.weight", "encoder.layers.22.attn.Wqkv.weight", "encoder.layers.22.attn.Wo.weight", "encoder.layers.22.mlp_norm.weight", "encoder.layers.22.mlp.Wi.weight", "encoder.layers.22.mlp.Wo.weight", "encoder.layers.23.attn_norm.weight", "encoder.layers.23.attn.Wqkv.weight", "encoder.layers.23.attn.Wo.weight", "encoder.layers.23.mlp_norm.weight", "encoder.layers.23.mlp.Wi.weight", "encoder.layers.23.mlp.Wo.weight", "encoder.layers.24.attn_norm.weight", "encoder.layers.24.attn.Wqkv.weight", "encoder.layers.24.attn.Wo.weight", "encoder.layers.24.mlp_norm.weight", "encoder.layers.24.mlp.Wi.weight", "encoder.layers.24.mlp.Wo.weight", "encoder.layers.25.attn_norm.weight", "encoder.layers.25.attn.Wqkv.weight", "encoder.layers.25.attn.Wo.weight", "encoder.layers.25.mlp_norm.weight", "encoder.layers.25.mlp.Wi.weight", "encoder.layers.25.mlp.Wo.weight", "encoder.layers.26.attn_norm.weight", "encoder.layers.26.attn.Wqkv.weight", "encoder.layers.26.attn.Wo.weight", "encoder.layers.26.mlp_norm.weight", "encoder.layers.26.mlp.Wi.weight", "encoder.layers.26.mlp.Wo.weight", "encoder.layers.27.attn_norm.weight", "encoder.layers.27.attn.Wqkv.weight", "encoder.layers.27.attn.Wo.weight", "encoder.layers.27.mlp_norm.weight", "encoder.layers.27.mlp.Wi.weight", "encoder.layers.27.mlp.Wo.weight", "encoder.layers.28.attn_norm.weight", "encoder.layers.28.attn.Wqkv.weight", "encoder.layers.28.attn.Wo.weight", "encoder.layers.28.mlp_norm.weight", "encoder.layers.28.mlp.Wi.weight", "encoder.layers.28.mlp.Wo.weight", "encoder.layers.29.attn_norm.weight", "encoder.layers.29.attn.Wqkv.weight", "encoder.layers.29.attn.Wo.weight", "encoder.layers.29.mlp_norm.weight", "encoder.layers.29.mlp.Wi.weight", "encoder.layers.29.mlp.Wo.weight", "encoder.layers.30.attn_norm.weight", "encoder.layers.30.attn.Wqkv.weight", "encoder.layers.30.attn.Wo.weight", "encoder.layers.30.mlp_norm.weight", "encoder.layers.30.mlp.Wi.weight", "encoder.layers.30.mlp.Wo.weight", "encoder.layers.31.attn_norm.weight", "encoder.layers.31.attn.Wqkv.weight", "encoder.layers.31.attn.Wo.weight", "encoder.layers.31.mlp_norm.weight", "encoder.layers.31.mlp.Wi.weight", "encoder.layers.31.mlp.Wo.weight", "encoder.layers.32.attn_norm.weight", "encoder.layers.32.attn.Wqkv.weight", "encoder.layers.32.attn.Wo.weight", "encoder.layers.32.mlp_norm.weight", "encoder.layers.32.mlp.Wi.weight", "encoder.layers.32.mlp.Wo.weight", "encoder.layers.33.attn_norm.weight", "encoder.layers.33.attn.Wqkv.weight", "encoder.layers.33.attn.Wo.weight", "encoder.layers.33.mlp_norm.weight", "encoder.layers.33.mlp.Wi.weight", "encoder.layers.33.mlp.Wo.weight", "encoder.layers.34.attn_norm.weight", "encoder.layers.34.attn.Wqkv.weight", "encoder.layers.34.attn.Wo.weight", "encoder.layers.34.mlp_norm.weight", "encoder.layers.34.mlp.Wi.weight", "encoder.layers.34.mlp.Wo.weight", "encoder.layers.35.attn_norm.weight", "encoder.layers.35.attn.Wqkv.weight", "encoder.layers.35.attn.Wo.weight", "encoder.layers.35.mlp_norm.weight", "encoder.layers.35.mlp.Wi.weight", "encoder.layers.35.mlp.Wo.weight", "encoder.layers.36.attn_norm.weight", "encoder.layers.36.attn.Wqkv.weight", "encoder.layers.36.attn.Wo.weight", "encoder.layers.36.mlp_norm.weight", "encoder.layers.36.mlp.Wi.weight", "encoder.layers.36.mlp.Wo.weight", "encoder.layers.37.attn_norm.weight", "encoder.layers.37.attn.Wqkv.weight", "encoder.layers.37.attn.Wo.weight", "encoder.layers.37.mlp_norm.weight", "encoder.layers.37.mlp.Wi.weight", "encoder.layers.37.mlp.Wo.weight", "encoder.layers.38.attn_norm.weight", "encoder.layers.38.attn.Wqkv.weight", "encoder.layers.38.attn.Wo.weight", "encoder.layers.38.mlp_norm.weight", "encoder.layers.38.mlp.Wi.weight", "encoder.layers.38.mlp.Wo.weight", "encoder.layers.39.attn_norm.weight", "encoder.layers.39.attn.Wqkv.weight", "encoder.layers.39.attn.Wo.weight", "encoder.layers.39.mlp_norm.weight", "encoder.layers.39.mlp.Wi.weight", "encoder.layers.39.mlp.Wo.weight", "encoder.layers.40.attn_norm.weight", "encoder.layers.40.attn.Wqkv.weight", "encoder.layers.40.attn.Wo.weight", "encoder.layers.40.mlp_norm.weight", "encoder.layers.40.mlp.Wi.weight", "encoder.layers.40.mlp.Wo.weight", "encoder.layers.41.attn_norm.weight", "encoder.layers.41.attn.Wqkv.weight", "encoder.layers.41.attn.Wo.weight", "encoder.layers.41.mlp_norm.weight", "encoder.layers.41.mlp.Wi.weight", "encoder.layers.41.mlp.Wo.weight", "encoder.final_norm.weight", "head.layers.0.self_attn.in_proj_weight", "head.layers.0.self_attn.in_proj_bias", "head.layers.0.self_attn.out_proj.weight", "head.layers.0.self_attn.out_proj.bias", "head.layers.0.linear1.weight", "head.layers.0.linear1.bias", "head.layers.0.linear2.weight", "head.layers.0.linear2.bias", "head.layers.0.norm1.weight", "head.layers.0.norm1.bias", "head.layers.0.norm2.weight", "head.layers.0.norm2.bias", "head.layers.1.self_attn.in_proj_weight", "head.layers.1.self_attn.in_proj_bias", "head.layers.1.self_attn.out_proj.weight", "head.layers.1.self_attn.out_proj.bias", "head.layers.1.linear1.weight", "head.layers.1.linear1.bias", "head.layers.1.linear2.weight", "head.layers.1.linear2.bias", "head.layers.1.norm1.weight", "head.layers.1.norm1.bias", "head.layers.1.norm2.weight", "head.layers.1.norm2.bias", "type_emb.weight", "scorer.0.weight", "scorer.0.bias", "scorer.1.weight", "scorer.1.bias", "scorer.3.weight", "scorer.3.bias" ], "frozen_parameter_names": [ "encoder.embeddings.tok_embeddings.weight", "act_head.0.weight", "act_head.0.bias", "act_head.2.weight", "act_head.2.bias" ], "trainable_parameters": 541054977, "frozen_parameters": 51840770, "input_embeddings_frozen": true, "action_head_frozen": true, "trainable_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16" }, "quick_manifest_sha256": "661b4083e6957c4718e564cefddd425b87158ffb4e9e996f8eb065cf5180a099", "eval_steps": 4, "source_sha256": { "finetune/experiment/laya/train_batched.py": "2de91e8a1cd0dbbb02e628e9ac8e8b1bf646ee883692c27f8caa7c5f9c7dc7ab", "finetune/experiment/laya/model.py": "49145c9b85abe11b1191ebecd985202c9b321a4f50cab80f496d3f6dd1ac17b2", "finetune/experiment/laya/gold_adapter.py": "81a23a640d67f6d07a89227b17e4f5fccdebb3f930c7d6e772dba2e6b54c4c19", "finetune/experiment/epoch_coverage.py": "7c8181968ffc075f5a9c21667bf44484ca6a19527c34c68737a84c01b7ba3dff", "finetune/experiment/checkpoint_publication.py": "db6b97589a2500512cf74a9cfc4adcd46189cde4b0711cd12a5d6164c6f0c5d8", "finetune/experiment/quick_eval.py": "4423877807ac0419afb0e132c5b84c03360f8443cfc8b6dfcab292169db98f17", "finetune/experiment/runtime_projection.py": "c8dd4bab78ac10846f5093a6e4974b456792d089b6711f0979398964656406b3", "finetune/prepare_dataset/common.py": "6fcb200112576d432a75c27ff9b4afae82812a2e17038112a9c949c854d01808", "finetune/experiment/laya/variants.py": "e6eff79eb5272fbb8d4efdc6cccd2dddc1780e30b3ced72880b8b5e1f6e5799d", "finetune/experiment/laya/launch_batched.py": "5abe70e44c3271bce79e40432625d3b507538cc47bd7bd85d261cb53ea0403ad", "finetune/experiment/laya/upstream/laya/common.py": "f231d42fcec84da203222fcaa89c083b22776e00341e66e118183d754e1dcabf" } }