| { | |
| "layers": 16, | |
| "hidden": 1024, | |
| "inter": 1536, | |
| "heads_q": 8, | |
| "heads_kv": 4, | |
| "vocab": 151939, | |
| "mask_id": 151671, | |
| "k_heads": 8, | |
| "n_big": 2097152, | |
| "n_tri": 12582912, | |
| "edim": 256, | |
| "fusion_layers": [ | |
| 1, | |
| 7 | |
| ], | |
| "teacher": "jinaai/jina-embeddings-v5-text-small-retrieval", | |
| "objective": "mlm-only-0.20-span3-8-mask100", | |
| "tokenizer": "jinaai/jina-embeddings-v5-text-small-retrieval", | |
| "schedule_version": 2, | |
| "books_epoch": 2, | |
| "books_gated": true, | |
| "warmup_steps": 2000, | |
| "engram_warmup_steps": 2000, | |
| "stage_switch_step": 50000, | |
| "stage2_lr_mult": 1.0, | |
| "stage_bridge_steps": 500, | |
| "force_stage": 1, | |
| "finalize": true, | |
| "final_decay_steps": 20000, | |
| "migration_warmup_steps": 500, | |
| "base_lr": 0.00015, | |
| "old_base_lr": 0.0003, | |
| "lr_stepdown_bridge_steps": 500 | |
| } |