| { | |
| "layers": 16, | |
| "hidden": 1024, | |
| "inter": 1536, | |
| "heads_q": 8, | |
| "heads_kv": 4, | |
| "vocab": 151939, | |
| "mask_id": 151671, | |
| "k_heads": 8, | |
| "n_big": 2097152, | |
| "n_tri": 12582912, | |
| "edim": 256, | |
| "fusion_layers": [ | |
| 1, | |
| 7 | |
| ], | |
| "teacher": "jinaai/jina-embeddings-v5-text-small-retrieval", | |
| "phase": "C", | |
| "objective": "cosine-1.0+mrl-each-0.3+infoNCE-learnable-1.0+gor-0.1+mlm-0.1", | |
| "tokenizer": "jinaai/jina-embeddings-v5-text-small-retrieval", | |
| "schedule_version": 1, | |
| "books_epoch": 3, | |
| "seq_len": 1024, | |
| "seq_long": 4096, | |
| "long_frac": 0.25, | |
| "mix_short": { | |
| "web": 0.5, | |
| "books": 0.5 | |
| }, | |
| "mix_long": { | |
| "web": 0.2, | |
| "books": 0.8 | |
| }, | |
| "pool_mode": "dual", | |
| "pool_queries": 8, | |
| "pool_ablate_baseline": false, | |
| "query_prefix": "Query:", | |
| "doc_prefix": "Document:", | |
| "warmup_steps": 500, | |
| "finalize": true, | |
| "final_decay_steps": 40000, | |
| "planned_steps": 100000, | |
| "engram_unfreeze_frac": 0.05, | |
| "engram_unfreeze_step": 3863, | |
| "base_lr": 0.00015, | |
| "distill_lambda": 1.0, | |
| "mrl_distill_lambda": 0.3, | |
| "nce_lambda": 1.0, | |
| "gor_lambda": 0.1, | |
| "mlm_lambda": 0.1, | |
| "log_temp_init": -2.995732273553991, | |
| "false_neg_sim_cap": 0.95, | |
| "bank_size": 65536, | |
| "ance_refresh_steps": 5000, | |
| "n_hard": 3, | |
| "adjacent_frac": 0.5, | |
| "min_pair_tokens": 128, | |
| "mrl_lambda": 0.01, | |
| "phaseB_repo": "electroglyph/berty-phaseB" | |
| } |