| { | |
| "layers": 16, | |
| "hidden": 1024, | |
| "inter": 1536, | |
| "heads_q": 8, | |
| "heads_kv": 4, | |
| "vocab": 151939, | |
| "mask_id": 151671, | |
| "k_heads": 8, | |
| "n_big": 2097152, | |
| "n_tri": 12582912, | |
| "edim": 256, | |
| "fusion_layers": [ | |
| 1, | |
| 7 | |
| ], | |
| "teacher": "jinaai/jina-embeddings-v5-text-small-retrieval", | |
| "phase": "B", | |
| "objective": "joint-mlm-0.20-span3-8-mask100+infoNCE", | |
| "tokenizer": "jinaai/jina-embeddings-v5-text-small-retrieval", | |
| "schedule_version": 1, | |
| "books_epoch": 2, | |
| "seq_len": 1024, | |
| "mix": { | |
| "web": 0.35, | |
| "books": 0.65 | |
| }, | |
| "warmup_steps": 500, | |
| "finalize": false, | |
| "final_decay_steps": 20000, | |
| "base_lr": 0.00015, | |
| "contrast_lambda": 1.0, | |
| "contrast_temp": 0.1, | |
| "false_neg_sim_cap": 0.95, | |
| "adjacent_frac": 1.0, | |
| "min_pair_tokens": 128, | |
| "mrl_lambda": 0.01, | |
| "phaseA_repo": "electroglyph/berty-phaseA" | |
| } |