config.json

{
 "layers": 16,
 "hidden": 1024,
 "inter": 1536,
 "heads_q": 8,
 "heads_kv": 4,
 "vocab": 151939,
 "mask_id": 151671,
 "k_heads": 8,
 "n_big": 2097152,
 "n_tri": 12582912,
 "edim": 256,
 "fusion_layers": [
  1,
  7
 ],
 "teacher": "jinaai/jina-embeddings-v5-text-small-retrieval",
 "objective": "mlm-only-0.20-span3-8-mask100",
 "tokenizer": "jinaai/jina-embeddings-v5-text-small-retrieval",
 "schedule_version": 2,
 "books_epoch": 2,
 "books_gated": true,
 "warmup_steps": 2000,
 "engram_warmup_steps": 2000,
 "stage_switch_step": 50000,
 "stage2_lr_mult": 1.0,
 "stage_bridge_steps": 500,
 "force_stage": 1,
 "finalize": true,
 "final_decay_steps": 20000,
 "migration_warmup_steps": 500,
 "base_lr": 0.00015,
 "old_base_lr": 0.0003,
 "lr_stepdown_bridge_steps": 500
}