Hugging Face's logo

electroglyph
/
berty-phaseB

Model card Files
xet
 Community
berty-phaseB
File size: 805 Bytes
9f3c596
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
{
 "layers": 16,
 "hidden": 1024,
 "inter": 1536,
 "heads_q": 8,
 "heads_kv": 4,
 "vocab": 151939,
 "mask_id": 151671,
 "k_heads": 8,
 "n_big": 2097152,
 "n_tri": 12582912,
 "edim": 256,
 "fusion_layers": [
  1,
  7
 ],
 "teacher": "jinaai/jina-embeddings-v5-text-small-retrieval",
 "phase": "B",
 "objective": "joint-mlm-0.20-span3-8-mask100+infoNCE",
 "tokenizer": "jinaai/jina-embeddings-v5-text-small-retrieval",
 "schedule_version": 1,
 "books_epoch": 2,
 "seq_len": 1024,
 "mix": {
  "web": 0.35,
  "books": 0.65
 },
 "warmup_steps": 500,
 "finalize": false,
 "final_decay_steps": 20000,
 "base_lr": 0.00015,
 "contrast_lambda": 1.0,
 "contrast_temp": 0.1,
 "false_neg_sim_cap": 0.95,
 "adjacent_frac": 1.0,
 "min_pair_tokens": 128,
 "mrl_lambda": 0.01,
 "phaseA_repo": "electroglyph/berty-phaseA"
}