Hugging Face's logo

electroglyph
/
berty-phaseC

Model card Files
xet
 Community
berty-phaseC
File size: 1,301 Bytes
18a4cc7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
{
 "layers": 16,
 "hidden": 1024,
 "inter": 1536,
 "heads_q": 8,
 "heads_kv": 4,
 "vocab": 151939,
 "mask_id": 151671,
 "k_heads": 8,
 "n_big": 2097152,
 "n_tri": 12582912,
 "edim": 256,
 "fusion_layers": [
  1,
  7
 ],
 "teacher": "jinaai/jina-embeddings-v5-text-small-retrieval",
 "phase": "C",
 "objective": "cosine-1.0+mrl-each-0.3+infoNCE-learnable-1.0+gor-0.1+mlm-0.1",
 "tokenizer": "jinaai/jina-embeddings-v5-text-small-retrieval",
 "schedule_version": 1,
 "books_epoch": 3,
 "seq_len": 1024,
 "seq_long": 4096,
 "long_frac": 0.25,
 "mix_short": {
  "web": 0.5,
  "books": 0.5
 },
 "mix_long": {
  "web": 0.2,
  "books": 0.8
 },
 "pool_mode": "dual",
 "pool_queries": 8,
 "pool_ablate_baseline": false,
 "query_prefix": "Query:",
 "doc_prefix": "Document:",
 "warmup_steps": 500,
 "finalize": true,
 "final_decay_steps": 40000,
 "planned_steps": 100000,
 "engram_unfreeze_frac": 0.05,
 "engram_unfreeze_step": 3863,
 "base_lr": 0.00015,
 "distill_lambda": 1.0,
 "mrl_distill_lambda": 0.3,
 "nce_lambda": 1.0,
 "gor_lambda": 0.1,
 "mlm_lambda": 0.1,
 "log_temp_init": -2.995732273553991,
 "false_neg_sim_cap": 0.95,
 "bank_size": 65536,
 "ance_refresh_steps": 5000,
 "n_hard": 3,
 "adjacent_frac": 0.5,
 "min_pair_tokens": 128,
 "mrl_lambda": 0.01,
 "phaseB_repo": "electroglyph/berty-phaseB"
}