Hugging Face's logo

electroglyph
/
berty-phaseD

Model card Files
xet
 Community
berty-phaseD
File size: 2,164 Bytes
cd3d68a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
{
 "phase": "D",
 "init_repo": "electroglyph/berty-phaseC",
 "sources": [
  {
   "name": "msmarco-bm25",
   "repo": "sentence-transformers/msmarco-bm25",
   "pats": [
    "triplet-50/*.parquet"
   ],
   "weight": 0.26,
   "w_end": 0.14,
   "fmt": "m50"
  },
  {
   "name": "msmarco-dense",
   "repo": "sentence-transformers/msmarco-co-condenser-margin-mse-cls-v1",
   "pats": [
    "triplet-50/*.parquet"
   ],
   "weight": 0.26,
   "w_end": 0.38,
   "fmt": "m50"
  },
  {
   "name": "hotpotqa",
   "repo": "sentence-transformers/hotpotqa",
   "pats": [
    "triplet-20/*.parquet"
   ],
   "weight": 0.12,
   "w_end": 0.12,
   "fmt": "h20"
  },
  {
   "name": "nq",
   "repo": "Tevatron/wikipedia-nq",
   "pats": [
    "nq-train.jsonl.gz"
   ],
   "weight": 0.1,
   "w_end": 0.1,
   "fmt": "dpr"
  },
  {
   "name": "quora",
   "repo": "sentence-transformers/quora-duplicates",
   "pats": [
    "triplet-all/*.parquet",
    "pair-class/*.parquet"
   ],
   "weight": 0.09,
   "w_end": 0.09,
   "fmt": "quora"
  },
  {
   "name": "nli",
   "repo": "sentence-transformers/all-nli",
   "pats": [
    "triplet/train-*.parquet"
   ],
   "weight": 0.07,
   "w_end": 0.07,
   "fmt": "nli"
  },
  {
   "name": "pairs",
   "repo": "sentence-transformers/eli5",
   "pats": [
    "pair/*.parquet"
   ],
   "weight": 0.04,
   "w_end": 0.04,
   "fmt": "qa"
  },
  {
   "name": "squad",
   "repo": "sentence-transformers/squad",
   "pats": [
    "pair/*.parquet"
   ],
   "weight": 0.02,
   "w_end": 0.02,
   "fmt": "qa"
  },
  {
   "name": "stackex",
   "repo": "sentence-transformers/stackexchange-duplicates",
   "pats": [
    "post-post-pair/*.parquet"
   ],
   "weight": 0.04,
   "w_end": 0.04,
   "fmt": "t12",
   "cols": [
    "post1",
    "post2"
   ]
  }
 ],
 "base_lr": 5e-05,
 "seq_q": 64,
 "seq_d": 512,
 "h_frozen": [
  7,
  4
 ],
 "n_ance": [
  2,
  6
 ],
 "curr_anneal_steps": 15000,
 "h_hard_eval": 15,
 "n_pos": 2,
 "nce_lambda": 1.0,
 "bwd_lambda": 0.5,
 "kl_lambda": 0.4,
 "kl_temp": 1.0,
 "distill_lambda": 0.05,
 "reservoir": 262144,
 "scifact_probe": true,
 "batch_cfg": {
  "micro_q": 64,
  "accum": 2,
  "q_target": "128 queries/step",
  "vram": 95.0
 }
}