| { | |
| "phase": "D", | |
| "init_repo": "electroglyph/berty-phaseC", | |
| "sources": [ | |
| { | |
| "name": "msmarco-bm25", | |
| "repo": "sentence-transformers/msmarco-bm25", | |
| "pats": [ | |
| "triplet-50/*.parquet" | |
| ], | |
| "weight": 0.26, | |
| "w_end": 0.14, | |
| "fmt": "m50" | |
| }, | |
| { | |
| "name": "msmarco-dense", | |
| "repo": "sentence-transformers/msmarco-co-condenser-margin-mse-cls-v1", | |
| "pats": [ | |
| "triplet-50/*.parquet" | |
| ], | |
| "weight": 0.26, | |
| "w_end": 0.38, | |
| "fmt": "m50" | |
| }, | |
| { | |
| "name": "hotpotqa", | |
| "repo": "sentence-transformers/hotpotqa", | |
| "pats": [ | |
| "triplet-20/*.parquet" | |
| ], | |
| "weight": 0.12, | |
| "w_end": 0.12, | |
| "fmt": "h20" | |
| }, | |
| { | |
| "name": "nq", | |
| "repo": "Tevatron/wikipedia-nq", | |
| "pats": [ | |
| "nq-train.jsonl.gz" | |
| ], | |
| "weight": 0.1, | |
| "w_end": 0.1, | |
| "fmt": "dpr" | |
| }, | |
| { | |
| "name": "quora", | |
| "repo": "sentence-transformers/quora-duplicates", | |
| "pats": [ | |
| "triplet-all/*.parquet", | |
| "pair-class/*.parquet" | |
| ], | |
| "weight": 0.09, | |
| "w_end": 0.09, | |
| "fmt": "quora" | |
| }, | |
| { | |
| "name": "nli", | |
| "repo": "sentence-transformers/all-nli", | |
| "pats": [ | |
| "triplet/train-*.parquet" | |
| ], | |
| "weight": 0.07, | |
| "w_end": 0.07, | |
| "fmt": "nli" | |
| }, | |
| { | |
| "name": "pairs", | |
| "repo": "sentence-transformers/eli5", | |
| "pats": [ | |
| "pair/*.parquet" | |
| ], | |
| "weight": 0.04, | |
| "w_end": 0.04, | |
| "fmt": "qa" | |
| }, | |
| { | |
| "name": "squad", | |
| "repo": "sentence-transformers/squad", | |
| "pats": [ | |
| "pair/*.parquet" | |
| ], | |
| "weight": 0.02, | |
| "w_end": 0.02, | |
| "fmt": "qa" | |
| }, | |
| { | |
| "name": "stackex", | |
| "repo": "sentence-transformers/stackexchange-duplicates", | |
| "pats": [ | |
| "post-post-pair/*.parquet" | |
| ], | |
| "weight": 0.04, | |
| "w_end": 0.04, | |
| "fmt": "t12", | |
| "cols": [ | |
| "post1", | |
| "post2" | |
| ] | |
| } | |
| ], | |
| "base_lr": 5e-05, | |
| "seq_q": 64, | |
| "seq_d": 512, | |
| "h_frozen": [ | |
| 7, | |
| 4 | |
| ], | |
| "n_ance": [ | |
| 2, | |
| 6 | |
| ], | |
| "curr_anneal_steps": 15000, | |
| "h_hard_eval": 15, | |
| "n_pos": 2, | |
| "nce_lambda": 1.0, | |
| "bwd_lambda": 0.5, | |
| "kl_lambda": 0.4, | |
| "kl_temp": 1.0, | |
| "distill_lambda": 0.05, | |
| "reservoir": 262144, | |
| "scifact_probe": true, | |
| "batch_cfg": { | |
| "micro_q": 64, | |
| "accum": 2, | |
| "q_target": "128 queries/step", | |
| "vram": 95.0 | |
| } | |
| } |