esirvan commited on
Commit
4296bdc
·
verified ·
1 Parent(s): 83cfd16

LoCUS models: the four paper models and the four pretraining-split models (no chr20/21)

Browse files
README.md CHANGED
@@ -17,9 +17,15 @@ Code, fine-tuning recipes and the scripts that reproduce every result of the pap
17
  | `locus-2m/` | LoCUS-2M: width 128, 4 kb context |
18
  | `twin-fixed-2m/` | Tokenizer study: LoCUS-2M with fixed 10 bp segments every 5 bp |
19
  | `twin-bpe-2m/` | Tokenizer study: LoCUS-2M with BPE tokens (vocabulary 4,096) as segments |
 
 
 
 
20
 
21
- Each folder has `config.json` (the pretraining configuration and step) and `model.safetensors` (the weights). All four
22
- are pretrained on GRCh38 chromosomes 3-22.
 
 
23
 
24
  ```python
25
  import locus
 
17
  | `locus-2m/` | LoCUS-2M: width 128, 4 kb context |
18
  | `twin-fixed-2m/` | Tokenizer study: LoCUS-2M with fixed 10 bp segments every 5 bp |
19
  | `twin-bpe-2m/` | Tokenizer study: LoCUS-2M with BPE tokens (vocabulary 4,096) as segments |
20
+ | `locus-0.5m-no-chr20-21/` | Pretraining-split ablation: LoCUS-0.5M pretrained without chromosomes 20 and 21 |
21
+ | `locus-2m-no-chr20-21/` | Pretraining-split ablation: LoCUS-2M pretrained without chromosomes 20 and 21 |
22
+ | `twin-fixed-2m-no-chr20-21/` | Pretraining-split ablation: twin-fixed-2m pretrained without chromosomes 20 and 21 |
23
+ | `twin-bpe-2m-no-chr20-21/` | Pretraining-split ablation: twin-bpe-2m pretrained without chromosomes 20 and 21 (its BPE vocabulary trained on those chromosomes) |
24
 
25
+ Each folder has `config.json` (the pretraining configuration and step), `model.safetensors` (the weights) and a model card.
26
+ The first four are pretrained on GRCh38 chromosomes 3-22. The four `-no-chr20-21` models are the same models pretrained on
27
+ chromosomes 1-19 and 22, without the chromosomes of the Nucleotide Transformer benchmark's test sets, everything else
28
+ unchanged; the paper uses them to check that seeing the test chromosomes in pretraining does not inflate the test scores.
29
 
30
  ```python
31
  import locus
locus-0.5m-no-chr20-21/README.md ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: locus
3
+ tags:
4
+ - dna
5
+ - genomics
6
+ - locally-consistent-parsing
7
+ ---
8
+
9
+ # locus-0.5m-no-chr20-21
10
+
11
+ Pretraining-split ablation: LoCUS-0.5M pretrained without chromosomes 20 and 21 (the NT18 test chromosomes).
12
+
13
+ - Segments: lcp (level 2)
14
+ - Width 64, 2 heads, base attention radii [32, 128], 6 segment layers
15
+ - Pretrained for 359,466 steps on GRCh38 chromosomes 1-19 and 22, crops of 256-4096 bp
16
+
17
+ ## Use
18
+
19
+ ```python
20
+ import locus
21
+
22
+ model = locus.load("locus-0.5m-no-chr20-21")
23
+ vectors = model.embed(["ACGT" * 100])
24
+ ```
25
+
26
+ Code, fine-tuning recipes and the scripts that reproduce every result of the paper: see the LoCUS repository.
locus-0.5m-no-chr20-21/config.json ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "model": {
4
+ "model_dim": 64,
5
+ "attention_heads": 2,
6
+ "ffn_dim": 256,
7
+ "base_attention_radii": [
8
+ 32,
9
+ 128
10
+ ],
11
+ "segment_layers": 6,
12
+ "base_embedding_dim": 16,
13
+ "dropout": 0.1,
14
+ "max_segment_length": 512,
15
+ "rope_theta": 10000.0,
16
+ "rope_position_scale": 5.0
17
+ },
18
+ "data": {
19
+ "genome_windows": "data/GRCh38.p14.acgt.w4096.s4096",
20
+ "context_length": 4096,
21
+ "min_context_length": 256,
22
+ "crops_per_read": 2,
23
+ "train_sources": [
24
+ "NC_000001.11",
25
+ "NC_000002.12",
26
+ "NC_000003.12",
27
+ "NC_000004.12",
28
+ "NC_000005.10",
29
+ "NC_000006.12",
30
+ "NC_000007.14",
31
+ "NC_000008.11",
32
+ "NC_000009.12",
33
+ "NC_000010.11",
34
+ "NC_000011.10",
35
+ "NC_000012.12",
36
+ "NC_000013.11",
37
+ "NC_000014.9",
38
+ "NC_000015.10",
39
+ "NC_000016.10",
40
+ "NC_000017.11",
41
+ "NC_000018.10",
42
+ "NC_000019.10",
43
+ "NC_000022.11"
44
+ ],
45
+ "validation_sources": [],
46
+ "test_sources": [
47
+ "NC_000020.11",
48
+ "NC_000021.9"
49
+ ],
50
+ "target_margin": 32,
51
+ "mask_probability": 0.15,
52
+ "mask_replacement_probability": 0.8,
53
+ "random_replacement_probability": 0.1,
54
+ "segmenter": "lcp",
55
+ "lcp_level": 2,
56
+ "lcp_library": "build/lcptools/lib/liblcptools.so",
57
+ "fixed_length": 10,
58
+ "fixed_stride": 5,
59
+ "bpe_tokenizer": "data/tokenizers/bpe-4096.json",
60
+ "example_workers": 3
61
+ },
62
+ "training": {
63
+ "run_name": "locus-0.5m-no-chr20-21",
64
+ "seed": 42,
65
+ "batch_size": 16,
66
+ "max_steps": 359466,
67
+ "learning_rate": 0.0006,
68
+ "warmup_steps": 3600,
69
+ "weight_decay": 0.01,
70
+ "adam_beta1": 0.9,
71
+ "adam_beta2": 0.98,
72
+ "adam_epsilon": 1e-06,
73
+ "gradient_clip_norm": 1.0,
74
+ "shuffle_buffer_size": 512,
75
+ "mixed_precision": true,
76
+ "device": "cuda",
77
+ "log_every": 10,
78
+ "checkpoint_every": 1000,
79
+ "keep_checkpoint_every": 10000,
80
+ "keep_checkpoint_steps": []
81
+ }
82
+ },
83
+ "step": 359466
84
+ }
locus-0.5m-no-chr20-21/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:454eed66505ce9a40eac2199ce0c529f8bb8e0e0b3eda9939ab2af811161d44a
3
+ size 1897864
locus-2m-no-chr20-21/README.md ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: locus
3
+ tags:
4
+ - dna
5
+ - genomics
6
+ - locally-consistent-parsing
7
+ ---
8
+
9
+ # locus-2m-no-chr20-21
10
+
11
+ Pretraining-split ablation: LoCUS-2M pretrained without chromosomes 20 and 21 (the NT18 test chromosomes).
12
+
13
+ - Segments: lcp (level 2)
14
+ - Width 128, 4 heads, base attention radii [32, 128], 6 segment layers
15
+ - Pretrained for 359,466 steps on GRCh38 chromosomes 1-19 and 22, crops of 256-4096 bp
16
+
17
+ ## Use
18
+
19
+ ```python
20
+ import locus
21
+
22
+ model = locus.load("locus-2m-no-chr20-21")
23
+ vectors = model.embed(["ACGT" * 100])
24
+ ```
25
+
26
+ Code, fine-tuning recipes and the scripts that reproduce every result of the paper: see the LoCUS repository.
locus-2m-no-chr20-21/config.json ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "model": {
4
+ "model_dim": 128,
5
+ "attention_heads": 4,
6
+ "ffn_dim": 512,
7
+ "base_attention_radii": [
8
+ 32,
9
+ 128
10
+ ],
11
+ "segment_layers": 6,
12
+ "base_embedding_dim": 16,
13
+ "dropout": 0.1,
14
+ "max_segment_length": 512,
15
+ "rope_theta": 10000.0,
16
+ "rope_position_scale": 5.0
17
+ },
18
+ "data": {
19
+ "genome_windows": "data/GRCh38.p14.acgt.w4096.s4096",
20
+ "context_length": 4096,
21
+ "min_context_length": 256,
22
+ "crops_per_read": 2,
23
+ "train_sources": [
24
+ "NC_000001.11",
25
+ "NC_000002.12",
26
+ "NC_000003.12",
27
+ "NC_000004.12",
28
+ "NC_000005.10",
29
+ "NC_000006.12",
30
+ "NC_000007.14",
31
+ "NC_000008.11",
32
+ "NC_000009.12",
33
+ "NC_000010.11",
34
+ "NC_000011.10",
35
+ "NC_000012.12",
36
+ "NC_000013.11",
37
+ "NC_000014.9",
38
+ "NC_000015.10",
39
+ "NC_000016.10",
40
+ "NC_000017.11",
41
+ "NC_000018.10",
42
+ "NC_000019.10",
43
+ "NC_000022.11"
44
+ ],
45
+ "validation_sources": [],
46
+ "test_sources": [
47
+ "NC_000020.11",
48
+ "NC_000021.9"
49
+ ],
50
+ "target_margin": 32,
51
+ "mask_probability": 0.15,
52
+ "mask_replacement_probability": 0.8,
53
+ "random_replacement_probability": 0.1,
54
+ "segmenter": "lcp",
55
+ "lcp_level": 2,
56
+ "lcp_library": "build/lcptools/lib/liblcptools.so",
57
+ "fixed_length": 10,
58
+ "fixed_stride": 5,
59
+ "bpe_tokenizer": "data/tokenizers/bpe-4096.json",
60
+ "example_workers": 3
61
+ },
62
+ "training": {
63
+ "run_name": "locus-2m-no-chr20-21",
64
+ "seed": 42,
65
+ "batch_size": 16,
66
+ "max_steps": 359466,
67
+ "learning_rate": 0.0006,
68
+ "warmup_steps": 3600,
69
+ "weight_decay": 0.01,
70
+ "adam_beta1": 0.9,
71
+ "adam_beta2": 0.98,
72
+ "adam_epsilon": 1e-06,
73
+ "gradient_clip_norm": 1.0,
74
+ "shuffle_buffer_size": 512,
75
+ "mixed_precision": true,
76
+ "device": "cuda",
77
+ "log_every": 10,
78
+ "checkpoint_every": 1000,
79
+ "keep_checkpoint_every": 10000,
80
+ "keep_checkpoint_steps": []
81
+ }
82
+ },
83
+ "step": 359466
84
+ }
locus-2m-no-chr20-21/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ccd2c6d71be96579b5d8ba5810dd4e2bee28935eb58e010de6867f14bb91994f
3
+ size 6962256
twin-bpe-2m-no-chr20-21/README.md ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: locus
3
+ tags:
4
+ - dna
5
+ - genomics
6
+ - locally-consistent-parsing
7
+ ---
8
+
9
+ # twin-bpe-2m-no-chr20-21
10
+
11
+ Pretraining-split ablation: twin-bpe-2m pretrained without chromosomes 20 and 21, with a BPE vocabulary trained on its own pretraining chromosomes.
12
+
13
+ - Segments: bpe
14
+ - Width 128, 4 heads, base attention radii [32, 128], 6 segment layers
15
+ - Pretrained for 359,466 steps on GRCh38 chromosomes 1-19 and 22, crops of 256-4096 bp
16
+
17
+ ## Use
18
+
19
+ ```python
20
+ import locus
21
+
22
+ model = locus.load("twin-bpe-2m-no-chr20-21")
23
+ vectors = model.embed(["ACGT" * 100])
24
+ ```
25
+
26
+ Code, fine-tuning recipes and the scripts that reproduce every result of the paper: see the LoCUS repository.
twin-bpe-2m-no-chr20-21/config.json ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "model": {
4
+ "model_dim": 128,
5
+ "attention_heads": 4,
6
+ "ffn_dim": 512,
7
+ "base_attention_radii": [
8
+ 32,
9
+ 128
10
+ ],
11
+ "segment_layers": 6,
12
+ "base_embedding_dim": 16,
13
+ "dropout": 0.1,
14
+ "max_segment_length": 512,
15
+ "rope_theta": 10000.0,
16
+ "rope_position_scale": 5.0
17
+ },
18
+ "data": {
19
+ "genome_windows": "data/GRCh38.p14.acgt.w4096.s4096",
20
+ "context_length": 4096,
21
+ "min_context_length": 256,
22
+ "crops_per_read": 2,
23
+ "train_sources": [
24
+ "NC_000001.11",
25
+ "NC_000002.12",
26
+ "NC_000003.12",
27
+ "NC_000004.12",
28
+ "NC_000005.10",
29
+ "NC_000006.12",
30
+ "NC_000007.14",
31
+ "NC_000008.11",
32
+ "NC_000009.12",
33
+ "NC_000010.11",
34
+ "NC_000011.10",
35
+ "NC_000012.12",
36
+ "NC_000013.11",
37
+ "NC_000014.9",
38
+ "NC_000015.10",
39
+ "NC_000016.10",
40
+ "NC_000017.11",
41
+ "NC_000018.10",
42
+ "NC_000019.10",
43
+ "NC_000022.11"
44
+ ],
45
+ "validation_sources": [],
46
+ "test_sources": [
47
+ "NC_000020.11",
48
+ "NC_000021.9"
49
+ ],
50
+ "target_margin": 32,
51
+ "mask_probability": 0.15,
52
+ "mask_replacement_probability": 0.8,
53
+ "random_replacement_probability": 0.1,
54
+ "segmenter": "bpe",
55
+ "lcp_level": 2,
56
+ "lcp_library": "build/lcptools/lib/liblcptools.so",
57
+ "fixed_length": 10,
58
+ "fixed_stride": 5,
59
+ "bpe_tokenizer": "data/tokenizers/bpe-4096-1gb-seed42-no-chr20-21.json",
60
+ "example_workers": 3
61
+ },
62
+ "training": {
63
+ "run_name": "twin-bpe-2m-no-chr20-21",
64
+ "seed": 42,
65
+ "batch_size": 16,
66
+ "max_steps": 359466,
67
+ "learning_rate": 0.0006,
68
+ "warmup_steps": 3600,
69
+ "weight_decay": 0.01,
70
+ "adam_beta1": 0.9,
71
+ "adam_beta2": 0.98,
72
+ "adam_epsilon": 1e-06,
73
+ "gradient_clip_norm": 1.0,
74
+ "shuffle_buffer_size": 512,
75
+ "mixed_precision": true,
76
+ "device": "cuda",
77
+ "log_every": 10,
78
+ "checkpoint_every": 1000,
79
+ "keep_checkpoint_every": 10000,
80
+ "keep_checkpoint_steps": []
81
+ }
82
+ },
83
+ "step": 359466
84
+ }
twin-bpe-2m-no-chr20-21/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:60c0b45139c7e65fd02a12a50301656925c4496cac988a7454d207513001330f
3
+ size 6962256
twin-fixed-2m-no-chr20-21/README.md ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: locus
3
+ tags:
4
+ - dna
5
+ - genomics
6
+ - locally-consistent-parsing
7
+ ---
8
+
9
+ # twin-fixed-2m-no-chr20-21
10
+
11
+ Pretraining-split ablation: twin-fixed-2m pretrained without chromosomes 20 and 21.
12
+
13
+ - Segments: fixed
14
+ - Width 128, 4 heads, base attention radii [32, 128], 6 segment layers
15
+ - Pretrained for 359,466 steps on GRCh38 chromosomes 1-19 and 22, crops of 256-4096 bp
16
+
17
+ ## Use
18
+
19
+ ```python
20
+ import locus
21
+
22
+ model = locus.load("twin-fixed-2m-no-chr20-21")
23
+ vectors = model.embed(["ACGT" * 100])
24
+ ```
25
+
26
+ Code, fine-tuning recipes and the scripts that reproduce every result of the paper: see the LoCUS repository.
twin-fixed-2m-no-chr20-21/config.json ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "model": {
4
+ "model_dim": 128,
5
+ "attention_heads": 4,
6
+ "ffn_dim": 512,
7
+ "base_attention_radii": [
8
+ 32,
9
+ 128
10
+ ],
11
+ "segment_layers": 6,
12
+ "base_embedding_dim": 16,
13
+ "dropout": 0.1,
14
+ "max_segment_length": 512,
15
+ "rope_theta": 10000.0,
16
+ "rope_position_scale": 5.0
17
+ },
18
+ "data": {
19
+ "genome_windows": "data/GRCh38.p14.acgt.w4096.s4096",
20
+ "context_length": 4096,
21
+ "min_context_length": 256,
22
+ "crops_per_read": 2,
23
+ "train_sources": [
24
+ "NC_000001.11",
25
+ "NC_000002.12",
26
+ "NC_000003.12",
27
+ "NC_000004.12",
28
+ "NC_000005.10",
29
+ "NC_000006.12",
30
+ "NC_000007.14",
31
+ "NC_000008.11",
32
+ "NC_000009.12",
33
+ "NC_000010.11",
34
+ "NC_000011.10",
35
+ "NC_000012.12",
36
+ "NC_000013.11",
37
+ "NC_000014.9",
38
+ "NC_000015.10",
39
+ "NC_000016.10",
40
+ "NC_000017.11",
41
+ "NC_000018.10",
42
+ "NC_000019.10",
43
+ "NC_000022.11"
44
+ ],
45
+ "validation_sources": [],
46
+ "test_sources": [
47
+ "NC_000020.11",
48
+ "NC_000021.9"
49
+ ],
50
+ "target_margin": 32,
51
+ "mask_probability": 0.15,
52
+ "mask_replacement_probability": 0.8,
53
+ "random_replacement_probability": 0.1,
54
+ "segmenter": "fixed",
55
+ "lcp_level": 2,
56
+ "lcp_library": "build/lcptools/lib/liblcptools.so",
57
+ "fixed_length": 10,
58
+ "fixed_stride": 5,
59
+ "bpe_tokenizer": "data/tokenizers/bpe-4096.json",
60
+ "example_workers": 3
61
+ },
62
+ "training": {
63
+ "run_name": "twin-fixed-2m-no-chr20-21",
64
+ "seed": 42,
65
+ "batch_size": 16,
66
+ "max_steps": 359466,
67
+ "learning_rate": 0.0006,
68
+ "warmup_steps": 3600,
69
+ "weight_decay": 0.01,
70
+ "adam_beta1": 0.9,
71
+ "adam_beta2": 0.98,
72
+ "adam_epsilon": 1e-06,
73
+ "gradient_clip_norm": 1.0,
74
+ "shuffle_buffer_size": 512,
75
+ "mixed_precision": true,
76
+ "device": "cuda",
77
+ "log_every": 10,
78
+ "checkpoint_every": 1000,
79
+ "keep_checkpoint_every": 10000,
80
+ "keep_checkpoint_steps": []
81
+ }
82
+ },
83
+ "step": 359466
84
+ }
twin-fixed-2m-no-chr20-21/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:58064a38204331a0ef69ddd99a3f29ef645c1627e8261aa059d946f4b948ebe6
3
+ size 6962256