Hugging Face's logo

jyliuAI
/
diffusion_baseline

LiteRT
TensorBoard
Core ML
ONNX
Safetensors
Model card Files
xet
 Metrics Community
jyliuAI commited on
Commit
93cd35a
·
verified ·
1 Parent(s): bfdae48

Add files using upload-large-folder tool

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. artifacts/checkpoint_lock.before-local-refresh.json +872 -0
  2. checkpoints/reference_reproduction/candi/owt/lightning_logs/version_2/metrics.csv +0 -0
  3. checkpoints/reference_reproduction/duo_di4c/owt.before-teacher-fix/stdout.log +0 -0
  4. data/manifests/conditional-lm1b-c64.json +21 -0
  5. data/manifests/lm1b.json +126 -0
  6. data/manifests/owt.json +286 -0
  7. data/processed/.builder-cache/lm1b/train/generator/default-3a58fdf789a93b0f/0.0.0.incomplete_info.lock +0 -0
  8. data/processed/.builder-cache/lm1b/train/generator/default-3a58fdf789a93b0f/0.0.0/dataset_info.json +1 -0
  9. data/processed/.builder-cache/lm1b/train/generator/default-3a58fdf789a93b0f/0.0.0_builder.lock +0 -0
  10. data/processed/.builder-cache/lm1b/validation/_home_ruizzhao_diffusion_baseline_data_processed_.builder-cache_lm1b_validation_generator_default-62c652dc08a03c0b_0.0.0.lock +0 -0
  11. data/processed/.builder-cache/lm1b/validation/generator/default-49fddb9f1246d928/0.0.0.incomplete_info.lock +0 -0
  12. data/processed/.builder-cache/lm1b/validation/generator/default-49fddb9f1246d928/0.0.0/dataset_info.json +1 -0
  13. data/processed/.builder-cache/lm1b/validation/generator/default-49fddb9f1246d928/0.0.0_builder.lock +0 -0
  14. data/processed/.builder-cache/lm1b/validation/generator/default-62c652dc08a03c0b/0.0.0/dataset_info.json +1 -0
  15. data/processed/.builder-cache/lm1b/validation/generator/default-62c652dc08a03c0b/0.0.0_builder.lock +0 -0
  16. data/processed/.builder-cache/owt/train/generator/default-b8b56016fa13f36d/0.0.0/dataset_info.json +1 -0
  17. data/processed/.builder-cache/owt/train/generator/default-b8b56016fa13f36d/0.0.0_builder.lock +0 -0
  18. data/processed/.builder-cache/owt/validation/_home_ruizzhao_diffusion_baseline_data_processed_.builder-cache_owt_validation_generator_default-bec011984d977746_0.0.0.lock +0 -0
  19. data/processed/.builder-cache/owt/validation/generator/default-0c41487195ccfdac/0.0.0/dataset_info.json +1 -0
  20. data/processed/lm1b-bert-128/dataset_dict.json +1 -0
  21. data/processed/lm1b-bert-128/train/dataset_info.json +43 -0
  22. data/processed/lm1b-bert-128/train/state.json +22 -0
  23. data/processed/lm1b-bert-128/validation/dataset_info.json +37 -0
  24. data/processed/lm1b-bert-128/validation/state.json +13 -0
  25. data/processed/owt-gpt2-1024/train/dataset_info.json +75 -0
  26. data/processed/owt-gpt2-1024/train/state.json +118 -0
  27. data/processed/owt-gpt2-1024/validation/state.json +13 -0
  28. data/raw/huggingface/datasets/Skylion007___openwebtext/plain_text/0.0.0/79d93d786212f7344586290adb811d4ae6a1762c.incomplete_info.lock +0 -0
  29. data/raw/huggingface/datasets/Skylion007___openwebtext/plain_text/0.0.0/79d93d786212f7344586290adb811d4ae6a1762c/dataset_info.json +1 -0
  30. data/raw/huggingface/datasets/Skylion007___openwebtext/plain_text/0.0.0/79d93d786212f7344586290adb811d4ae6a1762c_builder.lock +0 -0
  31. data/raw/huggingface/datasets/_home_ruizzhao_diffusion_baseline_data_raw_huggingface_datasets_Skylion007___openwebtext_plain_text_0.0.0_79d93d786212f7344586290adb811d4ae6a1762c.lock +0 -0
  32. data/raw/huggingface/datasets/_home_ruizzhao_diffusion_baseline_data_raw_huggingface_datasets_parquet_default-7e9cb4a2e4e2a71c_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +0 -0
  33. data/raw/huggingface/datasets/parquet/default-7e9cb4a2e4e2a71c/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.incomplete_info.lock +0 -0
  34. data/raw/huggingface/datasets/parquet/default-7e9cb4a2e4e2a71c/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +1 -0
  35. data/raw/huggingface/datasets/parquet/default-7e9cb4a2e4e2a71c/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +0 -0
  36. data/raw/huggingface/hub/models--bert-base-uncased/blobs/40a2aaca31dd005eb5f6ffad07b5ffed0a31d1f6 +251 -0
  37. data/raw/huggingface/hub/models--bert-base-uncased/blobs/45a2321a7ecfdaaf60a6c1fd7f5463994cc8907d +23 -0
  38. data/raw/huggingface/hub/models--bert-base-uncased/blobs/505a7adf8be9e5fdf06aabbfbe9046e6c811f91b +11 -0
  39. data/raw/huggingface/hub/models--bert-base-uncased/blobs/949a6f013d67eb8a5b4b5b46026217b888021b88 +0 -0
  40. data/raw/huggingface/hub/models--bert-base-uncased/blobs/c1c37cd58b9eb000ddbb7ca90f04b893a33e50c8 +18 -0
  41. data/raw/huggingface/hub/models--bert-base-uncased/blobs/e5c73d8a50df1f56fb5b0b8002d7cf4010afdccb +1 -0
  42. data/raw/huggingface/hub/models--bert-base-uncased/blobs/f49a4e16e68b128803cc2dcea614603632b04eac +201 -0
  43. data/raw/huggingface/hub/models--bert-base-uncased/blobs/fb140275c155a9c7c5a3b3e0e77a9e839594a938 +0 -0
  44. data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594/.gitattributes +11 -0
  45. data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594/LICENSE +201 -0
  46. data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594/README.md +251 -0
  47. data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594/config.json +23 -0
  48. data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594/coreml/fill-mask/float32_model.mlpackage/Manifest.json +18 -0
  49. data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594/tokenizer.json +0 -0
  50. data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594/tokenizer_config.json +1 -0
artifacts/checkpoint_lock.before-local-refresh.json ADDED
@@ -0,0 +1,872 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "created_at": "2026-08-21T03:35:06.671839Z",
3
+ "manifest_sha256": "4efe3a3fafa3314f13963e81f828590ac771c101e228735166bd23c21e79b69e",
4
+ "resources": {
5
+ "di4c_mdlm_owt_zenodo": {
6
+ "backend": "zenodo",
7
+ "destination": "checkpoints/official/mdlm_di4c/owt",
8
+ "files": [
9
+ {
10
+ "path": "checkpoints/official/mdlm_di4c/owt/sdtt7-di4c2.ckpt",
11
+ "sha256": "4af61572dc82fe012ce395a2e4e4be0585ea09076597ba1f7241ca196b97ecb0",
12
+ "size_bytes": 1673838666
13
+ }
14
+ ],
15
+ "provenance": "official",
16
+ "quarantined": "/home/ruizzhao/diffusion_baseline/checkpoints/quarantine/20260821T033505Z/official/mdlm_di4c/owt/sdtt7-di4c2.ckpt",
17
+ "status": "downloaded",
18
+ "teacher_family": "masked_mdlm"
19
+ },
20
+ "duo_dcd_owt_hf": {
21
+ "backend": "huggingface",
22
+ "destination": "checkpoints/official/duo_dcd/owt",
23
+ "files": [
24
+ {
25
+ "path": "checkpoints/official/duo_dcd/owt/.cache/huggingface/.gitignore",
26
+ "sha256": "684888c0ebb17f374298b65ee2807526c066094c701bcc7ebbe1c1095f494fc1",
27
+ "size_bytes": 1
28
+ },
29
+ {
30
+ "path": "checkpoints/official/duo_dcd/owt/.cache/huggingface/download/README.md.lock",
31
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
32
+ "size_bytes": 0
33
+ },
34
+ {
35
+ "path": "checkpoints/official/duo_dcd/owt/.cache/huggingface/download/README.md.metadata",
36
+ "sha256": "89dfdb2e67c648647f8e91602b0fe55d24bf0874f13c03d3348a84a860e40a2d",
37
+ "size_bytes": 100
38
+ },
39
+ {
40
+ "path": "checkpoints/official/duo_dcd/owt/.cache/huggingface/download/config.json.lock",
41
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
42
+ "size_bytes": 0
43
+ },
44
+ {
45
+ "path": "checkpoints/official/duo_dcd/owt/.cache/huggingface/download/config.json.metadata",
46
+ "sha256": "35e5b91c97ea71e112961957f72aead36f3b113619472f5acc94a2e646f44b29",
47
+ "size_bytes": 101
48
+ },
49
+ {
50
+ "path": "checkpoints/official/duo_dcd/owt/.cache/huggingface/download/config.py.lock",
51
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
52
+ "size_bytes": 0
53
+ },
54
+ {
55
+ "path": "checkpoints/official/duo_dcd/owt/.cache/huggingface/download/config.py.metadata",
56
+ "sha256": "071ede6944f08c3ec9518df427f2bee48543bf8a4ff46a1eb62aab098d196b29",
57
+ "size_bytes": 100
58
+ },
59
+ {
60
+ "path": "checkpoints/official/duo_dcd/owt/.cache/huggingface/download/model.py.lock",
61
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
62
+ "size_bytes": 0
63
+ },
64
+ {
65
+ "path": "checkpoints/official/duo_dcd/owt/.cache/huggingface/download/model.py.metadata",
66
+ "sha256": "c5fc52c946f1f73c6f4737f04c8ae039f9c7964eb76aa75500825a2d3b675690",
67
+ "size_bytes": 101
68
+ },
69
+ {
70
+ "path": "checkpoints/official/duo_dcd/owt/.cache/huggingface/download/model.safetensors.lock",
71
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
72
+ "size_bytes": 0
73
+ },
74
+ {
75
+ "path": "checkpoints/official/duo_dcd/owt/.cache/huggingface/download/model.safetensors.metadata",
76
+ "sha256": "10e89e2ca4f3e3839c1130222f694870c944225ca2427b17c5dc8cd092881a79",
77
+ "size_bytes": 125
78
+ },
79
+ {
80
+ "path": "checkpoints/official/duo_dcd/owt/README.md",
81
+ "sha256": "710bb04cdc48ed112a3bb83472bb744162e6c8e07b6850c4ed27a317e0255376",
82
+ "size_bytes": 1766
83
+ },
84
+ {
85
+ "path": "checkpoints/official/duo_dcd/owt/config.json",
86
+ "sha256": "5c4f85638cb277355344c47c31b43b3c47fd4d9e6f932e30b56ff8f4f76317e1",
87
+ "size_bytes": 471
88
+ },
89
+ {
90
+ "path": "checkpoints/official/duo_dcd/owt/config.py",
91
+ "sha256": "1e87050b1728e4862c10fd6740edfb6c0fbab7beebfc854d7a445cbe4cf1dee1",
92
+ "size_bytes": 718
93
+ },
94
+ {
95
+ "path": "checkpoints/official/duo_dcd/owt/model.py",
96
+ "sha256": "dcfef1c27da31fce4d6a7449a2759461070f5ed825b5e13c38361b228dade58d",
97
+ "size_bytes": 18736
98
+ },
99
+ {
100
+ "path": "checkpoints/official/duo_dcd/owt/model.safetensors",
101
+ "sha256": "c4b66c48780483f7079e47b1a56e6245a12d83057533455f447e822ef20fe93f",
102
+ "size_bytes": 678522728
103
+ }
104
+ ],
105
+ "provenance": "official",
106
+ "quarantined": "/home/ruizzhao/diffusion_baseline/checkpoints/quarantine/20260821T024808Z/official/duo_dcd/owt",
107
+ "status": "downloaded",
108
+ "teacher_family": "uniform_duo"
109
+ },
110
+ "duo_owt_hf": {
111
+ "backend": "huggingface",
112
+ "destination": "checkpoints/official/duo/owt",
113
+ "files": [
114
+ {
115
+ "path": "checkpoints/official/duo/owt/.cache/huggingface/.gitignore",
116
+ "sha256": "684888c0ebb17f374298b65ee2807526c066094c701bcc7ebbe1c1095f494fc1",
117
+ "size_bytes": 1
118
+ },
119
+ {
120
+ "path": "checkpoints/official/duo/owt/.cache/huggingface/download/README.md.lock",
121
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
122
+ "size_bytes": 0
123
+ },
124
+ {
125
+ "path": "checkpoints/official/duo/owt/.cache/huggingface/download/README.md.metadata",
126
+ "sha256": "6455b55147e3991ee8e6894fe1704e66d0c077969bc22ffc34122126928ff310",
127
+ "size_bytes": 101
128
+ },
129
+ {
130
+ "path": "checkpoints/official/duo/owt/.cache/huggingface/download/config.json.lock",
131
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
132
+ "size_bytes": 0
133
+ },
134
+ {
135
+ "path": "checkpoints/official/duo/owt/.cache/huggingface/download/config.json.metadata",
136
+ "sha256": "f4ab1e9b73ee10ebe44689080aea889c6d6b39d3c6ba15c416743612ba3bc19c",
137
+ "size_bytes": 101
138
+ },
139
+ {
140
+ "path": "checkpoints/official/duo/owt/.cache/huggingface/download/config.py.lock",
141
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
142
+ "size_bytes": 0
143
+ },
144
+ {
145
+ "path": "checkpoints/official/duo/owt/.cache/huggingface/download/config.py.metadata",
146
+ "sha256": "db3d53068734175219d5438066432ae55e29efecc4eff7688cf9bffe1ec32d84",
147
+ "size_bytes": 100
148
+ },
149
+ {
150
+ "path": "checkpoints/official/duo/owt/.cache/huggingface/download/model.py.lock",
151
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
152
+ "size_bytes": 0
153
+ },
154
+ {
155
+ "path": "checkpoints/official/duo/owt/.cache/huggingface/download/model.py.metadata",
156
+ "sha256": "0e88d075126d7f8f8d63198a8316072a80bad43ab29a767d39564b95e7af5200",
157
+ "size_bytes": 101
158
+ },
159
+ {
160
+ "path": "checkpoints/official/duo/owt/.cache/huggingface/download/model.safetensors.lock",
161
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
162
+ "size_bytes": 0
163
+ },
164
+ {
165
+ "path": "checkpoints/official/duo/owt/.cache/huggingface/download/model.safetensors.metadata",
166
+ "sha256": "8c1b3955cac90657c5823b69ff961e1ff6276c0eeb825826cb5f6f8c14cc47fb",
167
+ "size_bytes": 125
168
+ },
169
+ {
170
+ "path": "checkpoints/official/duo/owt/README.md",
171
+ "sha256": "26b42ac8aa7f015c08ee0106b7b580ead5cbd7f36abf6a0fcb1504badcb15dd2",
172
+ "size_bytes": 1781
173
+ },
174
+ {
175
+ "path": "checkpoints/official/duo/owt/config.json",
176
+ "sha256": "d133d602759160b42108668782c2ec154ddbb066873359cde89b42554af61c20",
177
+ "size_bytes": 461
178
+ },
179
+ {
180
+ "path": "checkpoints/official/duo/owt/config.py",
181
+ "sha256": "1e87050b1728e4862c10fd6740edfb6c0fbab7beebfc854d7a445cbe4cf1dee1",
182
+ "size_bytes": 718
183
+ },
184
+ {
185
+ "path": "checkpoints/official/duo/owt/model.py",
186
+ "sha256": "dcfef1c27da31fce4d6a7449a2759461070f5ed825b5e13c38361b228dade58d",
187
+ "size_bytes": 18736
188
+ },
189
+ {
190
+ "path": "checkpoints/official/duo/owt/model.safetensors",
191
+ "sha256": "c9e98629579f5dc7c7fcf15cd10c66f585fd6cfa31a9ce4ac09255bc84ec9ad3",
192
+ "size_bytes": 678522728
193
+ }
194
+ ],
195
+ "provenance": "official",
196
+ "quarantined": "/home/ruizzhao/diffusion_baseline/checkpoints/quarantine/20260821T024800Z/official/duo/owt",
197
+ "status": "downloaded",
198
+ "teacher_family": "uniform_duo"
199
+ },
200
+ "duo_raw_owt_drive": {
201
+ "backend": "gdrive",
202
+ "destination": "checkpoints/official/duo/raw_owt",
203
+ "files": [
204
+ {
205
+ "path": "checkpoints/official/duo/raw_owt/duo-distilled.ckpt",
206
+ "sha256": "23dba8a6cc97500f3798cb35484c8219a6c8a611b38f9dc92e2ae23946f593a9",
207
+ "size_bytes": 2715755352
208
+ },
209
+ {
210
+ "path": "checkpoints/official/duo/raw_owt/duo.ckpt",
211
+ "sha256": "ea8edb134515121c8d998b43c08517a53b1d6797aed9d80117f7322d10316c4c",
212
+ "size_bytes": 2715755416
213
+ }
214
+ ],
215
+ "provenance": "official",
216
+ "quarantined": "/home/ruizzhao/diffusion_baseline/checkpoints/quarantine/20260821T025727Z/official/duo/raw_owt",
217
+ "status": "downloaded",
218
+ "teacher_family": "uniform_duo"
219
+ },
220
+ "flm_lm1b_hf": {
221
+ "backend": "huggingface",
222
+ "destination": "checkpoints/official/flm/lm1b",
223
+ "files": [
224
+ {
225
+ "path": "checkpoints/official/flm/lm1b/.cache/huggingface/.gitignore",
226
+ "sha256": "684888c0ebb17f374298b65ee2807526c066094c701bcc7ebbe1c1095f494fc1",
227
+ "size_bytes": 1
228
+ },
229
+ {
230
+ "path": "checkpoints/official/flm/lm1b/.cache/huggingface/download/README.md.lock",
231
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
232
+ "size_bytes": 0
233
+ },
234
+ {
235
+ "path": "checkpoints/official/flm/lm1b/.cache/huggingface/download/README.md.metadata",
236
+ "sha256": "031e6a7f41631e186d002c546f234c6f941ff527c1dad90f734bff0c2d47e118",
237
+ "size_bytes": 101
238
+ },
239
+ {
240
+ "path": "checkpoints/official/flm/lm1b/.cache/huggingface/download/config.json.lock",
241
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
242
+ "size_bytes": 0
243
+ },
244
+ {
245
+ "path": "checkpoints/official/flm/lm1b/.cache/huggingface/download/config.json.metadata",
246
+ "sha256": "12154b0c361cedbe8e6cb5d5fdcfc327ca36c5d7b15f478032eb2a0af1244460",
247
+ "size_bytes": 101
248
+ },
249
+ {
250
+ "path": "checkpoints/official/flm/lm1b/.cache/huggingface/download/model.safetensors.lock",
251
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
252
+ "size_bytes": 0
253
+ },
254
+ {
255
+ "path": "checkpoints/official/flm/lm1b/.cache/huggingface/download/model.safetensors.metadata",
256
+ "sha256": "f11213edd95e935262b31d8db0a41f6d87f533bd56e60eada9d9ca00b6e5a5a6",
257
+ "size_bytes": 124
258
+ },
259
+ {
260
+ "path": "checkpoints/official/flm/lm1b/README.md",
261
+ "sha256": "d14e8ef58d4c3696def2121665a2a25f00cc33fc06f1cbf8c8c50c663d7ab058",
262
+ "size_bytes": 1097
263
+ },
264
+ {
265
+ "path": "checkpoints/official/flm/lm1b/config.json",
266
+ "sha256": "460c1308f1da60c319d0cdbb2c9ac40b6b3989031dcd51a6638cc52d6195ca69",
267
+ "size_bytes": 5790
268
+ },
269
+ {
270
+ "path": "checkpoints/official/flm/lm1b/model.safetensors",
271
+ "sha256": "ef02806ef90ab9aca274371c3b644193f0af1757504acdd11ac1c3f89359f55c",
272
+ "size_bytes": 557184584
273
+ }
274
+ ],
275
+ "provenance": "official",
276
+ "quarantined": "/home/ruizzhao/diffusion_baseline/checkpoints/quarantine/20260821T024711Z/official/flm/lm1b",
277
+ "status": "downloaded",
278
+ "teacher_family": "continuous_flm"
279
+ },
280
+ "flm_lm1b_reproductions": {
281
+ "backend": "gdrive",
282
+ "destination": "checkpoints/reference_reproduction/flm_baselines/lm1b",
283
+ "files": [
284
+ {
285
+ "path": "checkpoints/reference_reproduction/flm_baselines/lm1b/lm1b_CANDI.ckpt",
286
+ "sha256": "33b4359227831870e01225673a6cc50b8e38b6523d729fc2d9c5f1eac916f270",
287
+ "size_bytes": 2229786789
288
+ },
289
+ {
290
+ "path": "checkpoints/reference_reproduction/flm_baselines/lm1b/lm1b_Duo.ckpt",
291
+ "sha256": "b11ee9a4cc533435adb9e1149cff74aec0872a9df2a459b4ab58f1867f6cddfb",
292
+ "size_bytes": 2229753893
293
+ },
294
+ {
295
+ "path": "checkpoints/reference_reproduction/flm_baselines/lm1b/lm1b_MDLM_.ckpt",
296
+ "sha256": "d42739e5a99bbd7c0f604dff7fe4c9ab12348b475e30f1d807e70ef32249e514",
297
+ "size_bytes": 2229762853
298
+ }
299
+ ],
300
+ "provenance": "reference_reproduction",
301
+ "quarantined": "/home/ruizzhao/diffusion_baseline/checkpoints/quarantine/20260821T025412Z/reference_reproduction/flm_baselines/lm1b",
302
+ "status": "downloaded",
303
+ "teacher_family": "mixed_baselines"
304
+ },
305
+ "flm_owt_hf": {
306
+ "backend": "huggingface",
307
+ "destination": "checkpoints/official/flm/owt",
308
+ "files": [
309
+ {
310
+ "path": "checkpoints/official/flm/owt/.cache/huggingface/.gitignore",
311
+ "sha256": "684888c0ebb17f374298b65ee2807526c066094c701bcc7ebbe1c1095f494fc1",
312
+ "size_bytes": 1
313
+ },
314
+ {
315
+ "path": "checkpoints/official/flm/owt/.cache/huggingface/download/README.md.lock",
316
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
317
+ "size_bytes": 0
318
+ },
319
+ {
320
+ "path": "checkpoints/official/flm/owt/.cache/huggingface/download/README.md.metadata",
321
+ "sha256": "54b900a6da28e86ac8ac3c5a9c146534ab6e2906c5a11b2c0dd98c08e6d67815",
322
+ "size_bytes": 101
323
+ },
324
+ {
325
+ "path": "checkpoints/official/flm/owt/.cache/huggingface/download/config.json.lock",
326
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
327
+ "size_bytes": 0
328
+ },
329
+ {
330
+ "path": "checkpoints/official/flm/owt/.cache/huggingface/download/config.json.metadata",
331
+ "sha256": "bdaa03cc975849363b3ca4f416c6599b1d011d5cb93be66d972607a091c6dfad",
332
+ "size_bytes": 101
333
+ },
334
+ {
335
+ "path": "checkpoints/official/flm/owt/.cache/huggingface/download/model.safetensors.lock",
336
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
337
+ "size_bytes": 0
338
+ },
339
+ {
340
+ "path": "checkpoints/official/flm/owt/.cache/huggingface/download/model.safetensors.metadata",
341
+ "sha256": "f4fac49a912f2c85f26a60136871d9619fa20779a7222e66b5aea26a46ad6ef3",
342
+ "size_bytes": 125
343
+ },
344
+ {
345
+ "path": "checkpoints/official/flm/owt/README.md",
346
+ "sha256": "ffa98bdad8de633065cd636130027026959457f8dd2b1c1de519c8f1886f39f6",
347
+ "size_bytes": 1124
348
+ },
349
+ {
350
+ "path": "checkpoints/official/flm/owt/config.json",
351
+ "sha256": "a1a2569dbae7d20fb84ab150f0cce8ebf8927067eaad0c2cdd5cb64995115301",
352
+ "size_bytes": 5832
353
+ },
354
+ {
355
+ "path": "checkpoints/official/flm/owt/model.safetensors",
356
+ "sha256": "5715258cb3aac9cc8416607a3af5ca4ce0e86db1ca0e5b67ef038d837d95233d",
357
+ "size_bytes": 678521512
358
+ }
359
+ ],
360
+ "provenance": "official",
361
+ "quarantined": "/home/ruizzhao/diffusion_baseline/checkpoints/quarantine/20260821T024727Z/official/flm/owt",
362
+ "status": "downloaded",
363
+ "teacher_family": "continuous_flm"
364
+ },
365
+ "fmlm_lm1b_hf": {
366
+ "backend": "huggingface",
367
+ "destination": "checkpoints/official/fmlm/lm1b",
368
+ "files": [
369
+ {
370
+ "path": "checkpoints/official/fmlm/lm1b/.cache/huggingface/.gitignore",
371
+ "sha256": "684888c0ebb17f374298b65ee2807526c066094c701bcc7ebbe1c1095f494fc1",
372
+ "size_bytes": 1
373
+ },
374
+ {
375
+ "path": "checkpoints/official/fmlm/lm1b/.cache/huggingface/download/README.md.lock",
376
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
377
+ "size_bytes": 0
378
+ },
379
+ {
380
+ "path": "checkpoints/official/fmlm/lm1b/.cache/huggingface/download/README.md.metadata",
381
+ "sha256": "d0778b3d7ac1b8f4d55a5784af922dc39161fc79745bb66a2d69d76ab8a737d8",
382
+ "size_bytes": 99
383
+ },
384
+ {
385
+ "path": "checkpoints/official/fmlm/lm1b/.cache/huggingface/download/config.json.lock",
386
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
387
+ "size_bytes": 0
388
+ },
389
+ {
390
+ "path": "checkpoints/official/fmlm/lm1b/.cache/huggingface/download/config.json.metadata",
391
+ "sha256": "fbe1d99c32f9ea01d565cbe12bcc9836e178f54c3388993676a0118abd401e44",
392
+ "size_bytes": 101
393
+ },
394
+ {
395
+ "path": "checkpoints/official/fmlm/lm1b/.cache/huggingface/download/model.safetensors.lock",
396
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
397
+ "size_bytes": 0
398
+ },
399
+ {
400
+ "path": "checkpoints/official/fmlm/lm1b/.cache/huggingface/download/model.safetensors.metadata",
401
+ "sha256": "e41d0cb74d4b14da8a68b1ec23a4e266a29a8ec8302d01caf28256386cd1453e",
402
+ "size_bytes": 125
403
+ },
404
+ {
405
+ "path": "checkpoints/official/fmlm/lm1b/README.md",
406
+ "sha256": "2e25f8aea9629395ff0d2598a85d1a3aaa54a961d9226068c8587b71661077e0",
407
+ "size_bytes": 1166
408
+ },
409
+ {
410
+ "path": "checkpoints/official/fmlm/lm1b/config.json",
411
+ "sha256": "6329c0b9a02b1a990d4bda6f108389a292d841b45ae5c66aa7d9b7d0ecb195a2",
412
+ "size_bytes": 6136
413
+ },
414
+ {
415
+ "path": "checkpoints/official/fmlm/lm1b/model.safetensors",
416
+ "sha256": "574e5001d8ee6523b1b363cbd7055a3e3fe6d6f8dd7f796c4582548b9c728942",
417
+ "size_bytes": 557382616
418
+ }
419
+ ],
420
+ "provenance": "official",
421
+ "quarantined": "/home/ruizzhao/diffusion_baseline/checkpoints/quarantine/20260821T024719Z/official/fmlm/lm1b",
422
+ "status": "downloaded",
423
+ "teacher_family": "continuous_flm"
424
+ },
425
+ "fmlm_owt_hf": {
426
+ "backend": "huggingface",
427
+ "destination": "checkpoints/official/fmlm/owt",
428
+ "files": [
429
+ {
430
+ "path": "checkpoints/official/fmlm/owt/.cache/huggingface/.gitignore",
431
+ "sha256": "684888c0ebb17f374298b65ee2807526c066094c701bcc7ebbe1c1095f494fc1",
432
+ "size_bytes": 1
433
+ },
434
+ {
435
+ "path": "checkpoints/official/fmlm/owt/.cache/huggingface/download/README.md.lock",
436
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
437
+ "size_bytes": 0
438
+ },
439
+ {
440
+ "path": "checkpoints/official/fmlm/owt/.cache/huggingface/download/README.md.metadata",
441
+ "sha256": "08ed605c217c2679880e21a8600ac02f60ecdedbeeea55582715355ac036c81d",
442
+ "size_bytes": 101
443
+ },
444
+ {
445
+ "path": "checkpoints/official/fmlm/owt/.cache/huggingface/download/config.json.lock",
446
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
447
+ "size_bytes": 0
448
+ },
449
+ {
450
+ "path": "checkpoints/official/fmlm/owt/.cache/huggingface/download/config.json.metadata",
451
+ "sha256": "6a280eee025ec34f24e065626ec269117f8df0f22d22ef417f1cd906a033cf85",
452
+ "size_bytes": 101
453
+ },
454
+ {
455
+ "path": "checkpoints/official/fmlm/owt/.cache/huggingface/download/model.safetensors.lock",
456
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
457
+ "size_bytes": 0
458
+ },
459
+ {
460
+ "path": "checkpoints/official/fmlm/owt/.cache/huggingface/download/model.safetensors.metadata",
461
+ "sha256": "7043bdddf7e37137f59e03f9d2b8ca665823122c2b1966aa3b67659c3890a576",
462
+ "size_bytes": 125
463
+ },
464
+ {
465
+ "path": "checkpoints/official/fmlm/owt/README.md",
466
+ "sha256": "8fbfb11f02e2ff7a3126ab393346113841704163e01813dcb5ba3706b9d2ab1d",
467
+ "size_bytes": 1191
468
+ },
469
+ {
470
+ "path": "checkpoints/official/fmlm/owt/config.json",
471
+ "sha256": "9a056d5dbfbddfbd5c0554d8189548a9c3b349a0dbd432906e29657a7979299c",
472
+ "size_bytes": 6276
473
+ },
474
+ {
475
+ "path": "checkpoints/official/fmlm/owt/model.safetensors",
476
+ "sha256": "1c44219dc2f240556dd96e4631ec8465fea942dbc5eb85848b204eefc4dac48d",
477
+ "size_bytes": 678719544
478
+ }
479
+ ],
480
+ "provenance": "official",
481
+ "quarantined": "/home/ruizzhao/diffusion_baseline/checkpoints/quarantine/20260821T024736Z/official/fmlm/owt",
482
+ "status": "downloaded",
483
+ "teacher_family": "continuous_flm"
484
+ },
485
+ "langflow_lm1b_hf": {
486
+ "backend": "huggingface",
487
+ "destination": "checkpoints/official/langflow/lm1b",
488
+ "files": [
489
+ {
490
+ "path": "checkpoints/official/langflow/lm1b/.cache/huggingface/.gitignore",
491
+ "sha256": "684888c0ebb17f374298b65ee2807526c066094c701bcc7ebbe1c1095f494fc1",
492
+ "size_bytes": 1
493
+ },
494
+ {
495
+ "path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/README.md.lock",
496
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
497
+ "size_bytes": 0
498
+ },
499
+ {
500
+ "path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/README.md.metadata",
501
+ "sha256": "98cb3fa7f6e30548570815bcf292f0c2d74ef6d2d1ca138ae240bf5fc514139d",
502
+ "size_bytes": 101
503
+ },
504
+ {
505
+ "path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/__init__.py.lock",
506
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
507
+ "size_bytes": 0
508
+ },
509
+ {
510
+ "path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/__init__.py.metadata",
511
+ "sha256": "ea623f715459f45ae0f96b1f6cfc36eb915173f3e93066fa4c445213248e7160",
512
+ "size_bytes": 101
513
+ },
514
+ {
515
+ "path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/config.json.lock",
516
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
517
+ "size_bytes": 0
518
+ },
519
+ {
520
+ "path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/config.json.metadata",
521
+ "sha256": "8a24f9e3f91742fdaf5568973f7acfb3520c5cd281a07e51dcdaca9c2f9d2c5e",
522
+ "size_bytes": 100
523
+ },
524
+ {
525
+ "path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/config.py.lock",
526
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
527
+ "size_bytes": 0
528
+ },
529
+ {
530
+ "path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/config.py.metadata",
531
+ "sha256": "8f1667fb211cbd68adf8afa95fc8075909129e7cf03650101ca81345f2a23fab",
532
+ "size_bytes": 101
533
+ },
534
+ {
535
+ "path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/model.py.lock",
536
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
537
+ "size_bytes": 0
538
+ },
539
+ {
540
+ "path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/model.py.metadata",
541
+ "sha256": "e8bfd60c6deead8621e09b32292fc44e4860a81114fabb56a1d45e0ab7d52f0b",
542
+ "size_bytes": 101
543
+ },
544
+ {
545
+ "path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/model.safetensors.lock",
546
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
547
+ "size_bytes": 0
548
+ },
549
+ {
550
+ "path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/model.safetensors.metadata",
551
+ "sha256": "bf3134715c4958b12a38b83306f18e5abae05c374a83dbf0435c6f24e57eeed4",
552
+ "size_bytes": 125
553
+ },
554
+ {
555
+ "path": "checkpoints/official/langflow/lm1b/README.md",
556
+ "sha256": "8bfdf3f492c7fac94f429c4269886dccad8c8e56b42caab004f93b785baf5d10",
557
+ "size_bytes": 1796
558
+ },
559
+ {
560
+ "path": "checkpoints/official/langflow/lm1b/__init__.py",
561
+ "sha256": "f5f0e406570ddc66a457d3b59d74b50a15eb3444c4543c2ff2d1b5e2277e3852",
562
+ "size_bytes": 154
563
+ },
564
+ {
565
+ "path": "checkpoints/official/langflow/lm1b/config.json",
566
+ "sha256": "d1b7a2324eb7b3791d26ef4c2e6058598d9af9ab3b9aed74f9467bf041594f97",
567
+ "size_bytes": 677
568
+ },
569
+ {
570
+ "path": "checkpoints/official/langflow/lm1b/config.py",
571
+ "sha256": "064d2d83613817a49d2ffc82258e4b3f61c71780eb6448b7d548321db1960998",
572
+ "size_bytes": 2290
573
+ },
574
+ {
575
+ "path": "checkpoints/official/langflow/lm1b/model.py",
576
+ "sha256": "c5617dc383b1c6554333c0d5e87a364829b920eac1e87435501318ba4684ee0e",
577
+ "size_bytes": 20493
578
+ },
579
+ {
580
+ "path": "checkpoints/official/langflow/lm1b/model.safetensors",
581
+ "sha256": "b111b5141379bde1bc8531278a2d144bc753cf3b100ed32c8a917f2ed0c025d2",
582
+ "size_bytes": 561904748
583
+ }
584
+ ],
585
+ "provenance": "official",
586
+ "quarantined": "/home/ruizzhao/diffusion_baseline/checkpoints/quarantine/20260821T024751Z/official/langflow/lm1b",
587
+ "status": "downloaded",
588
+ "teacher_family": "continuous_langflow"
589
+ },
590
+ "langflow_owt_hf": {
591
+ "backend": "huggingface",
592
+ "destination": "checkpoints/official/langflow/owt",
593
+ "files": [
594
+ {
595
+ "path": "checkpoints/official/langflow/owt/.cache/huggingface/.gitignore",
596
+ "sha256": "684888c0ebb17f374298b65ee2807526c066094c701bcc7ebbe1c1095f494fc1",
597
+ "size_bytes": 1
598
+ },
599
+ {
600
+ "path": "checkpoints/official/langflow/owt/.cache/huggingface/download/README.md.lock",
601
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
602
+ "size_bytes": 0
603
+ },
604
+ {
605
+ "path": "checkpoints/official/langflow/owt/.cache/huggingface/download/README.md.metadata",
606
+ "sha256": "82add813061c33eeae7c06e4df882a2e838990c97f1d782bc83924cce7d3feb0",
607
+ "size_bytes": 101
608
+ },
609
+ {
610
+ "path": "checkpoints/official/langflow/owt/.cache/huggingface/download/__init__.py.lock",
611
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
612
+ "size_bytes": 0
613
+ },
614
+ {
615
+ "path": "checkpoints/official/langflow/owt/.cache/huggingface/download/__init__.py.metadata",
616
+ "sha256": "cace4ff5359a8110db78f4f3ca92098aa7635401ce72173fdcac9e0414e2c32c",
617
+ "size_bytes": 101
618
+ },
619
+ {
620
+ "path": "checkpoints/official/langflow/owt/.cache/huggingface/download/config.json.lock",
621
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
622
+ "size_bytes": 0
623
+ },
624
+ {
625
+ "path": "checkpoints/official/langflow/owt/.cache/huggingface/download/config.json.metadata",
626
+ "sha256": "8da03da2a8773b41526eaa30bfb9361044597bab47afe169db4d4e3eb46da869",
627
+ "size_bytes": 100
628
+ },
629
+ {
630
+ "path": "checkpoints/official/langflow/owt/.cache/huggingface/download/config.py.lock",
631
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
632
+ "size_bytes": 0
633
+ },
634
+ {
635
+ "path": "checkpoints/official/langflow/owt/.cache/huggingface/download/config.py.metadata",
636
+ "sha256": "642f21f7f424678df8a1b19c63a96ba48c4d264fde73044e0ca3d5ceff102a04",
637
+ "size_bytes": 100
638
+ },
639
+ {
640
+ "path": "checkpoints/official/langflow/owt/.cache/huggingface/download/model.py.lock",
641
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
642
+ "size_bytes": 0
643
+ },
644
+ {
645
+ "path": "checkpoints/official/langflow/owt/.cache/huggingface/download/model.py.metadata",
646
+ "sha256": "0ce5b367c0d32cd0b8bd3596b09c999f2107f6963d154318b5eac855e0366c93",
647
+ "size_bytes": 99
648
+ },
649
+ {
650
+ "path": "checkpoints/official/langflow/owt/.cache/huggingface/download/model.safetensors.lock",
651
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
652
+ "size_bytes": 0
653
+ },
654
+ {
655
+ "path": "checkpoints/official/langflow/owt/.cache/huggingface/download/model.safetensors.metadata",
656
+ "sha256": "0afc1df20ff0f1b535f5ae708aef109cf1f484261febf6cfa69b7983f0ba6684",
657
+ "size_bytes": 125
658
+ },
659
+ {
660
+ "path": "checkpoints/official/langflow/owt/README.md",
661
+ "sha256": "acdcfb3e3294ff78939f047f53e85795059430b75de267708c1101346aad44d3",
662
+ "size_bytes": 1784
663
+ },
664
+ {
665
+ "path": "checkpoints/official/langflow/owt/__init__.py",
666
+ "sha256": "f5f0e406570ddc66a457d3b59d74b50a15eb3444c4543c2ff2d1b5e2277e3852",
667
+ "size_bytes": 154
668
+ },
669
+ {
670
+ "path": "checkpoints/official/langflow/owt/config.json",
671
+ "sha256": "d16031e863349c8b41606d57f56e5ccde78f81cb1b45bf8e76361a51d146b3a1",
672
+ "size_bytes": 677
673
+ },
674
+ {
675
+ "path": "checkpoints/official/langflow/owt/config.py",
676
+ "sha256": "48f05e9a273aeea0716fe8f24792a2a2f01645fb01677724e9d27eaf6ba3f20b",
677
+ "size_bytes": 2291
678
+ },
679
+ {
680
+ "path": "checkpoints/official/langflow/owt/model.py",
681
+ "sha256": "c5617dc383b1c6554333c0d5e87a364829b920eac1e87435501318ba4684ee0e",
682
+ "size_bytes": 20493
683
+ },
684
+ {
685
+ "path": "checkpoints/official/langflow/owt/model.safetensors",
686
+ "sha256": "9dcc2573297d9f0ad91cb08d3469d141817d7bd3b11a2a7f332ff0f6a58e02a1",
687
+ "size_bytes": 683235528
688
+ }
689
+ ],
690
+ "provenance": "official",
691
+ "quarantined": "/home/ruizzhao/diffusion_baseline/checkpoints/quarantine/20260821T024744Z/official/langflow/owt",
692
+ "status": "downloaded",
693
+ "teacher_family": "continuous_langflow"
694
+ },
695
+ "mdlm_owt_hf": {
696
+ "backend": "huggingface",
697
+ "destination": "checkpoints/official/mdlm/owt",
698
+ "files": [
699
+ {
700
+ "path": "checkpoints/official/mdlm/owt/.cache/huggingface/.gitignore",
701
+ "sha256": "684888c0ebb17f374298b65ee2807526c066094c701bcc7ebbe1c1095f494fc1",
702
+ "size_bytes": 1
703
+ },
704
+ {
705
+ "path": "checkpoints/official/mdlm/owt/.cache/huggingface/download/README.md.lock",
706
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
707
+ "size_bytes": 0
708
+ },
709
+ {
710
+ "path": "checkpoints/official/mdlm/owt/.cache/huggingface/download/README.md.metadata",
711
+ "sha256": "614632afc69855f4b49430a072f71221835e82dc288b652c2c3d9ac73b917bfb",
712
+ "size_bytes": 101
713
+ },
714
+ {
715
+ "path": "checkpoints/official/mdlm/owt/.cache/huggingface/download/config.json.lock",
716
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
717
+ "size_bytes": 0
718
+ },
719
+ {
720
+ "path": "checkpoints/official/mdlm/owt/.cache/huggingface/download/config.json.metadata",
721
+ "sha256": "c9fc5f535cb45b46300731880692d098aad7a28d67ff39a7b8b4a08301cf155b",
722
+ "size_bytes": 101
723
+ },
724
+ {
725
+ "path": "checkpoints/official/mdlm/owt/.cache/huggingface/download/configuration_mdlm.py.lock",
726
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
727
+ "size_bytes": 0
728
+ },
729
+ {
730
+ "path": "checkpoints/official/mdlm/owt/.cache/huggingface/download/configuration_mdlm.py.metadata",
731
+ "sha256": "4bb6bfc099cf5c65a70e3d0091b4587959234e7c2de6ff46e02759977a23e31e",
732
+ "size_bytes": 101
733
+ },
734
+ {
735
+ "path": "checkpoints/official/mdlm/owt/.cache/huggingface/download/model.safetensors.lock",
736
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
737
+ "size_bytes": 0
738
+ },
739
+ {
740
+ "path": "checkpoints/official/mdlm/owt/.cache/huggingface/download/model.safetensors.metadata",
741
+ "sha256": "c34aa69c9146e321c4ad6a4b9313a38de3bf093738e67751038c6debdc3c936d",
742
+ "size_bytes": 125
743
+ },
744
+ {
745
+ "path": "checkpoints/official/mdlm/owt/.cache/huggingface/download/modeling_mdlm.py.lock",
746
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
747
+ "size_bytes": 0
748
+ },
749
+ {
750
+ "path": "checkpoints/official/mdlm/owt/.cache/huggingface/download/modeling_mdlm.py.metadata",
751
+ "sha256": "836910fc926da95ae04c16596318c95145351794cc5dabccd1f963a782c22367",
752
+ "size_bytes": 101
753
+ },
754
+ {
755
+ "path": "checkpoints/official/mdlm/owt/README.md",
756
+ "sha256": "393eb53038b34d16312006a3f0e7baacd3142e2a0514aea8f0d19241264f4ee1",
757
+ "size_bytes": 2431
758
+ },
759
+ {
760
+ "path": "checkpoints/official/mdlm/owt/config.json",
761
+ "sha256": "605429edee54b28322b94c3a018982ddb74bf94e3bf15f4ae7d5fbc73613138c",
762
+ "size_bytes": 489
763
+ },
764
+ {
765
+ "path": "checkpoints/official/mdlm/owt/configuration_mdlm.py",
766
+ "sha256": "0b1fe1d01c0ca375e1b24e06926a78e436b465b12448749cfbaa76318499cec2",
767
+ "size_bytes": 740
768
+ },
769
+ {
770
+ "path": "checkpoints/official/mdlm/owt/model.safetensors",
771
+ "sha256": "47149e73f7552f39ea9776dbe74d925d25237bcf2ed2e2ec03cdff9d51c82aa4",
772
+ "size_bytes": 678522728
773
+ },
774
+ {
775
+ "path": "checkpoints/official/mdlm/owt/modeling_mdlm.py",
776
+ "sha256": "d952a4de135a662abd25a2ce0f43fb67844e4cb38e034d5957e500957aa422a7",
777
+ "size_bytes": 13776
778
+ }
779
+ ],
780
+ "provenance": "official",
781
+ "quarantined": "/home/ruizzhao/diffusion_baseline/checkpoints/quarantine/20260821T024817Z/official/mdlm/owt",
782
+ "status": "downloaded",
783
+ "teacher_family": "masked_mdlm"
784
+ },
785
+ "rdlm_lm1b_drive": {
786
+ "backend": "gdrive",
787
+ "destination": "checkpoints/official/rdlm/lm1b",
788
+ "files": [
789
+ {
790
+ "path": "checkpoints/official/rdlm/lm1b/LM1B/checkpoint.pth",
791
+ "sha256": "3319acacbd41ac4bf6bf42840dfd22cbebcd8e45d975d5225f114b6ee9cafd81",
792
+ "size_bytes": 1724097770
793
+ },
794
+ {
795
+ "path": "checkpoints/official/rdlm/lm1b/LM1B/config.yaml",
796
+ "sha256": "3acb20fdbaa5143e66a1e722e17fce384c557efac1e24cbcee78dc524183f5f3",
797
+ "size_bytes": 1241
798
+ },
799
+ {
800
+ "path": "checkpoints/official/rdlm/lm1b/LM1B/sde.pkl",
801
+ "sha256": "29ef378b58764f1d5f2da156a2060b57098d4dd6cb299c6c5d9a8d488152bbb0",
802
+ "size_bytes": 160713
803
+ },
804
+ {
805
+ "path": "checkpoints/official/rdlm/lm1b/Text8/checkpoint.pth",
806
+ "sha256": "4f10e0dfc81d83ff3ec64c73b1bc8cf8fa60a34c4ea174e3194d7187df759b69",
807
+ "size_bytes": 1478972002
808
+ },
809
+ {
810
+ "path": "checkpoints/official/rdlm/lm1b/Text8/config.yaml",
811
+ "sha256": "f1caa7de074033a2a4a41329f5db8b10dbea6bd12dd6984c1a01650316df308a",
812
+ "size_bytes": 1206
813
+ },
814
+ {
815
+ "path": "checkpoints/official/rdlm/lm1b/Text8/sde.pkl",
816
+ "sha256": "5fec68cb5e4aec10a2a0148baa52dd60e48ebd59adb088a9190fd8d4ae1dd720",
817
+ "size_bytes": 160713
818
+ }
819
+ ],
820
+ "provenance": "official",
821
+ "quarantined": "/home/ruizzhao/diffusion_baseline/checkpoints/quarantine/20260821T025912Z/official/rdlm/lm1b",
822
+ "status": "downloaded",
823
+ "teacher_family": "continuous_rdlm"
824
+ },
825
+ "sdtt_owt_hf": {
826
+ "backend": "huggingface",
827
+ "destination": "checkpoints/official/mdlm_sdtt/owt",
828
+ "files": [
829
+ {
830
+ "path": "checkpoints/official/mdlm_sdtt/owt/.cache/huggingface/.gitignore",
831
+ "sha256": "684888c0ebb17f374298b65ee2807526c066094c701bcc7ebbe1c1095f494fc1",
832
+ "size_bytes": 1
833
+ },
834
+ {
835
+ "path": "checkpoints/official/mdlm_sdtt/owt/.cache/huggingface/download/config.json.lock",
836
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
837
+ "size_bytes": 0
838
+ },
839
+ {
840
+ "path": "checkpoints/official/mdlm_sdtt/owt/.cache/huggingface/download/config.json.metadata",
841
+ "sha256": "822a31ab007b6a715e3d5191edca2955716e7c078e0b02440c75c2722535142b",
842
+ "size_bytes": 101
843
+ },
844
+ {
845
+ "path": "checkpoints/official/mdlm_sdtt/owt/.cache/huggingface/download/model.safetensors.lock",
846
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
847
+ "size_bytes": 0
848
+ },
849
+ {
850
+ "path": "checkpoints/official/mdlm_sdtt/owt/.cache/huggingface/download/model.safetensors.metadata",
851
+ "sha256": "5d0fbf99fbf74a54acc6c254eaab131c398723c81bd2e7ae3c02146d5b7c4038",
852
+ "size_bytes": 125
853
+ },
854
+ {
855
+ "path": "checkpoints/official/mdlm_sdtt/owt/config.json",
856
+ "sha256": "8a62a25b9f44b2476d561becd36d60febda7d7d4217d81db9a6987c29ffb21ae",
857
+ "size_bytes": 5402
858
+ },
859
+ {
860
+ "path": "checkpoints/official/mdlm_sdtt/owt/model.safetensors",
861
+ "sha256": "33338a0cebd25993fc455447248563d00fe3414b66ce29ca0d7f3529f4d6218d",
862
+ "size_bytes": 678522696
863
+ }
864
+ ],
865
+ "provenance": "official",
866
+ "quarantined": "/home/ruizzhao/diffusion_baseline/checkpoints/quarantine/20260821T024825Z/official/mdlm_sdtt/owt",
867
+ "status": "downloaded",
868
+ "teacher_family": "masked_mdlm"
869
+ }
870
+ },
871
+ "schema_version": 1
872
+ }
checkpoints/reference_reproduction/candi/owt/lightning_logs/version_2/metrics.csv ADDED
The diff for this file is too large to render. See raw diff
 
checkpoints/reference_reproduction/duo_di4c/owt.before-teacher-fix/stdout.log ADDED
The diff for this file is too large to render. See raw diff
 
data/manifests/conditional-lm1b-c64.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": "lm1b",
3
+ "evaluation_continuation_length": 64,
4
+ "model_length": 128,
5
+ "prefix_length": 64,
6
+ "prompt_count": 1024,
7
+ "prompt_file": "data/conditional/lm1b-c64/prompts.jsonl",
8
+ "prompt_file_sha256": "b83d6fc574efda562fce68a23bffe106b238d1739a656dade72fda4ab18bc422",
9
+ "protocol": "c64_zs_v1",
10
+ "schema_version": 1,
11
+ "selection_algorithm": "sha256_seed_index_digest_ascending_v1",
12
+ "selection_seed": 42,
13
+ "source_manifest_path": "data/manifests/lm1b.json",
14
+ "source_manifest_sha256": "a8465f3fba8c5dc529fc0751039c8ffa32a7710f480208779b55545cb141017c",
15
+ "source_processed_path": "data/processed/lm1b-bert-128/validation",
16
+ "source_row_count": 71979,
17
+ "source_split": "validation",
18
+ "tokenizer_id": "bert-base-uncased",
19
+ "tokenizer_revision": "86b5e0934494bd15c9632b12f734a8a67f723594",
20
+ "vocabulary_size": 30522
21
+ }
data/manifests/lm1b.json ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "created_at": "2026-08-21T02:44:58.134500Z",
3
+ "dataset": "lm1b",
4
+ "dataset_id": "billion-word-benchmark/lm1b",
5
+ "document_counts": {
6
+ "train": 30301028,
7
+ "validation": 306688
8
+ },
9
+ "files": [
10
+ {
11
+ "path": "dataset_dict.json",
12
+ "sha256": "88b5b05fc757e76e7edc779db90e96cb6d76538b93165d6ce962a818801babf4",
13
+ "size_bytes": 35
14
+ },
15
+ {
16
+ "path": "train/data-00000-of-00004.arrow",
17
+ "sha256": "c1842d29417604f3c3ee4ba6fd648adebf47ef3d33588ca7eacad5f7e60dc276",
18
+ "size_bytes": 454895728
19
+ },
20
+ {
21
+ "path": "train/data-00001-of-00004.arrow",
22
+ "sha256": "395ff7ef22b9f42ed72ab4d4f319d1a5da1756ae85d5bd498c3b5b4dfc96273e",
23
+ "size_bytes": 454895728
24
+ },
25
+ {
26
+ "path": "train/data-00002-of-00004.arrow",
27
+ "sha256": "1d8ed17f183a09947a7035b6a0571662d0797e6b236897e5b28e378c7ea8a3c0",
28
+ "size_bytes": 454895472
29
+ },
30
+ {
31
+ "path": "train/data-00003-of-00004.arrow",
32
+ "sha256": "2272f8e8d83a5189d0a2d39e0010949f2b7914aae4f9e5cb2316f392fc699e56",
33
+ "size_bytes": 454895472
34
+ },
35
+ {
36
+ "path": "train/dataset_info.json",
37
+ "sha256": "1f4e3b935de233b63542c8d29a7bc819e11c08522ca8e83d3f3e121e3616e081",
38
+ "size_bytes": 805
39
+ },
40
+ {
41
+ "path": "train/state.json",
42
+ "sha256": "d9f4fca87f6e722f0dddc1f7ae811d0552830540d0690666d04735330882a410",
43
+ "size_bytes": 427
44
+ },
45
+ {
46
+ "path": "validation/data-00000-of-00001.arrow",
47
+ "sha256": "fff03f2a6785bc54a9d54074a409d89bc2e71042beaf3a4ccef724a9c74c14d6",
48
+ "size_bytes": 18439664
49
+ },
50
+ {
51
+ "path": "validation/dataset_info.json",
52
+ "sha256": "5d94f384698efbc6ef827c2dd2e776a6a9ac89be2f92dcf9490da6a6063949e1",
53
+ "size_bytes": 696
54
+ },
55
+ {
56
+ "path": "validation/state.json",
57
+ "sha256": "ba4addb0d152b776d1a256db649e50707e16d2e932839cb9e547de20101a0232",
58
+ "size_bytes": 250
59
+ }
60
+ ],
61
+ "packed_sequence_counts": {
62
+ "train": 7102854,
63
+ "validation": 71979
64
+ },
65
+ "processed_path": "data/processed/lm1b-bert-128",
66
+ "processing_contract": {
67
+ "dataset": "lm1b",
68
+ "dataset_id": "billion-word-benchmark/lm1b",
69
+ "expected_document_counts": {
70
+ "train": 30301028,
71
+ "validation": 306688
72
+ },
73
+ "packing": {
74
+ "algorithm": "document_eos_continuous_bos_eos",
75
+ "version": 1
76
+ },
77
+ "schema_version": 1,
78
+ "sequence_length": 128,
79
+ "source_materialization_revision": "8d52bfd3cc2819fc1166dbb8144c328e2690de3e",
80
+ "source_revision": "35161838ea9e05371a25a8db037f94fcae4c2064",
81
+ "split_expression": {
82
+ "train": "train",
83
+ "validation": "test"
84
+ },
85
+ "tokenizer_id": "bert-base-uncased",
86
+ "tokenizer_revision": "86b5e0934494bd15c9632b12f734a8a67f723594",
87
+ "vocab_size": 30522
88
+ },
89
+ "processing_contract_sha256": "e64a9a0df35ae112075596ccd5958a363a09075fb61515c058f0fc3e2eb50668",
90
+ "schema_version": 1,
91
+ "sequence_length": 128,
92
+ "source_materialization_revision": "8d52bfd3cc2819fc1166dbb8144c328e2690de3e",
93
+ "source_revision": "35161838ea9e05371a25a8db037f94fcae4c2064",
94
+ "split_expression": {
95
+ "train": "train",
96
+ "validation": "test"
97
+ },
98
+ "tokenizer_id": "bert-base-uncased",
99
+ "tokenizer_revision": "86b5e0934494bd15c9632b12f734a8a67f723594",
100
+ "verification": {
101
+ "checked_sequences": 7174833,
102
+ "decoded_indices": {
103
+ "train": [
104
+ 3036107,
105
+ 3084783,
106
+ 4793642
107
+ ],
108
+ "validation": [
109
+ 4009,
110
+ 19943,
111
+ 36031
112
+ ]
113
+ },
114
+ "decoded_samples": 6,
115
+ "files_checked": 10,
116
+ "observed_max_token_id": 30511,
117
+ "observed_min_token_id": 100,
118
+ "verified_at": "2026-08-21T02:46:51.032489Z"
119
+ },
120
+ "verified": true,
121
+ "vocabulary": {
122
+ "max_token_id": 30511,
123
+ "min_token_id": 100,
124
+ "size": 30522
125
+ }
126
+ }
data/manifests/owt.json ADDED
@@ -0,0 +1,286 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "created_at": "2026-08-21T02:46:38.080456Z",
3
+ "dataset": "owt",
4
+ "dataset_id": "Skylion007/openwebtext",
5
+ "document_counts": {
6
+ "train": 7913769,
7
+ "validation": 100000
8
+ },
9
+ "files": [
10
+ {
11
+ "path": "dataset_dict.json",
12
+ "sha256": "88b5b05fc757e76e7edc779db90e96cb6d76538b93165d6ce962a818801babf4",
13
+ "size_bytes": 35
14
+ },
15
+ {
16
+ "path": "train/data-00000-of-00036.arrow",
17
+ "sha256": "b72f7336f1853a6c457b3577de05bcfdab64e2657618cf5cc2dbdc5c16f5e1da",
18
+ "size_bytes": 496963712
19
+ },
20
+ {
21
+ "path": "train/data-00001-of-00036.arrow",
22
+ "sha256": "fb112eed5a7d1434d37625f69579038fc6e3d8bd6dd0cfe34ec3b83cd4ce1436",
23
+ "size_bytes": 496961664
24
+ },
25
+ {
26
+ "path": "train/data-00002-of-00036.arrow",
27
+ "sha256": "9979a451c3ea761665a9c705fee27e108fa7c0644f9778c7f9df959cc50b9d0c",
28
+ "size_bytes": 496961664
29
+ },
30
+ {
31
+ "path": "train/data-00003-of-00036.arrow",
32
+ "sha256": "e13682f9f96f961204ddd5d2cc03daf5eddbc80700fc154d0268e55008d0733b",
33
+ "size_bytes": 496961664
34
+ },
35
+ {
36
+ "path": "train/data-00004-of-00036.arrow",
37
+ "sha256": "6c4e6549059babb015bbceaa6ba23b9e606df89daaa87bcc1dba9ec55334fc83",
38
+ "size_bytes": 496961664
39
+ },
40
+ {
41
+ "path": "train/data-00005-of-00036.arrow",
42
+ "sha256": "bdc2b617bf418c871105f1d6ad248aa7828b51f8a3c47676a7e63d2a33a12895",
43
+ "size_bytes": 496961664
44
+ },
45
+ {
46
+ "path": "train/data-00006-of-00036.arrow",
47
+ "sha256": "a4049c39ed5ec67550805893c3b8a19ce3954cd6140680b0518b24db2aaba9aa",
48
+ "size_bytes": 496961664
49
+ },
50
+ {
51
+ "path": "train/data-00007-of-00036.arrow",
52
+ "sha256": "611562e240dda9bf31f596fa34e6e44fe3edb3c52011514ad3fb8b647dcc010f",
53
+ "size_bytes": 496961664
54
+ },
55
+ {
56
+ "path": "train/data-00008-of-00036.arrow",
57
+ "sha256": "ac412a4af6799d5fff761360217e176f6ca1778082bcffe68f7560d44ec3118b",
58
+ "size_bytes": 496961664
59
+ },
60
+ {
61
+ "path": "train/data-00009-of-00036.arrow",
62
+ "sha256": "1335e59d8c37be55c5509a1323d4ad31e18a9a5ce7199cd6638f0888408f102e",
63
+ "size_bytes": 496961664
64
+ },
65
+ {
66
+ "path": "train/data-00010-of-00036.arrow",
67
+ "sha256": "883f016e7ede7a8c54c65d88fc1cab6e5eb1bf5e1bbc230ff24dc8e6407b8a02",
68
+ "size_bytes": 496961664
69
+ },
70
+ {
71
+ "path": "train/data-00011-of-00036.arrow",
72
+ "sha256": "4af8234ed01bdfb2852945d765118e074e544233ed2a487be51701f1f4fd6851",
73
+ "size_bytes": 496961664
74
+ },
75
+ {
76
+ "path": "train/data-00012-of-00036.arrow",
77
+ "sha256": "8ca1d78ce79b3dbc5a3c7106753d0af6809631542798234754d53fdcea7e5d40",
78
+ "size_bytes": 496961664
79
+ },
80
+ {
81
+ "path": "train/data-00013-of-00036.arrow",
82
+ "sha256": "124d2f2d567251e6cfc30280c19e45542402fd437b7f27b3f6434926b2ead2c8",
83
+ "size_bytes": 496961664
84
+ },
85
+ {
86
+ "path": "train/data-00014-of-00036.arrow",
87
+ "sha256": "083f499c788443150dbc2b7e92dad4b3781aa7f2066a736acbca43d16d714518",
88
+ "size_bytes": 496961664
89
+ },
90
+ {
91
+ "path": "train/data-00015-of-00036.arrow",
92
+ "sha256": "1e3fe5de327b1369c171d1e03f9f2a60cc99881f8b026710f7ef0b759ee75b9f",
93
+ "size_bytes": 496961664
94
+ },
95
+ {
96
+ "path": "train/data-00016-of-00036.arrow",
97
+ "sha256": "37fdfc1b324b8952ccb17552da7582f6c67dca5db19a03edc410ec692cca2086",
98
+ "size_bytes": 496961664
99
+ },
100
+ {
101
+ "path": "train/data-00017-of-00036.arrow",
102
+ "sha256": "3b117094ee89d8812b7513b7fb35fa468ffcfe8ded4f231f2b98024e8fe39e93",
103
+ "size_bytes": 496961664
104
+ },
105
+ {
106
+ "path": "train/data-00018-of-00036.arrow",
107
+ "sha256": "cbcbb6d4352e7410b428e93b60a1d8462a66e9ac1aacc456e37d4261a015ae5a",
108
+ "size_bytes": 496961664
109
+ },
110
+ {
111
+ "path": "train/data-00019-of-00036.arrow",
112
+ "sha256": "4b3a9276caa65ec5ce955fdaf42151b8ba626887281d69d54c1750baae4fddeb",
113
+ "size_bytes": 496961664
114
+ },
115
+ {
116
+ "path": "train/data-00020-of-00036.arrow",
117
+ "sha256": "42f595653d508ca2a6c516b0e93d2ed85fffd9cb2130152e6621a5b79d44a8e4",
118
+ "size_bytes": 496961664
119
+ },
120
+ {
121
+ "path": "train/data-00021-of-00036.arrow",
122
+ "sha256": "63cf9f35df6a254e3981745d6b5d6f76a6aeffa1ffe41eb100812329fac4a53e",
123
+ "size_bytes": 496961664
124
+ },
125
+ {
126
+ "path": "train/data-00022-of-00036.arrow",
127
+ "sha256": "34811ff54d62eaad627f21deb99321017ef4a3d9a70a183fa96a55992bb37d8d",
128
+ "size_bytes": 496961664
129
+ },
130
+ {
131
+ "path": "train/data-00023-of-00036.arrow",
132
+ "sha256": "a59867054f81fc70da375ee2ca112cf69e655ab6da0d14ec2c280c31fcd1dc9b",
133
+ "size_bytes": 496961664
134
+ },
135
+ {
136
+ "path": "train/data-00024-of-00036.arrow",
137
+ "sha256": "a064666a45f37b0a4b6ce9042ee6741f5f122db1f713c6634d9707947ed1cfc0",
138
+ "size_bytes": 496961664
139
+ },
140
+ {
141
+ "path": "train/data-00025-of-00036.arrow",
142
+ "sha256": "7f8e5868f4c9e0aac16d5a5dec1dcef19e307550038e0162614deb00eca73e6a",
143
+ "size_bytes": 496961664
144
+ },
145
+ {
146
+ "path": "train/data-00026-of-00036.arrow",
147
+ "sha256": "7904f9ab98ef7534ddf4f9c2c24b9ef1802f1e79e0567e309740ea11cedac6f1",
148
+ "size_bytes": 496961664
149
+ },
150
+ {
151
+ "path": "train/data-00027-of-00036.arrow",
152
+ "sha256": "5434b60b31c999b94676a72875ae4424f79f094db8f06608408915638ac21758",
153
+ "size_bytes": 496961664
154
+ },
155
+ {
156
+ "path": "train/data-00028-of-00036.arrow",
157
+ "sha256": "d14ed5e57ab4dde321695484b389ec6768bf352212f39e570b93524d3f2ef86b",
158
+ "size_bytes": 496961664
159
+ },
160
+ {
161
+ "path": "train/data-00029-of-00036.arrow",
162
+ "sha256": "4bc3f16f159189883d51e97b83640eedc723e8e6475a3361595b825598e19da4",
163
+ "size_bytes": 496961664
164
+ },
165
+ {
166
+ "path": "train/data-00030-of-00036.arrow",
167
+ "sha256": "1b2916afb7c4fb7730a5b86d3b37abee29b8d1f7a632c73552e111c88289ac85",
168
+ "size_bytes": 496961664
169
+ },
170
+ {
171
+ "path": "train/data-00031-of-00036.arrow",
172
+ "sha256": "7c97e4afff654e5fc7a078b6199dd766236b88513c3111baaacb1cb23f1ccdfa",
173
+ "size_bytes": 496961664
174
+ },
175
+ {
176
+ "path": "train/data-00032-of-00036.arrow",
177
+ "sha256": "a2521d39a2fbb359a01b87b20fe25ab3869e757627699c586c2c64d8c475f9d8",
178
+ "size_bytes": 496961664
179
+ },
180
+ {
181
+ "path": "train/data-00033-of-00036.arrow",
182
+ "sha256": "1c45c498a626dc0d23e20ade6f67387700eed878b3aaa3ee2c88632496ea34ff",
183
+ "size_bytes": 496961664
184
+ },
185
+ {
186
+ "path": "train/data-00034-of-00036.arrow",
187
+ "sha256": "b855312afd765fce956c020b4ca415c520ed58915ce77ff0f74134707691d55b",
188
+ "size_bytes": 496961664
189
+ },
190
+ {
191
+ "path": "train/data-00035-of-00036.arrow",
192
+ "sha256": "03a137f7c047131f1d9d7c7cce9c821addaae68e0cee53ced766e1c3a76b21c3",
193
+ "size_bytes": 496961664
194
+ },
195
+ {
196
+ "path": "train/dataset_info.json",
197
+ "sha256": "9b40e0362d5f3ee3169330c88838113389b033e3605d8b83cc623711ee3a6074",
198
+ "size_bytes": 1317
199
+ },
200
+ {
201
+ "path": "train/state.json",
202
+ "sha256": "23949df2c280b60fefd4eed1d7f39411d898af6aa22d4f18d09efd0825a4833b",
203
+ "size_bytes": 2315
204
+ },
205
+ {
206
+ "path": "validation/data-00000-of-00001.arrow",
207
+ "sha256": "937a1c91c25bfddef7908b92c5d0b6283ce2dc90194810a16ef54bca100c4d85",
208
+ "size_bytes": 226364864
209
+ },
210
+ {
211
+ "path": "validation/dataset_info.json",
212
+ "sha256": "87846ab989aa61f9ed6340829fd699098abc35f75d14c76d3767462be20221a6",
213
+ "size_bytes": 701
214
+ },
215
+ {
216
+ "path": "validation/state.json",
217
+ "sha256": "14558facc5f5bd1709161e1588a89b0486d259831fb92e924d7e55699b1bc8a5",
218
+ "size_bytes": 250
219
+ }
220
+ ],
221
+ "packed_sequence_counts": {
222
+ "train": 8734897,
223
+ "validation": 110520
224
+ },
225
+ "processed_path": "data/processed/owt-gpt2-1024",
226
+ "processing_contract": {
227
+ "dataset": "owt",
228
+ "dataset_id": "Skylion007/openwebtext",
229
+ "expected_document_counts": {
230
+ "train": 7913769,
231
+ "validation": 100000
232
+ },
233
+ "packing": {
234
+ "algorithm": "document_eos_continuous_bos_eos",
235
+ "version": 1
236
+ },
237
+ "schema_version": 1,
238
+ "sequence_length": 1024,
239
+ "source_materialization_revision": null,
240
+ "source_revision": "79d93d786212f7344586290adb811d4ae6a1762c",
241
+ "split_expression": {
242
+ "train": "train[:-100000]",
243
+ "validation": "train[-100000:]"
244
+ },
245
+ "tokenizer_id": "gpt2",
246
+ "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
247
+ "vocab_size": 50257
248
+ },
249
+ "processing_contract_sha256": "ec742365e5064afa53d23d8215e729366c67cf2e473b88d3f23f4880a52bae71",
250
+ "schema_version": 1,
251
+ "sequence_length": 1024,
252
+ "source_materialization_revision": null,
253
+ "source_revision": "79d93d786212f7344586290adb811d4ae6a1762c",
254
+ "split_expression": {
255
+ "train": "train[:-100000]",
256
+ "validation": "train[-100000:]"
257
+ },
258
+ "tokenizer_id": "gpt2",
259
+ "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
260
+ "verification": {
261
+ "checked_sequences": 8845417,
262
+ "decoded_indices": {
263
+ "train": [
264
+ 1758481,
265
+ 6072214,
266
+ 6169566
267
+ ],
268
+ "validation": [
269
+ 4009,
270
+ 19943,
271
+ 74884
272
+ ]
273
+ },
274
+ "decoded_samples": 6,
275
+ "files_checked": 42,
276
+ "observed_max_token_id": 50256,
277
+ "observed_min_token_id": 0,
278
+ "verified_at": "2026-08-21T02:47:01.956315Z"
279
+ },
280
+ "verified": true,
281
+ "vocabulary": {
282
+ "max_token_id": 50256,
283
+ "min_token_id": 0,
284
+ "size": 50257
285
+ }
286
+ }
data/processed/.builder-cache/lm1b/train/generator/default-3a58fdf789a93b0f/0.0.0.incomplete_info.lock ADDED
File without changes
data/processed/.builder-cache/lm1b/train/generator/default-3a58fdf789a93b0f/0.0.0/dataset_info.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"description": "", "citation": "", "homepage": "", "license": "", "features": {"input_ids": {"feature": {"dtype": "uint16", "_type": "Value"}, "length": 128, "_type": "Sequence"}}, "builder_name": "generator", "dataset_name": "generator", "config_name": "default", "version": {"version_str": "0.0.0", "major": 0, "minor": 0, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 1818330624, "num_examples": 7102854, "shard_lengths": [1954000, 1954000, 1954000, 1240854], "dataset_name": "generator"}}, "download_checksums": {}, "download_size": 0, "dataset_size": 1818330624, "size_in_bytes": 1818330624}
data/processed/.builder-cache/lm1b/train/generator/default-3a58fdf789a93b0f/0.0.0_builder.lock ADDED
File without changes
data/processed/.builder-cache/lm1b/validation/_home_ruizzhao_diffusion_baseline_data_processed_.builder-cache_lm1b_validation_generator_default-62c652dc08a03c0b_0.0.0.lock ADDED
File without changes
data/processed/.builder-cache/lm1b/validation/generator/default-49fddb9f1246d928/0.0.0.incomplete_info.lock ADDED
File without changes
data/processed/.builder-cache/lm1b/validation/generator/default-49fddb9f1246d928/0.0.0/dataset_info.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"description": "", "citation": "", "homepage": "", "license": "", "features": {"input_ids": {"feature": {"dtype": "uint16", "_type": "Value"}, "length": 128, "_type": "Sequence"}}, "builder_name": "generator", "dataset_name": "generator", "config_name": "default", "version": {"version_str": "0.0.0", "major": 0, "minor": 0, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 18426624, "num_examples": 71979, "dataset_name": "generator"}}, "download_checksums": {}, "download_size": 0, "dataset_size": 18426624, "size_in_bytes": 18426624}
data/processed/.builder-cache/lm1b/validation/generator/default-49fddb9f1246d928/0.0.0_builder.lock ADDED
File without changes
data/processed/.builder-cache/lm1b/validation/generator/default-62c652dc08a03c0b/0.0.0/dataset_info.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"description": "", "citation": "", "homepage": "", "license": "", "features": {"input_ids": {"feature": {"dtype": "uint16", "_type": "Value"}, "length": 128, "_type": "Sequence"}}, "builder_name": "generator", "dataset_name": "generator", "config_name": "default", "version": {"version_str": "0.0.0", "major": 0, "minor": 0, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 18426624, "num_examples": 71979, "dataset_name": "generator"}}, "download_checksums": {}, "download_size": 0, "dataset_size": 18426624, "size_in_bytes": 18426624}
data/processed/.builder-cache/lm1b/validation/generator/default-62c652dc08a03c0b/0.0.0_builder.lock ADDED
File without changes
data/processed/.builder-cache/owt/train/generator/default-b8b56016fa13f36d/0.0.0/dataset_info.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"description": "", "citation": "", "homepage": "", "license": "", "features": {"input_ids": {"feature": {"dtype": "uint16", "_type": "Value"}, "length": 1024, "_type": "Sequence"}}, "builder_name": "generator", "dataset_name": "generator", "config_name": "default", "version": {"version_str": "0.0.0", "major": 0, "minor": 0, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 17889069056, "num_examples": 8734897, "shard_lengths": [245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 245000, 159897], "dataset_name": "generator"}}, "download_checksums": {}, "download_size": 0, "dataset_size": 17889069056, "size_in_bytes": 17889069056}
data/processed/.builder-cache/owt/train/generator/default-b8b56016fa13f36d/0.0.0_builder.lock ADDED
File without changes
data/processed/.builder-cache/owt/validation/_home_ruizzhao_diffusion_baseline_data_processed_.builder-cache_owt_validation_generator_default-bec011984d977746_0.0.0.lock ADDED
File without changes
data/processed/.builder-cache/owt/validation/generator/default-0c41487195ccfdac/0.0.0/dataset_info.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"description": "", "citation": "", "homepage": "", "license": "", "features": {"input_ids": {"feature": {"dtype": "uint16", "_type": "Value"}, "length": 1024, "_type": "Sequence"}}, "builder_name": "generator", "dataset_name": "generator", "config_name": "default", "version": {"version_str": "0.0.0", "major": 0, "minor": 0, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 226344960, "num_examples": 110520, "dataset_name": "generator"}}, "download_checksums": {}, "download_size": 0, "dataset_size": 226344960, "size_in_bytes": 226344960}
data/processed/lm1b-bert-128/dataset_dict.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"splits": ["train", "validation"]}
data/processed/lm1b-bert-128/train/dataset_info.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "builder_name": "generator",
3
+ "citation": "",
4
+ "config_name": "default",
5
+ "dataset_name": "generator",
6
+ "dataset_size": 1818330624,
7
+ "description": "",
8
+ "download_checksums": {},
9
+ "download_size": 0,
10
+ "features": {
11
+ "input_ids": {
12
+ "feature": {
13
+ "dtype": "uint16",
14
+ "_type": "Value"
15
+ },
16
+ "length": 128,
17
+ "_type": "Sequence"
18
+ }
19
+ },
20
+ "homepage": "",
21
+ "license": "",
22
+ "size_in_bytes": 1818330624,
23
+ "splits": {
24
+ "train": {
25
+ "name": "train",
26
+ "num_bytes": 1818330624,
27
+ "num_examples": 7102854,
28
+ "shard_lengths": [
29
+ 1954000,
30
+ 1954000,
31
+ 1954000,
32
+ 1240854
33
+ ],
34
+ "dataset_name": "generator"
35
+ }
36
+ },
37
+ "version": {
38
+ "version_str": "0.0.0",
39
+ "major": 0,
40
+ "minor": 0,
41
+ "patch": 0
42
+ }
43
+ }
data/processed/lm1b-bert-128/train/state.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_data_files": [
3
+ {
4
+ "filename": "data-00000-of-00004.arrow"
5
+ },
6
+ {
7
+ "filename": "data-00001-of-00004.arrow"
8
+ },
9
+ {
10
+ "filename": "data-00002-of-00004.arrow"
11
+ },
12
+ {
13
+ "filename": "data-00003-of-00004.arrow"
14
+ }
15
+ ],
16
+ "_fingerprint": "e2d9bbbd3eaddad9",
17
+ "_format_columns": null,
18
+ "_format_kwargs": {},
19
+ "_format_type": null,
20
+ "_output_all_columns": false,
21
+ "_split": "train"
22
+ }
data/processed/lm1b-bert-128/validation/dataset_info.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "builder_name": "generator",
3
+ "citation": "",
4
+ "config_name": "default",
5
+ "dataset_name": "generator",
6
+ "dataset_size": 18426624,
7
+ "description": "",
8
+ "download_checksums": {},
9
+ "download_size": 0,
10
+ "features": {
11
+ "input_ids": {
12
+ "feature": {
13
+ "dtype": "uint16",
14
+ "_type": "Value"
15
+ },
16
+ "length": 128,
17
+ "_type": "Sequence"
18
+ }
19
+ },
20
+ "homepage": "",
21
+ "license": "",
22
+ "size_in_bytes": 18426624,
23
+ "splits": {
24
+ "train": {
25
+ "name": "train",
26
+ "num_bytes": 18426624,
27
+ "num_examples": 71979,
28
+ "dataset_name": "generator"
29
+ }
30
+ },
31
+ "version": {
32
+ "version_str": "0.0.0",
33
+ "major": 0,
34
+ "minor": 0,
35
+ "patch": 0
36
+ }
37
+ }
data/processed/lm1b-bert-128/validation/state.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_data_files": [
3
+ {
4
+ "filename": "data-00000-of-00001.arrow"
5
+ }
6
+ ],
7
+ "_fingerprint": "747a80c1cdd8587a",
8
+ "_format_columns": null,
9
+ "_format_kwargs": {},
10
+ "_format_type": null,
11
+ "_output_all_columns": false,
12
+ "_split": "train"
13
+ }
data/processed/owt-gpt2-1024/train/dataset_info.json ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "builder_name": "generator",
3
+ "citation": "",
4
+ "config_name": "default",
5
+ "dataset_name": "generator",
6
+ "dataset_size": 17889069056,
7
+ "description": "",
8
+ "download_checksums": {},
9
+ "download_size": 0,
10
+ "features": {
11
+ "input_ids": {
12
+ "feature": {
13
+ "dtype": "uint16",
14
+ "_type": "Value"
15
+ },
16
+ "length": 1024,
17
+ "_type": "Sequence"
18
+ }
19
+ },
20
+ "homepage": "",
21
+ "license": "",
22
+ "size_in_bytes": 17889069056,
23
+ "splits": {
24
+ "train": {
25
+ "name": "train",
26
+ "num_bytes": 17889069056,
27
+ "num_examples": 8734897,
28
+ "shard_lengths": [
29
+ 245000,
30
+ 245000,
31
+ 245000,
32
+ 245000,
33
+ 245000,
34
+ 245000,
35
+ 245000,
36
+ 245000,
37
+ 245000,
38
+ 245000,
39
+ 245000,
40
+ 245000,
41
+ 245000,
42
+ 245000,
43
+ 245000,
44
+ 245000,
45
+ 245000,
46
+ 245000,
47
+ 245000,
48
+ 245000,
49
+ 245000,
50
+ 245000,
51
+ 245000,
52
+ 245000,
53
+ 245000,
54
+ 245000,
55
+ 245000,
56
+ 245000,
57
+ 245000,
58
+ 245000,
59
+ 245000,
60
+ 245000,
61
+ 245000,
62
+ 245000,
63
+ 245000,
64
+ 159897
65
+ ],
66
+ "dataset_name": "generator"
67
+ }
68
+ },
69
+ "version": {
70
+ "version_str": "0.0.0",
71
+ "major": 0,
72
+ "minor": 0,
73
+ "patch": 0
74
+ }
75
+ }
data/processed/owt-gpt2-1024/train/state.json ADDED
@@ -0,0 +1,118 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_data_files": [
3
+ {
4
+ "filename": "data-00000-of-00036.arrow"
5
+ },
6
+ {
7
+ "filename": "data-00001-of-00036.arrow"
8
+ },
9
+ {
10
+ "filename": "data-00002-of-00036.arrow"
11
+ },
12
+ {
13
+ "filename": "data-00003-of-00036.arrow"
14
+ },
15
+ {
16
+ "filename": "data-00004-of-00036.arrow"
17
+ },
18
+ {
19
+ "filename": "data-00005-of-00036.arrow"
20
+ },
21
+ {
22
+ "filename": "data-00006-of-00036.arrow"
23
+ },
24
+ {
25
+ "filename": "data-00007-of-00036.arrow"
26
+ },
27
+ {
28
+ "filename": "data-00008-of-00036.arrow"
29
+ },
30
+ {
31
+ "filename": "data-00009-of-00036.arrow"
32
+ },
33
+ {
34
+ "filename": "data-00010-of-00036.arrow"
35
+ },
36
+ {
37
+ "filename": "data-00011-of-00036.arrow"
38
+ },
39
+ {
40
+ "filename": "data-00012-of-00036.arrow"
41
+ },
42
+ {
43
+ "filename": "data-00013-of-00036.arrow"
44
+ },
45
+ {
46
+ "filename": "data-00014-of-00036.arrow"
47
+ },
48
+ {
49
+ "filename": "data-00015-of-00036.arrow"
50
+ },
51
+ {
52
+ "filename": "data-00016-of-00036.arrow"
53
+ },
54
+ {
55
+ "filename": "data-00017-of-00036.arrow"
56
+ },
57
+ {
58
+ "filename": "data-00018-of-00036.arrow"
59
+ },
60
+ {
61
+ "filename": "data-00019-of-00036.arrow"
62
+ },
63
+ {
64
+ "filename": "data-00020-of-00036.arrow"
65
+ },
66
+ {
67
+ "filename": "data-00021-of-00036.arrow"
68
+ },
69
+ {
70
+ "filename": "data-00022-of-00036.arrow"
71
+ },
72
+ {
73
+ "filename": "data-00023-of-00036.arrow"
74
+ },
75
+ {
76
+ "filename": "data-00024-of-00036.arrow"
77
+ },
78
+ {
79
+ "filename": "data-00025-of-00036.arrow"
80
+ },
81
+ {
82
+ "filename": "data-00026-of-00036.arrow"
83
+ },
84
+ {
85
+ "filename": "data-00027-of-00036.arrow"
86
+ },
87
+ {
88
+ "filename": "data-00028-of-00036.arrow"
89
+ },
90
+ {
91
+ "filename": "data-00029-of-00036.arrow"
92
+ },
93
+ {
94
+ "filename": "data-00030-of-00036.arrow"
95
+ },
96
+ {
97
+ "filename": "data-00031-of-00036.arrow"
98
+ },
99
+ {
100
+ "filename": "data-00032-of-00036.arrow"
101
+ },
102
+ {
103
+ "filename": "data-00033-of-00036.arrow"
104
+ },
105
+ {
106
+ "filename": "data-00034-of-00036.arrow"
107
+ },
108
+ {
109
+ "filename": "data-00035-of-00036.arrow"
110
+ }
111
+ ],
112
+ "_fingerprint": "72767e6016ca2433",
113
+ "_format_columns": null,
114
+ "_format_kwargs": {},
115
+ "_format_type": null,
116
+ "_output_all_columns": false,
117
+ "_split": "train"
118
+ }
data/processed/owt-gpt2-1024/validation/state.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_data_files": [
3
+ {
4
+ "filename": "data-00000-of-00001.arrow"
5
+ }
6
+ ],
7
+ "_fingerprint": "4e65e78468281926",
8
+ "_format_columns": null,
9
+ "_format_kwargs": {},
10
+ "_format_type": null,
11
+ "_output_all_columns": false,
12
+ "_split": "train"
13
+ }
data/raw/huggingface/datasets/Skylion007___openwebtext/plain_text/0.0.0/79d93d786212f7344586290adb811d4ae6a1762c.incomplete_info.lock ADDED
File without changes
data/raw/huggingface/datasets/Skylion007___openwebtext/plain_text/0.0.0/79d93d786212f7344586290adb811d4ae6a1762c/dataset_info.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"description": "", "citation": "", "homepage": "", "license": "", "features": {"text": {"dtype": "string", "_type": "Value"}}, "builder_name": "parquet", "dataset_name": "openwebtext", "config_name": "plain_text", "version": {"version_str": "0.0.0", "major": 0, "minor": 0, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 39769491688, "num_examples": 8013769, "shard_lengths": [101173, 100173, 100173, 100173, 102173, 101173, 101173, 102173, 101173, 101172, 101172, 101172, 101172, 101172, 101172, 101172, 101172, 101172, 102172, 102172, 100172, 101172, 100172, 101172, 102172, 101172, 101172, 101172, 101172, 101172, 102172, 100172, 101172, 101172, 101172, 101172, 101172, 101172, 101172, 101172, 100172, 101172, 101172, 101172, 101172, 101172, 101172, 101172, 101172, 101172, 101172, 100172, 101172, 101172, 101172, 101172, 101172, 101172, 101172, 102172, 101172, 101172, 101172, 101172, 101172, 102172, 101172, 101172, 102172, 101172, 102172, 101172, 102172, 102172, 102172, 100172, 101172, 102172, 101172, 16172], "dataset_name": "openwebtext"}}, "download_checksums": {"hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00000-of-00080.parquet": {"num_bytes": 302848326, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00001-of-00080.parquet": {"num_bytes": 305678548, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00002-of-00080.parquet": {"num_bytes": 304419240, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00003-of-00080.parquet": {"num_bytes": 304332042, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00004-of-00080.parquet": {"num_bytes": 301175397, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00005-of-00080.parquet": {"num_bytes": 301755705, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00006-of-00080.parquet": {"num_bytes": 303536715, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00007-of-00080.parquet": {"num_bytes": 300180244, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00008-of-00080.parquet": {"num_bytes": 302694767, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00009-of-00080.parquet": {"num_bytes": 302691677, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00010-of-00080.parquet": {"num_bytes": 302204895, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00011-of-00080.parquet": {"num_bytes": 302975845, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00012-of-00080.parquet": {"num_bytes": 302244422, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00013-of-00080.parquet": {"num_bytes": 302055257, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00014-of-00080.parquet": {"num_bytes": 304001822, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00015-of-00080.parquet": {"num_bytes": 303473398, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00016-of-00080.parquet": {"num_bytes": 304131969, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00017-of-00080.parquet": {"num_bytes": 301375243, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00018-of-00080.parquet": {"num_bytes": 301508378, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00019-of-00080.parquet": {"num_bytes": 301406222, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00020-of-00080.parquet": {"num_bytes": 303965188, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00021-of-00080.parquet": {"num_bytes": 303492063, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00022-of-00080.parquet": {"num_bytes": 304784301, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00023-of-00080.parquet": {"num_bytes": 305250514, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00024-of-00080.parquet": {"num_bytes": 300473667, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00025-of-00080.parquet": {"num_bytes": 303497159, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00026-of-00080.parquet": {"num_bytes": 300841872, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00027-of-00080.parquet": {"num_bytes": 301179312, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00028-of-00080.parquet": {"num_bytes": 302216916, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00029-of-00080.parquet": {"num_bytes": 302599929, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00030-of-00080.parquet": {"num_bytes": 299345408, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00031-of-00080.parquet": {"num_bytes": 302655351, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00032-of-00080.parquet": {"num_bytes": 303272946, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00033-of-00080.parquet": {"num_bytes": 301539738, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00034-of-00080.parquet": {"num_bytes": 302560727, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00035-of-00080.parquet": {"num_bytes": 301341568, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00036-of-00080.parquet": {"num_bytes": 303206782, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00037-of-00080.parquet": {"num_bytes": 302424854, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00038-of-00080.parquet": {"num_bytes": 302372305, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00039-of-00080.parquet": {"num_bytes": 301636847, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00040-of-00080.parquet": {"num_bytes": 303737749, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00041-of-00080.parquet": {"num_bytes": 304266038, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00042-of-00080.parquet": {"num_bytes": 302040879, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00043-of-00080.parquet": {"num_bytes": 305323537, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00044-of-00080.parquet": {"num_bytes": 301885118, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00045-of-00080.parquet": {"num_bytes": 303334655, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00046-of-00080.parquet": {"num_bytes": 302169880, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00047-of-00080.parquet": {"num_bytes": 302501111, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00048-of-00080.parquet": {"num_bytes": 302652427, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00049-of-00080.parquet": {"num_bytes": 301871994, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00050-of-00080.parquet": {"num_bytes": 303349696, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00051-of-00080.parquet": {"num_bytes": 305031989, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00052-of-00080.parquet": {"num_bytes": 303050982, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00053-of-00080.parquet": {"num_bytes": 300684516, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00054-of-00080.parquet": {"num_bytes": 303631897, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00055-of-00080.parquet": {"num_bytes": 302735425, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00056-of-00080.parquet": {"num_bytes": 303083314, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00057-of-00080.parquet": {"num_bytes": 300544465, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00058-of-00080.parquet": {"num_bytes": 303089061, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00059-of-00080.parquet": {"num_bytes": 301706009, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00060-of-00080.parquet": {"num_bytes": 301068661, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00061-of-00080.parquet": {"num_bytes": 302642138, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00062-of-00080.parquet": {"num_bytes": 301151001, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00063-of-00080.parquet": {"num_bytes": 300798197, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00064-of-00080.parquet": {"num_bytes": 303348229, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00065-of-00080.parquet": {"num_bytes": 301745696, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00066-of-00080.parquet": {"num_bytes": 299821953, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00067-of-00080.parquet": {"num_bytes": 301605059, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00068-of-00080.parquet": {"num_bytes": 303100801, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00069-of-00080.parquet": {"num_bytes": 301381334, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00070-of-00080.parquet": {"num_bytes": 300170424, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00071-of-00080.parquet": {"num_bytes": 301359176, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00072-of-00080.parquet": {"num_bytes": 301922808, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00073-of-00080.parquet": {"num_bytes": 300555941, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00074-of-00080.parquet": {"num_bytes": 300259549, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00075-of-00080.parquet": {"num_bytes": 302686876, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00076-of-00080.parquet": {"num_bytes": 302646710, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00077-of-00080.parquet": {"num_bytes": 303181602, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00078-of-00080.parquet": {"num_bytes": 300854488, "checksum": null}, "hf://datasets/Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c/plain_text/train-00079-of-00080.parquet": {"num_bytes": 302753464, "checksum": null}}, "download_size": 24193092408, "dataset_size": 39769491688, "size_in_bytes": 63962584096}
data/raw/huggingface/datasets/Skylion007___openwebtext/plain_text/0.0.0/79d93d786212f7344586290adb811d4ae6a1762c_builder.lock ADDED
File without changes
data/raw/huggingface/datasets/_home_ruizzhao_diffusion_baseline_data_raw_huggingface_datasets_Skylion007___openwebtext_plain_text_0.0.0_79d93d786212f7344586290adb811d4ae6a1762c.lock ADDED
File without changes
data/raw/huggingface/datasets/_home_ruizzhao_diffusion_baseline_data_raw_huggingface_datasets_parquet_default-7e9cb4a2e4e2a71c_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock ADDED
File without changes
data/raw/huggingface/datasets/parquet/default-7e9cb4a2e4e2a71c/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.incomplete_info.lock ADDED
File without changes
data/raw/huggingface/datasets/parquet/default-7e9cb4a2e4e2a71c/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"description": "", "citation": "", "homepage": "", "license": "", "features": {"text": {"dtype": "string", "_type": "Value"}}, "builder_name": "parquet", "dataset_name": "parquet", "config_name": "default", "version": {"version_str": "0.0.0", "major": 0, "minor": 0, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 4238194392, "num_examples": 30301028, "shard_lengths": [3574000, 3575000, 3575000, 3575000, 3574000, 3578000, 3576000, 3574000, 1700028], "dataset_name": "parquet"}, "test": {"name": "test", "num_bytes": 42941921, "num_examples": 306688, "dataset_name": "parquet"}}, "download_checksums": {"/home/ruizzhao/diffusion_baseline/data/raw/huggingface/hub/datasets--billion-word-benchmark--lm1b/snapshots/8d52bfd3cc2819fc1166dbb8144c328e2690de3e/plain_text/train/0000.parquet": {"num_bytes": 360836029, "checksum": null}, "/home/ruizzhao/diffusion_baseline/data/raw/huggingface/hub/datasets--billion-word-benchmark--lm1b/snapshots/8d52bfd3cc2819fc1166dbb8144c328e2690de3e/plain_text/train/0001.parquet": {"num_bytes": 360852401, "checksum": null}, "/home/ruizzhao/diffusion_baseline/data/raw/huggingface/hub/datasets--billion-word-benchmark--lm1b/snapshots/8d52bfd3cc2819fc1166dbb8144c328e2690de3e/plain_text/train/0002.parquet": {"num_bytes": 360878132, "checksum": null}, "/home/ruizzhao/diffusion_baseline/data/raw/huggingface/hub/datasets--billion-word-benchmark--lm1b/snapshots/8d52bfd3cc2819fc1166dbb8144c328e2690de3e/plain_text/train/0003.parquet": {"num_bytes": 360862579, "checksum": null}, "/home/ruizzhao/diffusion_baseline/data/raw/huggingface/hub/datasets--billion-word-benchmark--lm1b/snapshots/8d52bfd3cc2819fc1166dbb8144c328e2690de3e/plain_text/train/0004.parquet": {"num_bytes": 360996825, "checksum": null}, "/home/ruizzhao/diffusion_baseline/data/raw/huggingface/hub/datasets--billion-word-benchmark--lm1b/snapshots/8d52bfd3cc2819fc1166dbb8144c328e2690de3e/plain_text/train/0005.parquet": {"num_bytes": 360965725, "checksum": null}, "/home/ruizzhao/diffusion_baseline/data/raw/huggingface/hub/datasets--billion-word-benchmark--lm1b/snapshots/8d52bfd3cc2819fc1166dbb8144c328e2690de3e/plain_text/train/0006.parquet": {"num_bytes": 360928740, "checksum": null}, "/home/ruizzhao/diffusion_baseline/data/raw/huggingface/hub/datasets--billion-word-benchmark--lm1b/snapshots/8d52bfd3cc2819fc1166dbb8144c328e2690de3e/plain_text/train/0007.parquet": {"num_bytes": 360903236, "checksum": null}, "/home/ruizzhao/diffusion_baseline/data/raw/huggingface/hub/datasets--billion-word-benchmark--lm1b/snapshots/8d52bfd3cc2819fc1166dbb8144c328e2690de3e/plain_text/train/0008.parquet": {"num_bytes": 171608604, "checksum": null}, "/home/ruizzhao/diffusion_baseline/data/raw/huggingface/hub/datasets--billion-word-benchmark--lm1b/snapshots/8d52bfd3cc2819fc1166dbb8144c328e2690de3e/plain_text/test/0000.parquet": {"num_bytes": 30988440, "checksum": null}}, "download_size": 3089820711, "dataset_size": 4281136313, "size_in_bytes": 7370957024}
data/raw/huggingface/datasets/parquet/default-7e9cb4a2e4e2a71c/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock ADDED
File without changes
data/raw/huggingface/hub/models--bert-base-uncased/blobs/40a2aaca31dd005eb5f6ffad07b5ffed0a31d1f6 ADDED
@@ -0,0 +1,251 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language: en
3
+ tags:
4
+ - exbert
5
+ license: apache-2.0
6
+ datasets:
7
+ - bookcorpus
8
+ - wikipedia
9
+ ---
10
+
11
+ # BERT base model (uncased)
12
+
13
+ Pretrained model on English language using a masked language modeling (MLM) objective. It was introduced in
14
+ [this paper](https://arxiv.org/abs/1810.04805) and first released in
15
+ [this repository](https://github.com/google-research/bert). This model is uncased: it does not make a difference
16
+ between english and English.
17
+
18
+ Disclaimer: The team releasing BERT did not write a model card for this model so this model card has been written by
19
+ the Hugging Face team.
20
+
21
+ ## Model description
22
+
23
+ BERT is a transformers model pretrained on a large corpus of English data in a self-supervised fashion. This means it
24
+ was pretrained on the raw texts only, with no humans labeling them in any way (which is why it can use lots of
25
+ publicly available data) with an automatic process to generate inputs and labels from those texts. More precisely, it
26
+ was pretrained with two objectives:
27
+
28
+ - Masked language modeling (MLM): taking a sentence, the model randomly masks 15% of the words in the input then run
29
+ the entire masked sentence through the model and has to predict the masked words. This is different from traditional
30
+ recurrent neural networks (RNNs) that usually see the words one after the other, or from autoregressive models like
31
+ GPT which internally masks the future tokens. It allows the model to learn a bidirectional representation of the
32
+ sentence.
33
+ - Next sentence prediction (NSP): the models concatenates two masked sentences as inputs during pretraining. Sometimes
34
+ they correspond to sentences that were next to each other in the original text, sometimes not. The model then has to
35
+ predict if the two sentences were following each other or not.
36
+
37
+ This way, the model learns an inner representation of the English language that can then be used to extract features
38
+ useful for downstream tasks: if you have a dataset of labeled sentences, for instance, you can train a standard
39
+ classifier using the features produced by the BERT model as inputs.
40
+
41
+ ## Model variations
42
+
43
+ BERT has originally been released in base and large variations, for cased and uncased input text. The uncased models also strips out an accent markers.
44
+ Chinese and multilingual uncased and cased versions followed shortly after.
45
+ Modified preprocessing with whole word masking has replaced subpiece masking in a following work, with the release of two models.
46
+ Other 24 smaller models are released afterward.
47
+
48
+ The detailed release history can be found on the [google-research/bert readme](https://github.com/google-research/bert/blob/master/README.md) on github.
49
+
50
+ | Model | #params | Language |
51
+ |------------------------|--------------------------------|-------|
52
+ | [`bert-base-uncased`](https://huggingface.co/bert-base-uncased) | 110M | English |
53
+ | [`bert-large-uncased`](https://huggingface.co/bert-large-uncased) | 340M | English | sub
54
+ | [`bert-base-cased`](https://huggingface.co/bert-base-cased) | 110M | English |
55
+ | [`bert-large-cased`](https://huggingface.co/bert-large-cased) | 340M | English |
56
+ | [`bert-base-chinese`](https://huggingface.co/bert-base-chinese) | 110M | Chinese |
57
+ | [`bert-base-multilingual-cased`](https://huggingface.co/bert-base-multilingual-cased) | 110M | Multiple |
58
+ | [`bert-large-uncased-whole-word-masking`](https://huggingface.co/bert-large-uncased-whole-word-masking) | 340M | English |
59
+ | [`bert-large-cased-whole-word-masking`](https://huggingface.co/bert-large-cased-whole-word-masking) | 340M | English |
60
+
61
+ ## Intended uses & limitations
62
+
63
+ You can use the raw model for either masked language modeling or next sentence prediction, but it's mostly intended to
64
+ be fine-tuned on a downstream task. See the [model hub](https://huggingface.co/models?filter=bert) to look for
65
+ fine-tuned versions of a task that interests you.
66
+
67
+ Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked)
68
+ to make decisions, such as sequence classification, token classification or question answering. For tasks such as text
69
+ generation you should look at model like GPT2.
70
+
71
+ ### How to use
72
+
73
+ You can use this model directly with a pipeline for masked language modeling:
74
+
75
+ ```python
76
+ >>> from transformers import pipeline
77
+ >>> unmasker = pipeline('fill-mask', model='bert-base-uncased')
78
+ >>> unmasker("Hello I'm a [MASK] model.")
79
+
80
+ [{'sequence': "[CLS] hello i'm a fashion model. [SEP]",
81
+ 'score': 0.1073106899857521,
82
+ 'token': 4827,
83
+ 'token_str': 'fashion'},
84
+ {'sequence': "[CLS] hello i'm a role model. [SEP]",
85
+ 'score': 0.08774490654468536,
86
+ 'token': 2535,
87
+ 'token_str': 'role'},
88
+ {'sequence': "[CLS] hello i'm a new model. [SEP]",
89
+ 'score': 0.05338378623127937,
90
+ 'token': 2047,
91
+ 'token_str': 'new'},
92
+ {'sequence': "[CLS] hello i'm a super model. [SEP]",
93
+ 'score': 0.04667217284440994,
94
+ 'token': 3565,
95
+ 'token_str': 'super'},
96
+ {'sequence': "[CLS] hello i'm a fine model. [SEP]",
97
+ 'score': 0.027095865458250046,
98
+ 'token': 2986,
99
+ 'token_str': 'fine'}]
100
+ ```
101
+
102
+ Here is how to use this model to get the features of a given text in PyTorch:
103
+
104
+ ```python
105
+ from transformers import BertTokenizer, BertModel
106
+ tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
107
+ model = BertModel.from_pretrained("bert-base-uncased")
108
+ text = "Replace me by any text you'd like."
109
+ encoded_input = tokenizer(text, return_tensors='pt')
110
+ output = model(**encoded_input)
111
+ ```
112
+
113
+ and in TensorFlow:
114
+
115
+ ```python
116
+ from transformers import BertTokenizer, TFBertModel
117
+ tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
118
+ model = TFBertModel.from_pretrained("bert-base-uncased")
119
+ text = "Replace me by any text you'd like."
120
+ encoded_input = tokenizer(text, return_tensors='tf')
121
+ output = model(encoded_input)
122
+ ```
123
+
124
+ ### Limitations and bias
125
+
126
+ Even if the training data used for this model could be characterized as fairly neutral, this model can have biased
127
+ predictions:
128
+
129
+ ```python
130
+ >>> from transformers import pipeline
131
+ >>> unmasker = pipeline('fill-mask', model='bert-base-uncased')
132
+ >>> unmasker("The man worked as a [MASK].")
133
+
134
+ [{'sequence': '[CLS] the man worked as a carpenter. [SEP]',
135
+ 'score': 0.09747550636529922,
136
+ 'token': 10533,
137
+ 'token_str': 'carpenter'},
138
+ {'sequence': '[CLS] the man worked as a waiter. [SEP]',
139
+ 'score': 0.0523831807076931,
140
+ 'token': 15610,
141
+ 'token_str': 'waiter'},
142
+ {'sequence': '[CLS] the man worked as a barber. [SEP]',
143
+ 'score': 0.04962705448269844,
144
+ 'token': 13362,
145
+ 'token_str': 'barber'},
146
+ {'sequence': '[CLS] the man worked as a mechanic. [SEP]',
147
+ 'score': 0.03788609802722931,
148
+ 'token': 15893,
149
+ 'token_str': 'mechanic'},
150
+ {'sequence': '[CLS] the man worked as a salesman. [SEP]',
151
+ 'score': 0.037680890411138535,
152
+ 'token': 18968,
153
+ 'token_str': 'salesman'}]
154
+
155
+ >>> unmasker("The woman worked as a [MASK].")
156
+
157
+ [{'sequence': '[CLS] the woman worked as a nurse. [SEP]',
158
+ 'score': 0.21981462836265564,
159
+ 'token': 6821,
160
+ 'token_str': 'nurse'},
161
+ {'sequence': '[CLS] the woman worked as a waitress. [SEP]',
162
+ 'score': 0.1597415804862976,
163
+ 'token': 13877,
164
+ 'token_str': 'waitress'},
165
+ {'sequence': '[CLS] the woman worked as a maid. [SEP]',
166
+ 'score': 0.1154729500412941,
167
+ 'token': 10850,
168
+ 'token_str': 'maid'},
169
+ {'sequence': '[CLS] the woman worked as a prostitute. [SEP]',
170
+ 'score': 0.037968918681144714,
171
+ 'token': 19215,
172
+ 'token_str': 'prostitute'},
173
+ {'sequence': '[CLS] the woman worked as a cook. [SEP]',
174
+ 'score': 0.03042375110089779,
175
+ 'token': 5660,
176
+ 'token_str': 'cook'}]
177
+ ```
178
+
179
+ This bias will also affect all fine-tuned versions of this model.
180
+
181
+ ## Training data
182
+
183
+ The BERT model was pretrained on [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset consisting of 11,038
184
+ unpublished books and [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) (excluding lists, tables and
185
+ headers).
186
+
187
+ ## Training procedure
188
+
189
+ ### Preprocessing
190
+
191
+ The texts are lowercased and tokenized using WordPiece and a vocabulary size of 30,000. The inputs of the model are
192
+ then of the form:
193
+
194
+ ```
195
+ [CLS] Sentence A [SEP] Sentence B [SEP]
196
+ ```
197
+
198
+ With probability 0.5, sentence A and sentence B correspond to two consecutive sentences in the original corpus, and in
199
+ the other cases, it's another random sentence in the corpus. Note that what is considered a sentence here is a
200
+ consecutive span of text usually longer than a single sentence. The only constrain is that the result with the two
201
+ "sentences" has a combined length of less than 512 tokens.
202
+
203
+ The details of the masking procedure for each sentence are the following:
204
+ - 15% of the tokens are masked.
205
+ - In 80% of the cases, the masked tokens are replaced by `[MASK]`.
206
+ - In 10% of the cases, the masked tokens are replaced by a random token (different) from the one they replace.
207
+ - In the 10% remaining cases, the masked tokens are left as is.
208
+
209
+ ### Pretraining
210
+
211
+ The model was trained on 4 cloud TPUs in Pod configuration (16 TPU chips total) for one million steps with a batch size
212
+ of 256. The sequence length was limited to 128 tokens for 90% of the steps and 512 for the remaining 10%. The optimizer
213
+ used is Adam with a learning rate of 1e-4, \\(\beta_{1} = 0.9\\) and \\(\beta_{2} = 0.999\\), a weight decay of 0.01,
214
+ learning rate warmup for 10,000 steps and linear decay of the learning rate after.
215
+
216
+ ## Evaluation results
217
+
218
+ When fine-tuned on downstream tasks, this model achieves the following results:
219
+
220
+ Glue test results:
221
+
222
+ | Task | MNLI-(m/mm) | QQP | QNLI | SST-2 | CoLA | STS-B | MRPC | RTE | Average |
223
+ |:----:|:-----------:|:----:|:----:|:-----:|:----:|:-----:|:----:|:----:|:-------:|
224
+ | | 84.6/83.4 | 71.2 | 90.5 | 93.5 | 52.1 | 85.8 | 88.9 | 66.4 | 79.6 |
225
+
226
+
227
+ ### BibTeX entry and citation info
228
+
229
+ ```bibtex
230
+ @article{DBLP:journals/corr/abs-1810-04805,
231
+ author = {Jacob Devlin and
232
+ Ming{-}Wei Chang and
233
+ Kenton Lee and
234
+ Kristina Toutanova},
235
+ title = {{BERT:} Pre-training of Deep Bidirectional Transformers for Language
236
+ Understanding},
237
+ journal = {CoRR},
238
+ volume = {abs/1810.04805},
239
+ year = {2018},
240
+ url = {http://arxiv.org/abs/1810.04805},
241
+ archivePrefix = {arXiv},
242
+ eprint = {1810.04805},
243
+ timestamp = {Tue, 30 Oct 2018 20:39:56 +0100},
244
+ biburl = {https://dblp.org/rec/journals/corr/abs-1810-04805.bib},
245
+ bibsource = {dblp computer science bibliography, https://dblp.org}
246
+ }
247
+ ```
248
+
249
+ <a href="https://huggingface.co/exbert/?model=bert-base-uncased">
250
+ <img width="300px" src="https://cdn-media.huggingface.co/exbert/button.png">
251
+ </a>
data/raw/huggingface/hub/models--bert-base-uncased/blobs/45a2321a7ecfdaaf60a6c1fd7f5463994cc8907d ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "BertForMaskedLM"
4
+ ],
5
+ "attention_probs_dropout_prob": 0.1,
6
+ "gradient_checkpointing": false,
7
+ "hidden_act": "gelu",
8
+ "hidden_dropout_prob": 0.1,
9
+ "hidden_size": 768,
10
+ "initializer_range": 0.02,
11
+ "intermediate_size": 3072,
12
+ "layer_norm_eps": 1e-12,
13
+ "max_position_embeddings": 512,
14
+ "model_type": "bert",
15
+ "num_attention_heads": 12,
16
+ "num_hidden_layers": 12,
17
+ "pad_token_id": 0,
18
+ "position_embedding_type": "absolute",
19
+ "transformers_version": "4.6.0.dev0",
20
+ "type_vocab_size": 2,
21
+ "use_cache": true,
22
+ "vocab_size": 30522
23
+ }
data/raw/huggingface/hub/models--bert-base-uncased/blobs/505a7adf8be9e5fdf06aabbfbe9046e6c811f91b ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.bin.* filter=lfs diff=lfs merge=lfs -text
2
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.h5 filter=lfs diff=lfs merge=lfs -text
5
+ *.tflite filter=lfs diff=lfs merge=lfs -text
6
+ *.tar.gz filter=lfs diff=lfs merge=lfs -text
7
+ *.ot filter=lfs diff=lfs merge=lfs -text
8
+ *.onnx filter=lfs diff=lfs merge=lfs -text
9
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
10
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
11
+ model.safetensors filter=lfs diff=lfs merge=lfs -text
data/raw/huggingface/hub/models--bert-base-uncased/blobs/949a6f013d67eb8a5b4b5b46026217b888021b88 ADDED
The diff for this file is too large to render. See raw diff
 
data/raw/huggingface/hub/models--bert-base-uncased/blobs/c1c37cd58b9eb000ddbb7ca90f04b893a33e50c8 ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "fileFormatVersion": "1.0.0",
3
+ "itemInfoEntries": {
4
+ "9D749A46-ADA0-43CA-B5C2-8E722B91F41E": {
5
+ "author": "com.apple.CoreML",
6
+ "description": "CoreML Model Specification",
7
+ "name": "model.mlmodel",
8
+ "path": "com.apple.CoreML/model.mlmodel"
9
+ },
10
+ "D545B13F-2D5E-4CFB-BFF1-C10E9EFD70DA": {
11
+ "author": "com.apple.CoreML",
12
+ "description": "CoreML Model Weights",
13
+ "name": "weights",
14
+ "path": "com.apple.CoreML/weights"
15
+ }
16
+ },
17
+ "rootModelIdentifier": "9D749A46-ADA0-43CA-B5C2-8E722B91F41E"
18
+ }
data/raw/huggingface/hub/models--bert-base-uncased/blobs/e5c73d8a50df1f56fb5b0b8002d7cf4010afdccb ADDED
@@ -0,0 +1 @@
 
 
1
+ {"do_lower_case": true, "model_max_length": 512}
data/raw/huggingface/hub/models--bert-base-uncased/blobs/f49a4e16e68b128803cc2dcea614603632b04eac ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship, whether in Source or
36
+ Object form, made available under the License, as indicated by a
37
+ copyright notice that is included in or attached to the work
38
+ (an example is provided in the Appendix below).
39
+
40
+ "Derivative Works" shall mean any work, whether in Source or Object
41
+ form, that is based on (or derived from) the Work and for which the
42
+ editorial revisions, annotations, elaborations, or other modifications
43
+ represent, as a whole, an original work of authorship. For the purposes
44
+ of this License, Derivative Works shall not include works that remain
45
+ separable from, or merely link (or bind by name) to the interfaces of,
46
+ the Work and Derivative Works thereof.
47
+
48
+ "Contribution" shall mean any work of authorship, including
49
+ the original version of the Work and any modifications or additions
50
+ to that Work or Derivative Works thereof, that is intentionally
51
+ submitted to Licensor for inclusion in the Work by the copyright owner
52
+ or by an individual or Legal Entity authorized to submit on behalf of
53
+ the copyright owner. For the purposes of this definition, "submitted"
54
+ means any form of electronic, verbal, or written communication sent
55
+ to the Licensor or its representatives, including but not limited to
56
+ communication on electronic mailing lists, source code control systems,
57
+ and issue tracking systems that are managed by, or on behalf of, the
58
+ Licensor for the purpose of discussing and improving the Work, but
59
+ excluding communication that is conspicuously marked or otherwise
60
+ designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+ "Contributor" shall mean Licensor and any individual or Legal Entity
63
+ on behalf of whom a Contribution has been received by Licensor and
64
+ subsequently incorporated within the Work.
65
+
66
+ 2. Grant of Copyright License. Subject to the terms and conditions of
67
+ this License, each Contributor hereby grants to You a perpetual,
68
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+ copyright license to reproduce, prepare Derivative Works of,
70
+ publicly display, publicly perform, sublicense, and distribute the
71
+ Work and such Derivative Works in Source or Object form.
72
+
73
+ 3. Grant of Patent License. Subject to the terms and conditions of
74
+ this License, each Contributor hereby grants to You a perpetual,
75
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+ (except as stated in this section) patent license to make, have made,
77
+ use, offer to sell, sell, import, and otherwise transfer the Work,
78
+ where such license applies only to those patent claims licensable
79
+ by such Contributor that are necessarily infringed by their
80
+ Contribution(s) alone or by combination of their Contribution(s)
81
+ with the Work to which such Contribution(s) was submitted. If You
82
+ institute patent litigation against any entity (including a
83
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84
+ or a Contribution incorporated within the Work constitutes direct
85
+ or contributory patent infringement, then any patent licenses
86
+ granted to You under this License for that Work shall terminate
87
+ as of the date such litigation is filed.
88
+
89
+ 4. Redistribution. You may reproduce and distribute copies of the
90
+ Work or Derivative Works thereof in any medium, with or without
91
+ modifications, and in Source or Object form, provided that You
92
+ meet the following conditions:
93
+
94
+ (a) You must give any other recipients of the Work or
95
+ Derivative Works a copy of this License; and
96
+
97
+ (b) You must cause any modified files to carry prominent notices
98
+ stating that You changed the files; and
99
+
100
+ (c) You must retain, in the Source form of any Derivative Works
101
+ that You distribute, all copyright, patent, trademark, and
102
+ attribution notices from the Source form of the Work,
103
+ excluding those notices that do not pertain to any part of
104
+ the Derivative Works; and
105
+
106
+ (d) If the Work includes a "NOTICE" text file as part of its
107
+ distribution, then any Derivative Works that You distribute must
108
+ include a readable copy of the attribution notices contained
109
+ within such NOTICE file, excluding those notices that do not
110
+ pertain to any part of the Derivative Works, in at least one
111
+ of the following places: within a NOTICE text file distributed
112
+ as part of the Derivative Works; within the Source form or
113
+ documentation, if provided along with the Derivative Works; or,
114
+ within a display generated by the Derivative Works, if and
115
+ wherever such third-party notices normally appear. The contents
116
+ of the NOTICE file are for informational purposes only and
117
+ do not modify the License. You may add Your own attribution
118
+ notices within Derivative Works that You distribute, alongside
119
+ or as an addendum to the NOTICE text from the Work, provided
120
+ that such additional attribution notices cannot be construed
121
+ as modifying the License.
122
+
123
+ You may add Your own copyright statement to Your modifications and
124
+ may provide additional or different license terms and conditions
125
+ for use, reproduction, or distribution of Your modifications, or
126
+ for any such Derivative Works as a whole, provided Your use,
127
+ reproduction, and distribution of the Work otherwise complies with
128
+ the conditions stated in this License.
129
+
130
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131
+ any Contribution intentionally submitted for inclusion in the Work
132
+ by You to the Licensor shall be under the terms and conditions of
133
+ this License, without any additional terms or conditions.
134
+ Notwithstanding the above, nothing herein shall supersede or modify
135
+ the terms of any separate license agreement you may have executed
136
+ with Licensor regarding such Contributions.
137
+
138
+ 6. Trademarks. This License does not grant permission to use the trade
139
+ names, trademarks, service marks, or product names of the Licensor,
140
+ except as required for reasonable and customary use in describing the
141
+ origin of the Work and reproducing the content of the NOTICE file.
142
+
143
+ 7. Disclaimer of Warranty. Unless required by applicable law or
144
+ agreed to in writing, Licensor provides the Work (and each
145
+ Contributor provides its Contributions) on an "AS IS" BASIS,
146
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
+ implied, including, without limitation, any warranties or conditions
148
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
+ PARTICULAR PURPOSE. You are solely responsible for determining the
150
+ appropriateness of using or redistributing the Work and assume any
151
+ risks associated with Your exercise of permissions under this License.
152
+
153
+ 8. Limitation of Liability. In no event and under no legal theory,
154
+ whether in tort (including negligence), contract, or otherwise,
155
+ unless required by applicable law (such as deliberate and grossly
156
+ negligent acts) or agreed to in writing, shall any Contributor be
157
+ liable to You for damages, including any direct, indirect, special,
158
+ incidental, or consequential damages of any character arising as a
159
+ result of this License or out of the use or inability to use the
160
+ Work (including but not limited to damages for loss of goodwill,
161
+ work stoppage, computer failure or malfunction, or any and all
162
+ other commercial damages or losses), even if such Contributor
163
+ has been advised of the possibility of such damages.
164
+
165
+ 9. Accepting Warranty or Additional Liability. While redistributing
166
+ the Work or Derivative Works thereof, You may choose to offer,
167
+ and charge a fee for, acceptance of support, warranty, indemnity,
168
+ or other liability obligations and/or rights consistent with this
169
+ License. However, in accepting such obligations, You may act only
170
+ on Your own behalf and on Your sole responsibility, not on behalf
171
+ of any other Contributor, and only if You agree to indemnify,
172
+ defend, and hold each Contributor harmless for any liability
173
+ incurred by, or claims asserted against, such Contributor by reason
174
+ of your accepting any such warranty or additional liability.
175
+
176
+ END OF TERMS AND CONDITIONS
177
+
178
+ APPENDIX: How to apply the Apache License to your work.
179
+
180
+ To apply the Apache License to your work, attach the following
181
+ boilerplate notice, with the fields enclosed by brackets "[]"
182
+ replaced with your own identifying information. (Don't include
183
+ the brackets!) The text should be enclosed in the appropriate
184
+ comment syntax for the file format. We also recommend that a
185
+ file or class name and description of purpose be included on the
186
+ same "printed page" as the copyright notice for easier
187
+ identification within third-party archives.
188
+
189
+ Copyright [yyyy] [name of copyright owner]
190
+
191
+ Licensed under the Apache License, Version 2.0 (the "License");
192
+ you may not use this file except in compliance with the License.
193
+ You may obtain a copy of the License at
194
+
195
+ http://www.apache.org/licenses/LICENSE-2.0
196
+
197
+ Unless required by applicable law or agreed to in writing, software
198
+ distributed under the License is distributed on an "AS IS" BASIS,
199
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
+ See the License for the specific language governing permissions and
201
+ limitations under the License.
data/raw/huggingface/hub/models--bert-base-uncased/blobs/fb140275c155a9c7c5a3b3e0e77a9e839594a938 ADDED
The diff for this file is too large to render. See raw diff
 
data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594/.gitattributes ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.bin.* filter=lfs diff=lfs merge=lfs -text
2
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.h5 filter=lfs diff=lfs merge=lfs -text
5
+ *.tflite filter=lfs diff=lfs merge=lfs -text
6
+ *.tar.gz filter=lfs diff=lfs merge=lfs -text
7
+ *.ot filter=lfs diff=lfs merge=lfs -text
8
+ *.onnx filter=lfs diff=lfs merge=lfs -text
9
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
10
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
11
+ model.safetensors filter=lfs diff=lfs merge=lfs -text
data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594/LICENSE ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship, whether in Source or
36
+ Object form, made available under the License, as indicated by a
37
+ copyright notice that is included in or attached to the work
38
+ (an example is provided in the Appendix below).
39
+
40
+ "Derivative Works" shall mean any work, whether in Source or Object
41
+ form, that is based on (or derived from) the Work and for which the
42
+ editorial revisions, annotations, elaborations, or other modifications
43
+ represent, as a whole, an original work of authorship. For the purposes
44
+ of this License, Derivative Works shall not include works that remain
45
+ separable from, or merely link (or bind by name) to the interfaces of,
46
+ the Work and Derivative Works thereof.
47
+
48
+ "Contribution" shall mean any work of authorship, including
49
+ the original version of the Work and any modifications or additions
50
+ to that Work or Derivative Works thereof, that is intentionally
51
+ submitted to Licensor for inclusion in the Work by the copyright owner
52
+ or by an individual or Legal Entity authorized to submit on behalf of
53
+ the copyright owner. For the purposes of this definition, "submitted"
54
+ means any form of electronic, verbal, or written communication sent
55
+ to the Licensor or its representatives, including but not limited to
56
+ communication on electronic mailing lists, source code control systems,
57
+ and issue tracking systems that are managed by, or on behalf of, the
58
+ Licensor for the purpose of discussing and improving the Work, but
59
+ excluding communication that is conspicuously marked or otherwise
60
+ designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+ "Contributor" shall mean Licensor and any individual or Legal Entity
63
+ on behalf of whom a Contribution has been received by Licensor and
64
+ subsequently incorporated within the Work.
65
+
66
+ 2. Grant of Copyright License. Subject to the terms and conditions of
67
+ this License, each Contributor hereby grants to You a perpetual,
68
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+ copyright license to reproduce, prepare Derivative Works of,
70
+ publicly display, publicly perform, sublicense, and distribute the
71
+ Work and such Derivative Works in Source or Object form.
72
+
73
+ 3. Grant of Patent License. Subject to the terms and conditions of
74
+ this License, each Contributor hereby grants to You a perpetual,
75
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+ (except as stated in this section) patent license to make, have made,
77
+ use, offer to sell, sell, import, and otherwise transfer the Work,
78
+ where such license applies only to those patent claims licensable
79
+ by such Contributor that are necessarily infringed by their
80
+ Contribution(s) alone or by combination of their Contribution(s)
81
+ with the Work to which such Contribution(s) was submitted. If You
82
+ institute patent litigation against any entity (including a
83
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84
+ or a Contribution incorporated within the Work constitutes direct
85
+ or contributory patent infringement, then any patent licenses
86
+ granted to You under this License for that Work shall terminate
87
+ as of the date such litigation is filed.
88
+
89
+ 4. Redistribution. You may reproduce and distribute copies of the
90
+ Work or Derivative Works thereof in any medium, with or without
91
+ modifications, and in Source or Object form, provided that You
92
+ meet the following conditions:
93
+
94
+ (a) You must give any other recipients of the Work or
95
+ Derivative Works a copy of this License; and
96
+
97
+ (b) You must cause any modified files to carry prominent notices
98
+ stating that You changed the files; and
99
+
100
+ (c) You must retain, in the Source form of any Derivative Works
101
+ that You distribute, all copyright, patent, trademark, and
102
+ attribution notices from the Source form of the Work,
103
+ excluding those notices that do not pertain to any part of
104
+ the Derivative Works; and
105
+
106
+ (d) If the Work includes a "NOTICE" text file as part of its
107
+ distribution, then any Derivative Works that You distribute must
108
+ include a readable copy of the attribution notices contained
109
+ within such NOTICE file, excluding those notices that do not
110
+ pertain to any part of the Derivative Works, in at least one
111
+ of the following places: within a NOTICE text file distributed
112
+ as part of the Derivative Works; within the Source form or
113
+ documentation, if provided along with the Derivative Works; or,
114
+ within a display generated by the Derivative Works, if and
115
+ wherever such third-party notices normally appear. The contents
116
+ of the NOTICE file are for informational purposes only and
117
+ do not modify the License. You may add Your own attribution
118
+ notices within Derivative Works that You distribute, alongside
119
+ or as an addendum to the NOTICE text from the Work, provided
120
+ that such additional attribution notices cannot be construed
121
+ as modifying the License.
122
+
123
+ You may add Your own copyright statement to Your modifications and
124
+ may provide additional or different license terms and conditions
125
+ for use, reproduction, or distribution of Your modifications, or
126
+ for any such Derivative Works as a whole, provided Your use,
127
+ reproduction, and distribution of the Work otherwise complies with
128
+ the conditions stated in this License.
129
+
130
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131
+ any Contribution intentionally submitted for inclusion in the Work
132
+ by You to the Licensor shall be under the terms and conditions of
133
+ this License, without any additional terms or conditions.
134
+ Notwithstanding the above, nothing herein shall supersede or modify
135
+ the terms of any separate license agreement you may have executed
136
+ with Licensor regarding such Contributions.
137
+
138
+ 6. Trademarks. This License does not grant permission to use the trade
139
+ names, trademarks, service marks, or product names of the Licensor,
140
+ except as required for reasonable and customary use in describing the
141
+ origin of the Work and reproducing the content of the NOTICE file.
142
+
143
+ 7. Disclaimer of Warranty. Unless required by applicable law or
144
+ agreed to in writing, Licensor provides the Work (and each
145
+ Contributor provides its Contributions) on an "AS IS" BASIS,
146
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
+ implied, including, without limitation, any warranties or conditions
148
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
+ PARTICULAR PURPOSE. You are solely responsible for determining the
150
+ appropriateness of using or redistributing the Work and assume any
151
+ risks associated with Your exercise of permissions under this License.
152
+
153
+ 8. Limitation of Liability. In no event and under no legal theory,
154
+ whether in tort (including negligence), contract, or otherwise,
155
+ unless required by applicable law (such as deliberate and grossly
156
+ negligent acts) or agreed to in writing, shall any Contributor be
157
+ liable to You for damages, including any direct, indirect, special,
158
+ incidental, or consequential damages of any character arising as a
159
+ result of this License or out of the use or inability to use the
160
+ Work (including but not limited to damages for loss of goodwill,
161
+ work stoppage, computer failure or malfunction, or any and all
162
+ other commercial damages or losses), even if such Contributor
163
+ has been advised of the possibility of such damages.
164
+
165
+ 9. Accepting Warranty or Additional Liability. While redistributing
166
+ the Work or Derivative Works thereof, You may choose to offer,
167
+ and charge a fee for, acceptance of support, warranty, indemnity,
168
+ or other liability obligations and/or rights consistent with this
169
+ License. However, in accepting such obligations, You may act only
170
+ on Your own behalf and on Your sole responsibility, not on behalf
171
+ of any other Contributor, and only if You agree to indemnify,
172
+ defend, and hold each Contributor harmless for any liability
173
+ incurred by, or claims asserted against, such Contributor by reason
174
+ of your accepting any such warranty or additional liability.
175
+
176
+ END OF TERMS AND CONDITIONS
177
+
178
+ APPENDIX: How to apply the Apache License to your work.
179
+
180
+ To apply the Apache License to your work, attach the following
181
+ boilerplate notice, with the fields enclosed by brackets "[]"
182
+ replaced with your own identifying information. (Don't include
183
+ the brackets!) The text should be enclosed in the appropriate
184
+ comment syntax for the file format. We also recommend that a
185
+ file or class name and description of purpose be included on the
186
+ same "printed page" as the copyright notice for easier
187
+ identification within third-party archives.
188
+
189
+ Copyright [yyyy] [name of copyright owner]
190
+
191
+ Licensed under the Apache License, Version 2.0 (the "License");
192
+ you may not use this file except in compliance with the License.
193
+ You may obtain a copy of the License at
194
+
195
+ http://www.apache.org/licenses/LICENSE-2.0
196
+
197
+ Unless required by applicable law or agreed to in writing, software
198
+ distributed under the License is distributed on an "AS IS" BASIS,
199
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
+ See the License for the specific language governing permissions and
201
+ limitations under the License.
data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594/README.md ADDED
@@ -0,0 +1,251 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language: en
3
+ tags:
4
+ - exbert
5
+ license: apache-2.0
6
+ datasets:
7
+ - bookcorpus
8
+ - wikipedia
9
+ ---
10
+
11
+ # BERT base model (uncased)
12
+
13
+ Pretrained model on English language using a masked language modeling (MLM) objective. It was introduced in
14
+ [this paper](https://arxiv.org/abs/1810.04805) and first released in
15
+ [this repository](https://github.com/google-research/bert). This model is uncased: it does not make a difference
16
+ between english and English.
17
+
18
+ Disclaimer: The team releasing BERT did not write a model card for this model so this model card has been written by
19
+ the Hugging Face team.
20
+
21
+ ## Model description
22
+
23
+ BERT is a transformers model pretrained on a large corpus of English data in a self-supervised fashion. This means it
24
+ was pretrained on the raw texts only, with no humans labeling them in any way (which is why it can use lots of
25
+ publicly available data) with an automatic process to generate inputs and labels from those texts. More precisely, it
26
+ was pretrained with two objectives:
27
+
28
+ - Masked language modeling (MLM): taking a sentence, the model randomly masks 15% of the words in the input then run
29
+ the entire masked sentence through the model and has to predict the masked words. This is different from traditional
30
+ recurrent neural networks (RNNs) that usually see the words one after the other, or from autoregressive models like
31
+ GPT which internally masks the future tokens. It allows the model to learn a bidirectional representation of the
32
+ sentence.
33
+ - Next sentence prediction (NSP): the models concatenates two masked sentences as inputs during pretraining. Sometimes
34
+ they correspond to sentences that were next to each other in the original text, sometimes not. The model then has to
35
+ predict if the two sentences were following each other or not.
36
+
37
+ This way, the model learns an inner representation of the English language that can then be used to extract features
38
+ useful for downstream tasks: if you have a dataset of labeled sentences, for instance, you can train a standard
39
+ classifier using the features produced by the BERT model as inputs.
40
+
41
+ ## Model variations
42
+
43
+ BERT has originally been released in base and large variations, for cased and uncased input text. The uncased models also strips out an accent markers.
44
+ Chinese and multilingual uncased and cased versions followed shortly after.
45
+ Modified preprocessing with whole word masking has replaced subpiece masking in a following work, with the release of two models.
46
+ Other 24 smaller models are released afterward.
47
+
48
+ The detailed release history can be found on the [google-research/bert readme](https://github.com/google-research/bert/blob/master/README.md) on github.
49
+
50
+ | Model | #params | Language |
51
+ |------------------------|--------------------------------|-------|
52
+ | [`bert-base-uncased`](https://huggingface.co/bert-base-uncased) | 110M | English |
53
+ | [`bert-large-uncased`](https://huggingface.co/bert-large-uncased) | 340M | English | sub
54
+ | [`bert-base-cased`](https://huggingface.co/bert-base-cased) | 110M | English |
55
+ | [`bert-large-cased`](https://huggingface.co/bert-large-cased) | 340M | English |
56
+ | [`bert-base-chinese`](https://huggingface.co/bert-base-chinese) | 110M | Chinese |
57
+ | [`bert-base-multilingual-cased`](https://huggingface.co/bert-base-multilingual-cased) | 110M | Multiple |
58
+ | [`bert-large-uncased-whole-word-masking`](https://huggingface.co/bert-large-uncased-whole-word-masking) | 340M | English |
59
+ | [`bert-large-cased-whole-word-masking`](https://huggingface.co/bert-large-cased-whole-word-masking) | 340M | English |
60
+
61
+ ## Intended uses & limitations
62
+
63
+ You can use the raw model for either masked language modeling or next sentence prediction, but it's mostly intended to
64
+ be fine-tuned on a downstream task. See the [model hub](https://huggingface.co/models?filter=bert) to look for
65
+ fine-tuned versions of a task that interests you.
66
+
67
+ Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked)
68
+ to make decisions, such as sequence classification, token classification or question answering. For tasks such as text
69
+ generation you should look at model like GPT2.
70
+
71
+ ### How to use
72
+
73
+ You can use this model directly with a pipeline for masked language modeling:
74
+
75
+ ```python
76
+ >>> from transformers import pipeline
77
+ >>> unmasker = pipeline('fill-mask', model='bert-base-uncased')
78
+ >>> unmasker("Hello I'm a [MASK] model.")
79
+
80
+ [{'sequence': "[CLS] hello i'm a fashion model. [SEP]",
81
+ 'score': 0.1073106899857521,
82
+ 'token': 4827,
83
+ 'token_str': 'fashion'},
84
+ {'sequence': "[CLS] hello i'm a role model. [SEP]",
85
+ 'score': 0.08774490654468536,
86
+ 'token': 2535,
87
+ 'token_str': 'role'},
88
+ {'sequence': "[CLS] hello i'm a new model. [SEP]",
89
+ 'score': 0.05338378623127937,
90
+ 'token': 2047,
91
+ 'token_str': 'new'},
92
+ {'sequence': "[CLS] hello i'm a super model. [SEP]",
93
+ 'score': 0.04667217284440994,
94
+ 'token': 3565,
95
+ 'token_str': 'super'},
96
+ {'sequence': "[CLS] hello i'm a fine model. [SEP]",
97
+ 'score': 0.027095865458250046,
98
+ 'token': 2986,
99
+ 'token_str': 'fine'}]
100
+ ```
101
+
102
+ Here is how to use this model to get the features of a given text in PyTorch:
103
+
104
+ ```python
105
+ from transformers import BertTokenizer, BertModel
106
+ tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
107
+ model = BertModel.from_pretrained("bert-base-uncased")
108
+ text = "Replace me by any text you'd like."
109
+ encoded_input = tokenizer(text, return_tensors='pt')
110
+ output = model(**encoded_input)
111
+ ```
112
+
113
+ and in TensorFlow:
114
+
115
+ ```python
116
+ from transformers import BertTokenizer, TFBertModel
117
+ tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
118
+ model = TFBertModel.from_pretrained("bert-base-uncased")
119
+ text = "Replace me by any text you'd like."
120
+ encoded_input = tokenizer(text, return_tensors='tf')
121
+ output = model(encoded_input)
122
+ ```
123
+
124
+ ### Limitations and bias
125
+
126
+ Even if the training data used for this model could be characterized as fairly neutral, this model can have biased
127
+ predictions:
128
+
129
+ ```python
130
+ >>> from transformers import pipeline
131
+ >>> unmasker = pipeline('fill-mask', model='bert-base-uncased')
132
+ >>> unmasker("The man worked as a [MASK].")
133
+
134
+ [{'sequence': '[CLS] the man worked as a carpenter. [SEP]',
135
+ 'score': 0.09747550636529922,
136
+ 'token': 10533,
137
+ 'token_str': 'carpenter'},
138
+ {'sequence': '[CLS] the man worked as a waiter. [SEP]',
139
+ 'score': 0.0523831807076931,
140
+ 'token': 15610,
141
+ 'token_str': 'waiter'},
142
+ {'sequence': '[CLS] the man worked as a barber. [SEP]',
143
+ 'score': 0.04962705448269844,
144
+ 'token': 13362,
145
+ 'token_str': 'barber'},
146
+ {'sequence': '[CLS] the man worked as a mechanic. [SEP]',
147
+ 'score': 0.03788609802722931,
148
+ 'token': 15893,
149
+ 'token_str': 'mechanic'},
150
+ {'sequence': '[CLS] the man worked as a salesman. [SEP]',
151
+ 'score': 0.037680890411138535,
152
+ 'token': 18968,
153
+ 'token_str': 'salesman'}]
154
+
155
+ >>> unmasker("The woman worked as a [MASK].")
156
+
157
+ [{'sequence': '[CLS] the woman worked as a nurse. [SEP]',
158
+ 'score': 0.21981462836265564,
159
+ 'token': 6821,
160
+ 'token_str': 'nurse'},
161
+ {'sequence': '[CLS] the woman worked as a waitress. [SEP]',
162
+ 'score': 0.1597415804862976,
163
+ 'token': 13877,
164
+ 'token_str': 'waitress'},
165
+ {'sequence': '[CLS] the woman worked as a maid. [SEP]',
166
+ 'score': 0.1154729500412941,
167
+ 'token': 10850,
168
+ 'token_str': 'maid'},
169
+ {'sequence': '[CLS] the woman worked as a prostitute. [SEP]',
170
+ 'score': 0.037968918681144714,
171
+ 'token': 19215,
172
+ 'token_str': 'prostitute'},
173
+ {'sequence': '[CLS] the woman worked as a cook. [SEP]',
174
+ 'score': 0.03042375110089779,
175
+ 'token': 5660,
176
+ 'token_str': 'cook'}]
177
+ ```
178
+
179
+ This bias will also affect all fine-tuned versions of this model.
180
+
181
+ ## Training data
182
+
183
+ The BERT model was pretrained on [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset consisting of 11,038
184
+ unpublished books and [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) (excluding lists, tables and
185
+ headers).
186
+
187
+ ## Training procedure
188
+
189
+ ### Preprocessing
190
+
191
+ The texts are lowercased and tokenized using WordPiece and a vocabulary size of 30,000. The inputs of the model are
192
+ then of the form:
193
+
194
+ ```
195
+ [CLS] Sentence A [SEP] Sentence B [SEP]
196
+ ```
197
+
198
+ With probability 0.5, sentence A and sentence B correspond to two consecutive sentences in the original corpus, and in
199
+ the other cases, it's another random sentence in the corpus. Note that what is considered a sentence here is a
200
+ consecutive span of text usually longer than a single sentence. The only constrain is that the result with the two
201
+ "sentences" has a combined length of less than 512 tokens.
202
+
203
+ The details of the masking procedure for each sentence are the following:
204
+ - 15% of the tokens are masked.
205
+ - In 80% of the cases, the masked tokens are replaced by `[MASK]`.
206
+ - In 10% of the cases, the masked tokens are replaced by a random token (different) from the one they replace.
207
+ - In the 10% remaining cases, the masked tokens are left as is.
208
+
209
+ ### Pretraining
210
+
211
+ The model was trained on 4 cloud TPUs in Pod configuration (16 TPU chips total) for one million steps with a batch size
212
+ of 256. The sequence length was limited to 128 tokens for 90% of the steps and 512 for the remaining 10%. The optimizer
213
+ used is Adam with a learning rate of 1e-4, \\(\beta_{1} = 0.9\\) and \\(\beta_{2} = 0.999\\), a weight decay of 0.01,
214
+ learning rate warmup for 10,000 steps and linear decay of the learning rate after.
215
+
216
+ ## Evaluation results
217
+
218
+ When fine-tuned on downstream tasks, this model achieves the following results:
219
+
220
+ Glue test results:
221
+
222
+ | Task | MNLI-(m/mm) | QQP | QNLI | SST-2 | CoLA | STS-B | MRPC | RTE | Average |
223
+ |:----:|:-----------:|:----:|:----:|:-----:|:----:|:-----:|:----:|:----:|:-------:|
224
+ | | 84.6/83.4 | 71.2 | 90.5 | 93.5 | 52.1 | 85.8 | 88.9 | 66.4 | 79.6 |
225
+
226
+
227
+ ### BibTeX entry and citation info
228
+
229
+ ```bibtex
230
+ @article{DBLP:journals/corr/abs-1810-04805,
231
+ author = {Jacob Devlin and
232
+ Ming{-}Wei Chang and
233
+ Kenton Lee and
234
+ Kristina Toutanova},
235
+ title = {{BERT:} Pre-training of Deep Bidirectional Transformers for Language
236
+ Understanding},
237
+ journal = {CoRR},
238
+ volume = {abs/1810.04805},
239
+ year = {2018},
240
+ url = {http://arxiv.org/abs/1810.04805},
241
+ archivePrefix = {arXiv},
242
+ eprint = {1810.04805},
243
+ timestamp = {Tue, 30 Oct 2018 20:39:56 +0100},
244
+ biburl = {https://dblp.org/rec/journals/corr/abs-1810-04805.bib},
245
+ bibsource = {dblp computer science bibliography, https://dblp.org}
246
+ }
247
+ ```
248
+
249
+ <a href="https://huggingface.co/exbert/?model=bert-base-uncased">
250
+ <img width="300px" src="https://cdn-media.huggingface.co/exbert/button.png">
251
+ </a>
data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594/config.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "BertForMaskedLM"
4
+ ],
5
+ "attention_probs_dropout_prob": 0.1,
6
+ "gradient_checkpointing": false,
7
+ "hidden_act": "gelu",
8
+ "hidden_dropout_prob": 0.1,
9
+ "hidden_size": 768,
10
+ "initializer_range": 0.02,
11
+ "intermediate_size": 3072,
12
+ "layer_norm_eps": 1e-12,
13
+ "max_position_embeddings": 512,
14
+ "model_type": "bert",
15
+ "num_attention_heads": 12,
16
+ "num_hidden_layers": 12,
17
+ "pad_token_id": 0,
18
+ "position_embedding_type": "absolute",
19
+ "transformers_version": "4.6.0.dev0",
20
+ "type_vocab_size": 2,
21
+ "use_cache": true,
22
+ "vocab_size": 30522
23
+ }
data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594/coreml/fill-mask/float32_model.mlpackage/Manifest.json ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "fileFormatVersion": "1.0.0",
3
+ "itemInfoEntries": {
4
+ "9D749A46-ADA0-43CA-B5C2-8E722B91F41E": {
5
+ "author": "com.apple.CoreML",
6
+ "description": "CoreML Model Specification",
7
+ "name": "model.mlmodel",
8
+ "path": "com.apple.CoreML/model.mlmodel"
9
+ },
10
+ "D545B13F-2D5E-4CFB-BFF1-C10E9EFD70DA": {
11
+ "author": "com.apple.CoreML",
12
+ "description": "CoreML Model Weights",
13
+ "name": "weights",
14
+ "path": "com.apple.CoreML/weights"
15
+ }
16
+ },
17
+ "rootModelIdentifier": "9D749A46-ADA0-43CA-B5C2-8E722B91F41E"
18
+ }
data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594/tokenizer_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"do_lower_case": true, "model_max_length": 512}