Add files using upload-large-folder tool
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- artifacts/checkpoint_lock.json +877 -0
- artifacts/sources.yaml +24 -0
- checkpoints/quarantine/20260825T101647Z/official/rdlm/lm1b/LM1B/config.yaml +75 -0
- patches/langflow/README.md +40 -0
- results/conditional/metrics/lm1b/duo_dcd/steps_4/metrics.json +95 -0
- results/conditional/metrics/lm1b/duo_dcd/steps_8/metrics.json +95 -0
- results/conditional/metrics/owt/duo/steps_16/metrics.json +91 -0
- results/conditional/metrics/owt/duo/steps_8/metrics.json +91 -0
- results/conditional/metrics/owt/duo_dcd/steps_32/metrics.json +91 -0
- results/conditional/metrics/owt/duo_di4c/steps_16/metrics.json +91 -0
- results/conditional/metrics/owt/duo_di4c/steps_2/metrics.json +91 -0
- results/conditional/metrics/owt/duo_di4c/steps_32/metrics.json +91 -0
- results/conditional/metrics/owt/flm/steps_16/metrics.json +91 -0
- results/conditional/metrics/owt/flm/steps_8/metrics.json +91 -0
- results/conditional/metrics/owt/fmlm/steps_1/metrics.json +91 -0
- results/conditional/metrics/owt/fmlm/steps_4/metrics.json +91 -0
- results/conditional/metrics/owt/langflow/steps_1024/metrics.json +91 -0
- results/conditional/metrics/owt/langflow/steps_32/metrics.json +91 -0
- results/conditional/metrics/owt/langflow/steps_4/metrics.json +91 -0
- results/conditional/metrics/owt/langflow/steps_8/metrics.json +91 -0
- results/conditional/metrics/owt/mdlm/steps_1/metrics.json +91 -0
- results/conditional/metrics/owt/mdlm/steps_16/metrics.json +91 -0
- results/conditional/metrics/owt/mdlm/steps_8/metrics.json +91 -0
- results/conditional/metrics/owt/mdlm_di4c/steps_1/metrics.json +91 -0
- results/conditional/metrics/owt/mdlm_di4c/steps_2/metrics.json +91 -0
- results/conditional/metrics/owt/mdlm_sdtt/steps_1/metrics.json +91 -0
- results/conditional/metrics/owt/mdlm_sdtt/steps_2/metrics.json +91 -0
- results/conditional/metrics/owt/mdlm_sdtt/steps_32/metrics.json +91 -0
- results/conditional/metrics/owt/mdlm_sdtt/steps_8/metrics.json +91 -0
- results/conditional/samples/lm1b/duo/steps_1/stdout.log +173 -0
- results/conditional/samples/lm1b/duo/steps_1/upstream_hydra/config_tree.txt +169 -0
- results/conditional/samples/lm1b/duo/steps_1024/request.json +108 -0
- results/conditional/samples/lm1b/duo/steps_16/conversion_metadata.json +24 -0
- results/conditional/samples/lm1b/duo/steps_16/samples.jsonl +0 -0
- results/conditional/samples/lm1b/duo/steps_16/stderr.log +5 -0
- results/conditional/samples/lm1b/duo/steps_16/upstream_hydra/config_tree.txt +169 -0
- results/conditional/samples/lm1b/duo/steps_2/upstream_hydra/dlb_pinned_upstream_main.log +8 -0
- results/conditional/samples/lm1b/duo/steps_2/upstream_samples.json +7 -0
- results/conditional/samples/lm1b/duo/steps_32/request.json +108 -0
- results/conditional/samples/lm1b/duo/steps_32/samples.jsonl +0 -0
- results/conditional/samples/lm1b/duo/steps_32/upstream_hydra/dlb_pinned_upstream_main.log +8 -0
- results/conditional/samples/lm1b/duo/steps_32/upstream_samples.json +7 -0
- results/conditional/samples/lm1b/duo/steps_4/upstream_hydra/dlb_pinned_upstream_main.log +8 -0
- results/conditional/samples/lm1b/duo/steps_8/run_metadata.json +1 -0
- results/conditional/samples/lm1b/duo/steps_8/stdout.log +173 -0
- results/conditional/samples/lm1b/duo/steps_8/upstream_hydra/.hydra/hydra.yaml +181 -0
- results/conditional/samples/lm1b/duo/steps_8/upstream_samples.json +7 -0
- results/conditional/samples/lm1b/duo_di4c/steps_1/samples.jsonl +0 -0
- results/conditional/samples/lm1b/duo_di4c/steps_1/upstream_token_ids.json +0 -0
- results/conditional/samples/lm1b/flm/steps_1024/upstream_token_ids.json +4 -0
artifacts/checkpoint_lock.json
ADDED
|
@@ -0,0 +1,877 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"created_at": "2026-08-25T10:28:11.243440Z",
|
| 3 |
+
"manifest_sha256": "ce8ff6a327d9f1470fef9a7ac66813ac94d4cc50268bc2998627498dc6f3a152",
|
| 4 |
+
"resources": {
|
| 5 |
+
"di4c_mdlm_owt_zenodo": {
|
| 6 |
+
"backend": "zenodo",
|
| 7 |
+
"destination": "checkpoints/official/mdlm_di4c/owt",
|
| 8 |
+
"files": [
|
| 9 |
+
{
|
| 10 |
+
"path": "checkpoints/official/mdlm_di4c/owt/sdtt7-di4c2.ckpt",
|
| 11 |
+
"sha256": "4af61572dc82fe012ce395a2e4e4be0585ea09076597ba1f7241ca196b97ecb0",
|
| 12 |
+
"size_bytes": 1673838666
|
| 13 |
+
},
|
| 14 |
+
{
|
| 15 |
+
"path": "checkpoints/official/mdlm_di4c/owt/sdtt7-di4c2.ckpt.partial",
|
| 16 |
+
"sha256": "346367e275eed9f9a23272cad80e7b4354147e366bfcc670e85d8fbc3f486b34",
|
| 17 |
+
"size_bytes": 20971520
|
| 18 |
+
}
|
| 19 |
+
],
|
| 20 |
+
"provenance": "official",
|
| 21 |
+
"quarantined": null,
|
| 22 |
+
"status": "downloaded",
|
| 23 |
+
"teacher_family": "masked_mdlm"
|
| 24 |
+
},
|
| 25 |
+
"duo_dcd_owt_hf": {
|
| 26 |
+
"backend": "huggingface",
|
| 27 |
+
"destination": "checkpoints/official/duo_dcd/owt",
|
| 28 |
+
"files": [
|
| 29 |
+
{
|
| 30 |
+
"path": "checkpoints/official/duo_dcd/owt/.cache/huggingface/.gitignore",
|
| 31 |
+
"sha256": "684888c0ebb17f374298b65ee2807526c066094c701bcc7ebbe1c1095f494fc1",
|
| 32 |
+
"size_bytes": 1
|
| 33 |
+
},
|
| 34 |
+
{
|
| 35 |
+
"path": "checkpoints/official/duo_dcd/owt/.cache/huggingface/download/README.md.lock",
|
| 36 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 37 |
+
"size_bytes": 0
|
| 38 |
+
},
|
| 39 |
+
{
|
| 40 |
+
"path": "checkpoints/official/duo_dcd/owt/.cache/huggingface/download/README.md.metadata",
|
| 41 |
+
"sha256": "69088cfe187e6a210b737bacf9423071e8251bc4780a415a5fff9bee49c37476",
|
| 42 |
+
"size_bytes": 101
|
| 43 |
+
},
|
| 44 |
+
{
|
| 45 |
+
"path": "checkpoints/official/duo_dcd/owt/.cache/huggingface/download/config.json.lock",
|
| 46 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 47 |
+
"size_bytes": 0
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"path": "checkpoints/official/duo_dcd/owt/.cache/huggingface/download/config.json.metadata",
|
| 51 |
+
"sha256": "60513270388c2e843bacacc9e8c75f28c7d28bc7d682c4fab962f0e724aa3a34",
|
| 52 |
+
"size_bytes": 100
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"path": "checkpoints/official/duo_dcd/owt/.cache/huggingface/download/config.py.lock",
|
| 56 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 57 |
+
"size_bytes": 0
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"path": "checkpoints/official/duo_dcd/owt/.cache/huggingface/download/config.py.metadata",
|
| 61 |
+
"sha256": "0c6ab8f7d336df46629f133d599f65d298cae80be26f0a52ed66a17602ca0a6f",
|
| 62 |
+
"size_bytes": 100
|
| 63 |
+
},
|
| 64 |
+
{
|
| 65 |
+
"path": "checkpoints/official/duo_dcd/owt/.cache/huggingface/download/model.py.lock",
|
| 66 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 67 |
+
"size_bytes": 0
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"path": "checkpoints/official/duo_dcd/owt/.cache/huggingface/download/model.py.metadata",
|
| 71 |
+
"sha256": "3d360fddcfa2d397b800ce33a50ba61ef6e8d1cde4d45eebbbb8e73390e77a8e",
|
| 72 |
+
"size_bytes": 101
|
| 73 |
+
},
|
| 74 |
+
{
|
| 75 |
+
"path": "checkpoints/official/duo_dcd/owt/.cache/huggingface/download/model.safetensors.lock",
|
| 76 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 77 |
+
"size_bytes": 0
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"path": "checkpoints/official/duo_dcd/owt/.cache/huggingface/download/model.safetensors.metadata",
|
| 81 |
+
"sha256": "f242d0122dda0de169d5c47b3a7ec02d728f48116b628bd37f4e75dbf0e8c4af",
|
| 82 |
+
"size_bytes": 124
|
| 83 |
+
},
|
| 84 |
+
{
|
| 85 |
+
"path": "checkpoints/official/duo_dcd/owt/README.md",
|
| 86 |
+
"sha256": "710bb04cdc48ed112a3bb83472bb744162e6c8e07b6850c4ed27a317e0255376",
|
| 87 |
+
"size_bytes": 1766
|
| 88 |
+
},
|
| 89 |
+
{
|
| 90 |
+
"path": "checkpoints/official/duo_dcd/owt/config.json",
|
| 91 |
+
"sha256": "5c4f85638cb277355344c47c31b43b3c47fd4d9e6f932e30b56ff8f4f76317e1",
|
| 92 |
+
"size_bytes": 471
|
| 93 |
+
},
|
| 94 |
+
{
|
| 95 |
+
"path": "checkpoints/official/duo_dcd/owt/config.py",
|
| 96 |
+
"sha256": "1e87050b1728e4862c10fd6740edfb6c0fbab7beebfc854d7a445cbe4cf1dee1",
|
| 97 |
+
"size_bytes": 718
|
| 98 |
+
},
|
| 99 |
+
{
|
| 100 |
+
"path": "checkpoints/official/duo_dcd/owt/model.py",
|
| 101 |
+
"sha256": "dcfef1c27da31fce4d6a7449a2759461070f5ed825b5e13c38361b228dade58d",
|
| 102 |
+
"size_bytes": 18736
|
| 103 |
+
},
|
| 104 |
+
{
|
| 105 |
+
"path": "checkpoints/official/duo_dcd/owt/model.safetensors",
|
| 106 |
+
"sha256": "c4b66c48780483f7079e47b1a56e6245a12d83057533455f447e822ef20fe93f",
|
| 107 |
+
"size_bytes": 678522728
|
| 108 |
+
}
|
| 109 |
+
],
|
| 110 |
+
"provenance": "official",
|
| 111 |
+
"quarantined": null,
|
| 112 |
+
"status": "downloaded",
|
| 113 |
+
"teacher_family": "uniform_duo"
|
| 114 |
+
},
|
| 115 |
+
"duo_owt_hf": {
|
| 116 |
+
"backend": "huggingface",
|
| 117 |
+
"destination": "checkpoints/official/duo/owt",
|
| 118 |
+
"files": [
|
| 119 |
+
{
|
| 120 |
+
"path": "checkpoints/official/duo/owt/.cache/huggingface/.gitignore",
|
| 121 |
+
"sha256": "684888c0ebb17f374298b65ee2807526c066094c701bcc7ebbe1c1095f494fc1",
|
| 122 |
+
"size_bytes": 1
|
| 123 |
+
},
|
| 124 |
+
{
|
| 125 |
+
"path": "checkpoints/official/duo/owt/.cache/huggingface/download/README.md.lock",
|
| 126 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 127 |
+
"size_bytes": 0
|
| 128 |
+
},
|
| 129 |
+
{
|
| 130 |
+
"path": "checkpoints/official/duo/owt/.cache/huggingface/download/README.md.metadata",
|
| 131 |
+
"sha256": "8f08d2a1d12900b7b79708e5c8de144066003782457218e6c335c7d0ce336254",
|
| 132 |
+
"size_bytes": 101
|
| 133 |
+
},
|
| 134 |
+
{
|
| 135 |
+
"path": "checkpoints/official/duo/owt/.cache/huggingface/download/config.json.lock",
|
| 136 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 137 |
+
"size_bytes": 0
|
| 138 |
+
},
|
| 139 |
+
{
|
| 140 |
+
"path": "checkpoints/official/duo/owt/.cache/huggingface/download/config.json.metadata",
|
| 141 |
+
"sha256": "2516bd22b6f08906ebb56444d04b196c6bbec29102772bcdb91487ddccdb7855",
|
| 142 |
+
"size_bytes": 101
|
| 143 |
+
},
|
| 144 |
+
{
|
| 145 |
+
"path": "checkpoints/official/duo/owt/.cache/huggingface/download/config.py.lock",
|
| 146 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 147 |
+
"size_bytes": 0
|
| 148 |
+
},
|
| 149 |
+
{
|
| 150 |
+
"path": "checkpoints/official/duo/owt/.cache/huggingface/download/config.py.metadata",
|
| 151 |
+
"sha256": "e0a868cd4d51d0050dbbf0106309ff6fac58eadfdb4a0bfae7670a6bb745a210",
|
| 152 |
+
"size_bytes": 101
|
| 153 |
+
},
|
| 154 |
+
{
|
| 155 |
+
"path": "checkpoints/official/duo/owt/.cache/huggingface/download/model.py.lock",
|
| 156 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 157 |
+
"size_bytes": 0
|
| 158 |
+
},
|
| 159 |
+
{
|
| 160 |
+
"path": "checkpoints/official/duo/owt/.cache/huggingface/download/model.py.metadata",
|
| 161 |
+
"sha256": "d6e6a806ce46536f88700e4049a19037db2c4f6ede730e8b410f558acaee6945",
|
| 162 |
+
"size_bytes": 101
|
| 163 |
+
},
|
| 164 |
+
{
|
| 165 |
+
"path": "checkpoints/official/duo/owt/.cache/huggingface/download/model.safetensors.lock",
|
| 166 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 167 |
+
"size_bytes": 0
|
| 168 |
+
},
|
| 169 |
+
{
|
| 170 |
+
"path": "checkpoints/official/duo/owt/.cache/huggingface/download/model.safetensors.metadata",
|
| 171 |
+
"sha256": "8c3fb4eed0a3a4701cb5e5b5c0b2bda93f90595220ddbd3600b6be78236d32ed",
|
| 172 |
+
"size_bytes": 125
|
| 173 |
+
},
|
| 174 |
+
{
|
| 175 |
+
"path": "checkpoints/official/duo/owt/README.md",
|
| 176 |
+
"sha256": "26b42ac8aa7f015c08ee0106b7b580ead5cbd7f36abf6a0fcb1504badcb15dd2",
|
| 177 |
+
"size_bytes": 1781
|
| 178 |
+
},
|
| 179 |
+
{
|
| 180 |
+
"path": "checkpoints/official/duo/owt/config.json",
|
| 181 |
+
"sha256": "d133d602759160b42108668782c2ec154ddbb066873359cde89b42554af61c20",
|
| 182 |
+
"size_bytes": 461
|
| 183 |
+
},
|
| 184 |
+
{
|
| 185 |
+
"path": "checkpoints/official/duo/owt/config.py",
|
| 186 |
+
"sha256": "1e87050b1728e4862c10fd6740edfb6c0fbab7beebfc854d7a445cbe4cf1dee1",
|
| 187 |
+
"size_bytes": 718
|
| 188 |
+
},
|
| 189 |
+
{
|
| 190 |
+
"path": "checkpoints/official/duo/owt/model.py",
|
| 191 |
+
"sha256": "dcfef1c27da31fce4d6a7449a2759461070f5ed825b5e13c38361b228dade58d",
|
| 192 |
+
"size_bytes": 18736
|
| 193 |
+
},
|
| 194 |
+
{
|
| 195 |
+
"path": "checkpoints/official/duo/owt/model.safetensors",
|
| 196 |
+
"sha256": "c9e98629579f5dc7c7fcf15cd10c66f585fd6cfa31a9ce4ac09255bc84ec9ad3",
|
| 197 |
+
"size_bytes": 678522728
|
| 198 |
+
}
|
| 199 |
+
],
|
| 200 |
+
"provenance": "official",
|
| 201 |
+
"quarantined": null,
|
| 202 |
+
"status": "downloaded",
|
| 203 |
+
"teacher_family": "uniform_duo"
|
| 204 |
+
},
|
| 205 |
+
"duo_raw_owt_drive": {
|
| 206 |
+
"backend": "gdrive",
|
| 207 |
+
"destination": "checkpoints/official/duo/raw_owt",
|
| 208 |
+
"files": [
|
| 209 |
+
{
|
| 210 |
+
"path": "checkpoints/official/duo/raw_owt/duo-distilled.ckpt",
|
| 211 |
+
"sha256": "23dba8a6cc97500f3798cb35484c8219a6c8a611b38f9dc92e2ae23946f593a9",
|
| 212 |
+
"size_bytes": 2715755352
|
| 213 |
+
},
|
| 214 |
+
{
|
| 215 |
+
"path": "checkpoints/official/duo/raw_owt/duo.ckpt",
|
| 216 |
+
"sha256": "ea8edb134515121c8d998b43c08517a53b1d6797aed9d80117f7322d10316c4c",
|
| 217 |
+
"size_bytes": 2715755416
|
| 218 |
+
}
|
| 219 |
+
],
|
| 220 |
+
"provenance": "official",
|
| 221 |
+
"quarantined": null,
|
| 222 |
+
"status": "downloaded",
|
| 223 |
+
"teacher_family": "uniform_duo"
|
| 224 |
+
},
|
| 225 |
+
"flm_lm1b_hf": {
|
| 226 |
+
"backend": "huggingface",
|
| 227 |
+
"destination": "checkpoints/official/flm/lm1b",
|
| 228 |
+
"files": [
|
| 229 |
+
{
|
| 230 |
+
"path": "checkpoints/official/flm/lm1b/.cache/huggingface/.gitignore",
|
| 231 |
+
"sha256": "684888c0ebb17f374298b65ee2807526c066094c701bcc7ebbe1c1095f494fc1",
|
| 232 |
+
"size_bytes": 1
|
| 233 |
+
},
|
| 234 |
+
{
|
| 235 |
+
"path": "checkpoints/official/flm/lm1b/.cache/huggingface/download/README.md.lock",
|
| 236 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 237 |
+
"size_bytes": 0
|
| 238 |
+
},
|
| 239 |
+
{
|
| 240 |
+
"path": "checkpoints/official/flm/lm1b/.cache/huggingface/download/README.md.metadata",
|
| 241 |
+
"sha256": "b8ecc3c2f24041f15d5e0204335bade35298359c4f781ae83257f1407f25019b",
|
| 242 |
+
"size_bytes": 100
|
| 243 |
+
},
|
| 244 |
+
{
|
| 245 |
+
"path": "checkpoints/official/flm/lm1b/.cache/huggingface/download/config.json.lock",
|
| 246 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 247 |
+
"size_bytes": 0
|
| 248 |
+
},
|
| 249 |
+
{
|
| 250 |
+
"path": "checkpoints/official/flm/lm1b/.cache/huggingface/download/config.json.metadata",
|
| 251 |
+
"sha256": "81fa8fef6fc09f66b42de588c25e3fde3c813120baaff23b9db86d98ac6ead4c",
|
| 252 |
+
"size_bytes": 101
|
| 253 |
+
},
|
| 254 |
+
{
|
| 255 |
+
"path": "checkpoints/official/flm/lm1b/.cache/huggingface/download/model.safetensors.lock",
|
| 256 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 257 |
+
"size_bytes": 0
|
| 258 |
+
},
|
| 259 |
+
{
|
| 260 |
+
"path": "checkpoints/official/flm/lm1b/.cache/huggingface/download/model.safetensors.metadata",
|
| 261 |
+
"sha256": "aaa2f256dfdd93d8898253be702e14c0ff08c28d6a3b9e5be12becda7b9c35dc",
|
| 262 |
+
"size_bytes": 125
|
| 263 |
+
},
|
| 264 |
+
{
|
| 265 |
+
"path": "checkpoints/official/flm/lm1b/README.md",
|
| 266 |
+
"sha256": "d14e8ef58d4c3696def2121665a2a25f00cc33fc06f1cbf8c8c50c663d7ab058",
|
| 267 |
+
"size_bytes": 1097
|
| 268 |
+
},
|
| 269 |
+
{
|
| 270 |
+
"path": "checkpoints/official/flm/lm1b/config.json",
|
| 271 |
+
"sha256": "460c1308f1da60c319d0cdbb2c9ac40b6b3989031dcd51a6638cc52d6195ca69",
|
| 272 |
+
"size_bytes": 5790
|
| 273 |
+
},
|
| 274 |
+
{
|
| 275 |
+
"path": "checkpoints/official/flm/lm1b/model.safetensors",
|
| 276 |
+
"sha256": "ef02806ef90ab9aca274371c3b644193f0af1757504acdd11ac1c3f89359f55c",
|
| 277 |
+
"size_bytes": 557184584
|
| 278 |
+
}
|
| 279 |
+
],
|
| 280 |
+
"provenance": "official",
|
| 281 |
+
"quarantined": null,
|
| 282 |
+
"status": "downloaded",
|
| 283 |
+
"teacher_family": "continuous_flm"
|
| 284 |
+
},
|
| 285 |
+
"flm_lm1b_reproductions": {
|
| 286 |
+
"backend": "gdrive",
|
| 287 |
+
"destination": "checkpoints/reference_reproduction/flm_baselines/lm1b",
|
| 288 |
+
"files": [
|
| 289 |
+
{
|
| 290 |
+
"path": "checkpoints/reference_reproduction/flm_baselines/lm1b/lm1b_CANDI.ckpt",
|
| 291 |
+
"sha256": "33b4359227831870e01225673a6cc50b8e38b6523d729fc2d9c5f1eac916f270",
|
| 292 |
+
"size_bytes": 2229786789
|
| 293 |
+
},
|
| 294 |
+
{
|
| 295 |
+
"path": "checkpoints/reference_reproduction/flm_baselines/lm1b/lm1b_Duo.ckpt",
|
| 296 |
+
"sha256": "b11ee9a4cc533435adb9e1149cff74aec0872a9df2a459b4ab58f1867f6cddfb",
|
| 297 |
+
"size_bytes": 2229753893
|
| 298 |
+
},
|
| 299 |
+
{
|
| 300 |
+
"path": "checkpoints/reference_reproduction/flm_baselines/lm1b/lm1b_MDLM_.ckpt",
|
| 301 |
+
"sha256": "d42739e5a99bbd7c0f604dff7fe4c9ab12348b475e30f1d807e70ef32249e514",
|
| 302 |
+
"size_bytes": 2229762853
|
| 303 |
+
}
|
| 304 |
+
],
|
| 305 |
+
"provenance": "reference_reproduction",
|
| 306 |
+
"quarantined": null,
|
| 307 |
+
"status": "downloaded",
|
| 308 |
+
"teacher_family": "mixed_baselines"
|
| 309 |
+
},
|
| 310 |
+
"flm_owt_hf": {
|
| 311 |
+
"backend": "huggingface",
|
| 312 |
+
"destination": "checkpoints/official/flm/owt",
|
| 313 |
+
"files": [
|
| 314 |
+
{
|
| 315 |
+
"path": "checkpoints/official/flm/owt/.cache/huggingface/.gitignore",
|
| 316 |
+
"sha256": "684888c0ebb17f374298b65ee2807526c066094c701bcc7ebbe1c1095f494fc1",
|
| 317 |
+
"size_bytes": 1
|
| 318 |
+
},
|
| 319 |
+
{
|
| 320 |
+
"path": "checkpoints/official/flm/owt/.cache/huggingface/download/README.md.lock",
|
| 321 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 322 |
+
"size_bytes": 0
|
| 323 |
+
},
|
| 324 |
+
{
|
| 325 |
+
"path": "checkpoints/official/flm/owt/.cache/huggingface/download/README.md.metadata",
|
| 326 |
+
"sha256": "0a840440ceaece9a82eaa089339a8e4fb28058f709c563bdc9e05fa51b6cd03d",
|
| 327 |
+
"size_bytes": 100
|
| 328 |
+
},
|
| 329 |
+
{
|
| 330 |
+
"path": "checkpoints/official/flm/owt/.cache/huggingface/download/config.json.lock",
|
| 331 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 332 |
+
"size_bytes": 0
|
| 333 |
+
},
|
| 334 |
+
{
|
| 335 |
+
"path": "checkpoints/official/flm/owt/.cache/huggingface/download/config.json.metadata",
|
| 336 |
+
"sha256": "29ed765d2819d93c4b246e7b3e04918a0b484ea5e67e813f5b28fb58f77f7ef8",
|
| 337 |
+
"size_bytes": 101
|
| 338 |
+
},
|
| 339 |
+
{
|
| 340 |
+
"path": "checkpoints/official/flm/owt/.cache/huggingface/download/model.safetensors.lock",
|
| 341 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 342 |
+
"size_bytes": 0
|
| 343 |
+
},
|
| 344 |
+
{
|
| 345 |
+
"path": "checkpoints/official/flm/owt/.cache/huggingface/download/model.safetensors.metadata",
|
| 346 |
+
"sha256": "0f13bf670e919977aff3018fb83ad83f8f11cd06d08504d30157c1b78a6640a4",
|
| 347 |
+
"size_bytes": 125
|
| 348 |
+
},
|
| 349 |
+
{
|
| 350 |
+
"path": "checkpoints/official/flm/owt/README.md",
|
| 351 |
+
"sha256": "ffa98bdad8de633065cd636130027026959457f8dd2b1c1de519c8f1886f39f6",
|
| 352 |
+
"size_bytes": 1124
|
| 353 |
+
},
|
| 354 |
+
{
|
| 355 |
+
"path": "checkpoints/official/flm/owt/config.json",
|
| 356 |
+
"sha256": "a1a2569dbae7d20fb84ab150f0cce8ebf8927067eaad0c2cdd5cb64995115301",
|
| 357 |
+
"size_bytes": 5832
|
| 358 |
+
},
|
| 359 |
+
{
|
| 360 |
+
"path": "checkpoints/official/flm/owt/model.safetensors",
|
| 361 |
+
"sha256": "5715258cb3aac9cc8416607a3af5ca4ce0e86db1ca0e5b67ef038d837d95233d",
|
| 362 |
+
"size_bytes": 678521512
|
| 363 |
+
}
|
| 364 |
+
],
|
| 365 |
+
"provenance": "official",
|
| 366 |
+
"quarantined": null,
|
| 367 |
+
"status": "downloaded",
|
| 368 |
+
"teacher_family": "continuous_flm"
|
| 369 |
+
},
|
| 370 |
+
"fmlm_lm1b_hf": {
|
| 371 |
+
"backend": "huggingface",
|
| 372 |
+
"destination": "checkpoints/official/fmlm/lm1b",
|
| 373 |
+
"files": [
|
| 374 |
+
{
|
| 375 |
+
"path": "checkpoints/official/fmlm/lm1b/.cache/huggingface/.gitignore",
|
| 376 |
+
"sha256": "684888c0ebb17f374298b65ee2807526c066094c701bcc7ebbe1c1095f494fc1",
|
| 377 |
+
"size_bytes": 1
|
| 378 |
+
},
|
| 379 |
+
{
|
| 380 |
+
"path": "checkpoints/official/fmlm/lm1b/.cache/huggingface/download/README.md.lock",
|
| 381 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 382 |
+
"size_bytes": 0
|
| 383 |
+
},
|
| 384 |
+
{
|
| 385 |
+
"path": "checkpoints/official/fmlm/lm1b/.cache/huggingface/download/README.md.metadata",
|
| 386 |
+
"sha256": "98abfe72c542efe82809f4762447a66da117a08007f87cd3237a9f2e805fa8a6",
|
| 387 |
+
"size_bytes": 101
|
| 388 |
+
},
|
| 389 |
+
{
|
| 390 |
+
"path": "checkpoints/official/fmlm/lm1b/.cache/huggingface/download/config.json.lock",
|
| 391 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 392 |
+
"size_bytes": 0
|
| 393 |
+
},
|
| 394 |
+
{
|
| 395 |
+
"path": "checkpoints/official/fmlm/lm1b/.cache/huggingface/download/config.json.metadata",
|
| 396 |
+
"sha256": "a03bb6bb30d274bae2d90ec99b90630b31e41239e54787ed646973b85c60832c",
|
| 397 |
+
"size_bytes": 101
|
| 398 |
+
},
|
| 399 |
+
{
|
| 400 |
+
"path": "checkpoints/official/fmlm/lm1b/.cache/huggingface/download/model.safetensors.lock",
|
| 401 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 402 |
+
"size_bytes": 0
|
| 403 |
+
},
|
| 404 |
+
{
|
| 405 |
+
"path": "checkpoints/official/fmlm/lm1b/.cache/huggingface/download/model.safetensors.metadata",
|
| 406 |
+
"sha256": "108a2febf8352bcf426d06ffd007ec3ad6e64eed98cecad0557e5eeee7ebe4e2",
|
| 407 |
+
"size_bytes": 125
|
| 408 |
+
},
|
| 409 |
+
{
|
| 410 |
+
"path": "checkpoints/official/fmlm/lm1b/README.md",
|
| 411 |
+
"sha256": "2e25f8aea9629395ff0d2598a85d1a3aaa54a961d9226068c8587b71661077e0",
|
| 412 |
+
"size_bytes": 1166
|
| 413 |
+
},
|
| 414 |
+
{
|
| 415 |
+
"path": "checkpoints/official/fmlm/lm1b/config.json",
|
| 416 |
+
"sha256": "6329c0b9a02b1a990d4bda6f108389a292d841b45ae5c66aa7d9b7d0ecb195a2",
|
| 417 |
+
"size_bytes": 6136
|
| 418 |
+
},
|
| 419 |
+
{
|
| 420 |
+
"path": "checkpoints/official/fmlm/lm1b/model.safetensors",
|
| 421 |
+
"sha256": "574e5001d8ee6523b1b363cbd7055a3e3fe6d6f8dd7f796c4582548b9c728942",
|
| 422 |
+
"size_bytes": 557382616
|
| 423 |
+
}
|
| 424 |
+
],
|
| 425 |
+
"provenance": "official",
|
| 426 |
+
"quarantined": null,
|
| 427 |
+
"status": "downloaded",
|
| 428 |
+
"teacher_family": "continuous_flm"
|
| 429 |
+
},
|
| 430 |
+
"fmlm_owt_hf": {
|
| 431 |
+
"backend": "huggingface",
|
| 432 |
+
"destination": "checkpoints/official/fmlm/owt",
|
| 433 |
+
"files": [
|
| 434 |
+
{
|
| 435 |
+
"path": "checkpoints/official/fmlm/owt/.cache/huggingface/.gitignore",
|
| 436 |
+
"sha256": "684888c0ebb17f374298b65ee2807526c066094c701bcc7ebbe1c1095f494fc1",
|
| 437 |
+
"size_bytes": 1
|
| 438 |
+
},
|
| 439 |
+
{
|
| 440 |
+
"path": "checkpoints/official/fmlm/owt/.cache/huggingface/download/README.md.lock",
|
| 441 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 442 |
+
"size_bytes": 0
|
| 443 |
+
},
|
| 444 |
+
{
|
| 445 |
+
"path": "checkpoints/official/fmlm/owt/.cache/huggingface/download/README.md.metadata",
|
| 446 |
+
"sha256": "d61ca988a71720d70d94c0c5287e071e8376d40d23a678cfc0127cbdc41db2eb",
|
| 447 |
+
"size_bytes": 101
|
| 448 |
+
},
|
| 449 |
+
{
|
| 450 |
+
"path": "checkpoints/official/fmlm/owt/.cache/huggingface/download/config.json.lock",
|
| 451 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 452 |
+
"size_bytes": 0
|
| 453 |
+
},
|
| 454 |
+
{
|
| 455 |
+
"path": "checkpoints/official/fmlm/owt/.cache/huggingface/download/config.json.metadata",
|
| 456 |
+
"sha256": "a275470a30f4e98421a72441d4d5c833616ddcda9af9dff2f0bd0d3566b2610c",
|
| 457 |
+
"size_bytes": 100
|
| 458 |
+
},
|
| 459 |
+
{
|
| 460 |
+
"path": "checkpoints/official/fmlm/owt/.cache/huggingface/download/model.safetensors.lock",
|
| 461 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 462 |
+
"size_bytes": 0
|
| 463 |
+
},
|
| 464 |
+
{
|
| 465 |
+
"path": "checkpoints/official/fmlm/owt/.cache/huggingface/download/model.safetensors.metadata",
|
| 466 |
+
"sha256": "536655b1a752333996466b0ec6f14c72613dd2ba974b4761f71c336918dd048a",
|
| 467 |
+
"size_bytes": 125
|
| 468 |
+
},
|
| 469 |
+
{
|
| 470 |
+
"path": "checkpoints/official/fmlm/owt/README.md",
|
| 471 |
+
"sha256": "8fbfb11f02e2ff7a3126ab393346113841704163e01813dcb5ba3706b9d2ab1d",
|
| 472 |
+
"size_bytes": 1191
|
| 473 |
+
},
|
| 474 |
+
{
|
| 475 |
+
"path": "checkpoints/official/fmlm/owt/config.json",
|
| 476 |
+
"sha256": "9a056d5dbfbddfbd5c0554d8189548a9c3b349a0dbd432906e29657a7979299c",
|
| 477 |
+
"size_bytes": 6276
|
| 478 |
+
},
|
| 479 |
+
{
|
| 480 |
+
"path": "checkpoints/official/fmlm/owt/model.safetensors",
|
| 481 |
+
"sha256": "1c44219dc2f240556dd96e4631ec8465fea942dbc5eb85848b204eefc4dac48d",
|
| 482 |
+
"size_bytes": 678719544
|
| 483 |
+
}
|
| 484 |
+
],
|
| 485 |
+
"provenance": "official",
|
| 486 |
+
"quarantined": null,
|
| 487 |
+
"status": "downloaded",
|
| 488 |
+
"teacher_family": "continuous_flm"
|
| 489 |
+
},
|
| 490 |
+
"langflow_lm1b_hf": {
|
| 491 |
+
"backend": "huggingface",
|
| 492 |
+
"destination": "checkpoints/official/langflow/lm1b",
|
| 493 |
+
"files": [
|
| 494 |
+
{
|
| 495 |
+
"path": "checkpoints/official/langflow/lm1b/.cache/huggingface/.gitignore",
|
| 496 |
+
"sha256": "684888c0ebb17f374298b65ee2807526c066094c701bcc7ebbe1c1095f494fc1",
|
| 497 |
+
"size_bytes": 1
|
| 498 |
+
},
|
| 499 |
+
{
|
| 500 |
+
"path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/README.md.lock",
|
| 501 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 502 |
+
"size_bytes": 0
|
| 503 |
+
},
|
| 504 |
+
{
|
| 505 |
+
"path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/README.md.metadata",
|
| 506 |
+
"sha256": "6aafe420fb31b6aeff03e6ec6986f283dced5303fbfb638cd72f3984fa3f467d",
|
| 507 |
+
"size_bytes": 101
|
| 508 |
+
},
|
| 509 |
+
{
|
| 510 |
+
"path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/__init__.py.lock",
|
| 511 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 512 |
+
"size_bytes": 0
|
| 513 |
+
},
|
| 514 |
+
{
|
| 515 |
+
"path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/__init__.py.metadata",
|
| 516 |
+
"sha256": "8ab7a8f3350468df7b945cce135c216e54aca2d021966a15deb5e3e91f2eabcb",
|
| 517 |
+
"size_bytes": 101
|
| 518 |
+
},
|
| 519 |
+
{
|
| 520 |
+
"path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/config.json.lock",
|
| 521 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 522 |
+
"size_bytes": 0
|
| 523 |
+
},
|
| 524 |
+
{
|
| 525 |
+
"path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/config.json.metadata",
|
| 526 |
+
"sha256": "2107eec801ce5dea0a35d9636ab2e5d35851b31cb5637448b31936b2a34c63e7",
|
| 527 |
+
"size_bytes": 99
|
| 528 |
+
},
|
| 529 |
+
{
|
| 530 |
+
"path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/config.py.lock",
|
| 531 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 532 |
+
"size_bytes": 0
|
| 533 |
+
},
|
| 534 |
+
{
|
| 535 |
+
"path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/config.py.metadata",
|
| 536 |
+
"sha256": "9a1048500ef3132bc472b84395675f086cd01e741d9d5c0e8fefad7352f96bff",
|
| 537 |
+
"size_bytes": 101
|
| 538 |
+
},
|
| 539 |
+
{
|
| 540 |
+
"path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/model.py.lock",
|
| 541 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 542 |
+
"size_bytes": 0
|
| 543 |
+
},
|
| 544 |
+
{
|
| 545 |
+
"path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/model.py.metadata",
|
| 546 |
+
"sha256": "2abf4413c8e40d458814f6efa0b6e486ff5f5fe3b2ec216e21c5e4e1af8b78ad",
|
| 547 |
+
"size_bytes": 101
|
| 548 |
+
},
|
| 549 |
+
{
|
| 550 |
+
"path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/model.safetensors.lock",
|
| 551 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 552 |
+
"size_bytes": 0
|
| 553 |
+
},
|
| 554 |
+
{
|
| 555 |
+
"path": "checkpoints/official/langflow/lm1b/.cache/huggingface/download/model.safetensors.metadata",
|
| 556 |
+
"sha256": "935d3ef257d043f3b21d02a8b78e2fabf69b7d3fa6a6af9032cbf177c640041f",
|
| 557 |
+
"size_bytes": 125
|
| 558 |
+
},
|
| 559 |
+
{
|
| 560 |
+
"path": "checkpoints/official/langflow/lm1b/README.md",
|
| 561 |
+
"sha256": "8bfdf3f492c7fac94f429c4269886dccad8c8e56b42caab004f93b785baf5d10",
|
| 562 |
+
"size_bytes": 1796
|
| 563 |
+
},
|
| 564 |
+
{
|
| 565 |
+
"path": "checkpoints/official/langflow/lm1b/__init__.py",
|
| 566 |
+
"sha256": "f5f0e406570ddc66a457d3b59d74b50a15eb3444c4543c2ff2d1b5e2277e3852",
|
| 567 |
+
"size_bytes": 154
|
| 568 |
+
},
|
| 569 |
+
{
|
| 570 |
+
"path": "checkpoints/official/langflow/lm1b/config.json",
|
| 571 |
+
"sha256": "d1b7a2324eb7b3791d26ef4c2e6058598d9af9ab3b9aed74f9467bf041594f97",
|
| 572 |
+
"size_bytes": 677
|
| 573 |
+
},
|
| 574 |
+
{
|
| 575 |
+
"path": "checkpoints/official/langflow/lm1b/config.py",
|
| 576 |
+
"sha256": "064d2d83613817a49d2ffc82258e4b3f61c71780eb6448b7d548321db1960998",
|
| 577 |
+
"size_bytes": 2290
|
| 578 |
+
},
|
| 579 |
+
{
|
| 580 |
+
"path": "checkpoints/official/langflow/lm1b/model.py",
|
| 581 |
+
"sha256": "c5617dc383b1c6554333c0d5e87a364829b920eac1e87435501318ba4684ee0e",
|
| 582 |
+
"size_bytes": 20493
|
| 583 |
+
},
|
| 584 |
+
{
|
| 585 |
+
"path": "checkpoints/official/langflow/lm1b/model.safetensors",
|
| 586 |
+
"sha256": "b111b5141379bde1bc8531278a2d144bc753cf3b100ed32c8a917f2ed0c025d2",
|
| 587 |
+
"size_bytes": 561904748
|
| 588 |
+
}
|
| 589 |
+
],
|
| 590 |
+
"provenance": "official",
|
| 591 |
+
"quarantined": null,
|
| 592 |
+
"status": "downloaded",
|
| 593 |
+
"teacher_family": "continuous_langflow"
|
| 594 |
+
},
|
| 595 |
+
"langflow_owt_hf": {
|
| 596 |
+
"backend": "huggingface",
|
| 597 |
+
"destination": "checkpoints/official/langflow/owt",
|
| 598 |
+
"files": [
|
| 599 |
+
{
|
| 600 |
+
"path": "checkpoints/official/langflow/owt/.cache/huggingface/.gitignore",
|
| 601 |
+
"sha256": "684888c0ebb17f374298b65ee2807526c066094c701bcc7ebbe1c1095f494fc1",
|
| 602 |
+
"size_bytes": 1
|
| 603 |
+
},
|
| 604 |
+
{
|
| 605 |
+
"path": "checkpoints/official/langflow/owt/.cache/huggingface/download/README.md.lock",
|
| 606 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 607 |
+
"size_bytes": 0
|
| 608 |
+
},
|
| 609 |
+
{
|
| 610 |
+
"path": "checkpoints/official/langflow/owt/.cache/huggingface/download/README.md.metadata",
|
| 611 |
+
"sha256": "86a9bf77b97a7ff6f3b36794a01ea05f8dd3d81c67c19f5a463c82e383c4234e",
|
| 612 |
+
"size_bytes": 100
|
| 613 |
+
},
|
| 614 |
+
{
|
| 615 |
+
"path": "checkpoints/official/langflow/owt/.cache/huggingface/download/__init__.py.lock",
|
| 616 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 617 |
+
"size_bytes": 0
|
| 618 |
+
},
|
| 619 |
+
{
|
| 620 |
+
"path": "checkpoints/official/langflow/owt/.cache/huggingface/download/__init__.py.metadata",
|
| 621 |
+
"sha256": "f4cc5a80d703145f12ad390c16464671e3671ed9dc45b7c391d520ea517a34c4",
|
| 622 |
+
"size_bytes": 101
|
| 623 |
+
},
|
| 624 |
+
{
|
| 625 |
+
"path": "checkpoints/official/langflow/owt/.cache/huggingface/download/config.json.lock",
|
| 626 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 627 |
+
"size_bytes": 0
|
| 628 |
+
},
|
| 629 |
+
{
|
| 630 |
+
"path": "checkpoints/official/langflow/owt/.cache/huggingface/download/config.json.metadata",
|
| 631 |
+
"sha256": "debd08642c7f0cabcd04ff11159eb3565e3e69c259be505f741b99e26b660827",
|
| 632 |
+
"size_bytes": 101
|
| 633 |
+
},
|
| 634 |
+
{
|
| 635 |
+
"path": "checkpoints/official/langflow/owt/.cache/huggingface/download/config.py.lock",
|
| 636 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 637 |
+
"size_bytes": 0
|
| 638 |
+
},
|
| 639 |
+
{
|
| 640 |
+
"path": "checkpoints/official/langflow/owt/.cache/huggingface/download/config.py.metadata",
|
| 641 |
+
"sha256": "3b9138ecf7cd4a0dd48860edb4d6f790eff327324b9fe14546d345c3e6ebe6d8",
|
| 642 |
+
"size_bytes": 101
|
| 643 |
+
},
|
| 644 |
+
{
|
| 645 |
+
"path": "checkpoints/official/langflow/owt/.cache/huggingface/download/model.py.lock",
|
| 646 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 647 |
+
"size_bytes": 0
|
| 648 |
+
},
|
| 649 |
+
{
|
| 650 |
+
"path": "checkpoints/official/langflow/owt/.cache/huggingface/download/model.py.metadata",
|
| 651 |
+
"sha256": "77888cde2d09f4ed7496c7d4cb41c7d3c0ce327aedc216815adbbece52ceb09e",
|
| 652 |
+
"size_bytes": 101
|
| 653 |
+
},
|
| 654 |
+
{
|
| 655 |
+
"path": "checkpoints/official/langflow/owt/.cache/huggingface/download/model.safetensors.lock",
|
| 656 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 657 |
+
"size_bytes": 0
|
| 658 |
+
},
|
| 659 |
+
{
|
| 660 |
+
"path": "checkpoints/official/langflow/owt/.cache/huggingface/download/model.safetensors.metadata",
|
| 661 |
+
"sha256": "6072993b2800f68eac153b7d0126f723d3c5f0674b92d2266d1e207c9d4a74af",
|
| 662 |
+
"size_bytes": 125
|
| 663 |
+
},
|
| 664 |
+
{
|
| 665 |
+
"path": "checkpoints/official/langflow/owt/README.md",
|
| 666 |
+
"sha256": "acdcfb3e3294ff78939f047f53e85795059430b75de267708c1101346aad44d3",
|
| 667 |
+
"size_bytes": 1784
|
| 668 |
+
},
|
| 669 |
+
{
|
| 670 |
+
"path": "checkpoints/official/langflow/owt/__init__.py",
|
| 671 |
+
"sha256": "f5f0e406570ddc66a457d3b59d74b50a15eb3444c4543c2ff2d1b5e2277e3852",
|
| 672 |
+
"size_bytes": 154
|
| 673 |
+
},
|
| 674 |
+
{
|
| 675 |
+
"path": "checkpoints/official/langflow/owt/config.json",
|
| 676 |
+
"sha256": "d16031e863349c8b41606d57f56e5ccde78f81cb1b45bf8e76361a51d146b3a1",
|
| 677 |
+
"size_bytes": 677
|
| 678 |
+
},
|
| 679 |
+
{
|
| 680 |
+
"path": "checkpoints/official/langflow/owt/config.py",
|
| 681 |
+
"sha256": "48f05e9a273aeea0716fe8f24792a2a2f01645fb01677724e9d27eaf6ba3f20b",
|
| 682 |
+
"size_bytes": 2291
|
| 683 |
+
},
|
| 684 |
+
{
|
| 685 |
+
"path": "checkpoints/official/langflow/owt/model.py",
|
| 686 |
+
"sha256": "c5617dc383b1c6554333c0d5e87a364829b920eac1e87435501318ba4684ee0e",
|
| 687 |
+
"size_bytes": 20493
|
| 688 |
+
},
|
| 689 |
+
{
|
| 690 |
+
"path": "checkpoints/official/langflow/owt/model.safetensors",
|
| 691 |
+
"sha256": "9dcc2573297d9f0ad91cb08d3469d141817d7bd3b11a2a7f332ff0f6a58e02a1",
|
| 692 |
+
"size_bytes": 683235528
|
| 693 |
+
}
|
| 694 |
+
],
|
| 695 |
+
"provenance": "official",
|
| 696 |
+
"quarantined": null,
|
| 697 |
+
"status": "downloaded",
|
| 698 |
+
"teacher_family": "continuous_langflow"
|
| 699 |
+
},
|
| 700 |
+
"mdlm_owt_hf": {
|
| 701 |
+
"backend": "huggingface",
|
| 702 |
+
"destination": "checkpoints/official/mdlm/owt",
|
| 703 |
+
"files": [
|
| 704 |
+
{
|
| 705 |
+
"path": "checkpoints/official/mdlm/owt/.cache/huggingface/.gitignore",
|
| 706 |
+
"sha256": "684888c0ebb17f374298b65ee2807526c066094c701bcc7ebbe1c1095f494fc1",
|
| 707 |
+
"size_bytes": 1
|
| 708 |
+
},
|
| 709 |
+
{
|
| 710 |
+
"path": "checkpoints/official/mdlm/owt/.cache/huggingface/download/README.md.lock",
|
| 711 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 712 |
+
"size_bytes": 0
|
| 713 |
+
},
|
| 714 |
+
{
|
| 715 |
+
"path": "checkpoints/official/mdlm/owt/.cache/huggingface/download/README.md.metadata",
|
| 716 |
+
"sha256": "1652f27ebe4a407b678206dabb4d7e3d72cac3e494058e8593823d427b37b1ef",
|
| 717 |
+
"size_bytes": 101
|
| 718 |
+
},
|
| 719 |
+
{
|
| 720 |
+
"path": "checkpoints/official/mdlm/owt/.cache/huggingface/download/config.json.lock",
|
| 721 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 722 |
+
"size_bytes": 0
|
| 723 |
+
},
|
| 724 |
+
{
|
| 725 |
+
"path": "checkpoints/official/mdlm/owt/.cache/huggingface/download/config.json.metadata",
|
| 726 |
+
"sha256": "b32e50f030dd5f4facb9c3199b5c2d0acc29e4aae2c2c8bf7e272b158061a38a",
|
| 727 |
+
"size_bytes": 101
|
| 728 |
+
},
|
| 729 |
+
{
|
| 730 |
+
"path": "checkpoints/official/mdlm/owt/.cache/huggingface/download/configuration_mdlm.py.lock",
|
| 731 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 732 |
+
"size_bytes": 0
|
| 733 |
+
},
|
| 734 |
+
{
|
| 735 |
+
"path": "checkpoints/official/mdlm/owt/.cache/huggingface/download/configuration_mdlm.py.metadata",
|
| 736 |
+
"sha256": "f910a02570f8366ec1979966c0be82ad6cb84f8f932ba196402993caf7438325",
|
| 737 |
+
"size_bytes": 101
|
| 738 |
+
},
|
| 739 |
+
{
|
| 740 |
+
"path": "checkpoints/official/mdlm/owt/.cache/huggingface/download/model.safetensors.lock",
|
| 741 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 742 |
+
"size_bytes": 0
|
| 743 |
+
},
|
| 744 |
+
{
|
| 745 |
+
"path": "checkpoints/official/mdlm/owt/.cache/huggingface/download/model.safetensors.metadata",
|
| 746 |
+
"sha256": "c82b7f61093a683c67cac34bcf343e0519ecfabdac0d61066116b8ebc1bfbb6c",
|
| 747 |
+
"size_bytes": 125
|
| 748 |
+
},
|
| 749 |
+
{
|
| 750 |
+
"path": "checkpoints/official/mdlm/owt/.cache/huggingface/download/modeling_mdlm.py.lock",
|
| 751 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 752 |
+
"size_bytes": 0
|
| 753 |
+
},
|
| 754 |
+
{
|
| 755 |
+
"path": "checkpoints/official/mdlm/owt/.cache/huggingface/download/modeling_mdlm.py.metadata",
|
| 756 |
+
"sha256": "303cd538bc78aa866c2f6ac8f6ec758f97809a395f484040b266ace1add184c6",
|
| 757 |
+
"size_bytes": 101
|
| 758 |
+
},
|
| 759 |
+
{
|
| 760 |
+
"path": "checkpoints/official/mdlm/owt/README.md",
|
| 761 |
+
"sha256": "393eb53038b34d16312006a3f0e7baacd3142e2a0514aea8f0d19241264f4ee1",
|
| 762 |
+
"size_bytes": 2431
|
| 763 |
+
},
|
| 764 |
+
{
|
| 765 |
+
"path": "checkpoints/official/mdlm/owt/config.json",
|
| 766 |
+
"sha256": "605429edee54b28322b94c3a018982ddb74bf94e3bf15f4ae7d5fbc73613138c",
|
| 767 |
+
"size_bytes": 489
|
| 768 |
+
},
|
| 769 |
+
{
|
| 770 |
+
"path": "checkpoints/official/mdlm/owt/configuration_mdlm.py",
|
| 771 |
+
"sha256": "0b1fe1d01c0ca375e1b24e06926a78e436b465b12448749cfbaa76318499cec2",
|
| 772 |
+
"size_bytes": 740
|
| 773 |
+
},
|
| 774 |
+
{
|
| 775 |
+
"path": "checkpoints/official/mdlm/owt/model.safetensors",
|
| 776 |
+
"sha256": "47149e73f7552f39ea9776dbe74d925d25237bcf2ed2e2ec03cdff9d51c82aa4",
|
| 777 |
+
"size_bytes": 678522728
|
| 778 |
+
},
|
| 779 |
+
{
|
| 780 |
+
"path": "checkpoints/official/mdlm/owt/modeling_mdlm.py",
|
| 781 |
+
"sha256": "d952a4de135a662abd25a2ce0f43fb67844e4cb38e034d5957e500957aa422a7",
|
| 782 |
+
"size_bytes": 13776
|
| 783 |
+
}
|
| 784 |
+
],
|
| 785 |
+
"provenance": "official",
|
| 786 |
+
"quarantined": null,
|
| 787 |
+
"status": "downloaded",
|
| 788 |
+
"teacher_family": "masked_mdlm"
|
| 789 |
+
},
|
| 790 |
+
"rdlm_lm1b_drive": {
|
| 791 |
+
"backend": "gdrive",
|
| 792 |
+
"destination": "checkpoints/official/rdlm/lm1b",
|
| 793 |
+
"files": [
|
| 794 |
+
{
|
| 795 |
+
"path": "checkpoints/official/rdlm/lm1b/LM1B/checkpoint.pth",
|
| 796 |
+
"sha256": "3319acacbd41ac4bf6bf42840dfd22cbebcd8e45d975d5225f114b6ee9cafd81",
|
| 797 |
+
"size_bytes": 1724097770
|
| 798 |
+
},
|
| 799 |
+
{
|
| 800 |
+
"path": "checkpoints/official/rdlm/lm1b/LM1B/config.yaml",
|
| 801 |
+
"sha256": "3acb20fdbaa5143e66a1e722e17fce384c557efac1e24cbcee78dc524183f5f3",
|
| 802 |
+
"size_bytes": 1241
|
| 803 |
+
},
|
| 804 |
+
{
|
| 805 |
+
"path": "checkpoints/official/rdlm/lm1b/LM1B/sde.pkl",
|
| 806 |
+
"sha256": "29ef378b58764f1d5f2da156a2060b57098d4dd6cb299c6c5d9a8d488152bbb0",
|
| 807 |
+
"size_bytes": 160713
|
| 808 |
+
},
|
| 809 |
+
{
|
| 810 |
+
"path": "checkpoints/official/rdlm/lm1b/Text8/checkpoint.pth",
|
| 811 |
+
"sha256": "4f10e0dfc81d83ff3ec64c73b1bc8cf8fa60a34c4ea174e3194d7187df759b69",
|
| 812 |
+
"size_bytes": 1478972002
|
| 813 |
+
},
|
| 814 |
+
{
|
| 815 |
+
"path": "checkpoints/official/rdlm/lm1b/Text8/config.yaml",
|
| 816 |
+
"sha256": "f1caa7de074033a2a4a41329f5db8b10dbea6bd12dd6984c1a01650316df308a",
|
| 817 |
+
"size_bytes": 1206
|
| 818 |
+
},
|
| 819 |
+
{
|
| 820 |
+
"path": "checkpoints/official/rdlm/lm1b/Text8/sde.pkl",
|
| 821 |
+
"sha256": "5fec68cb5e4aec10a2a0148baa52dd60e48ebd59adb088a9190fd8d4ae1dd720",
|
| 822 |
+
"size_bytes": 160713
|
| 823 |
+
}
|
| 824 |
+
],
|
| 825 |
+
"provenance": "official",
|
| 826 |
+
"quarantined": null,
|
| 827 |
+
"status": "downloaded",
|
| 828 |
+
"teacher_family": "continuous_rdlm"
|
| 829 |
+
},
|
| 830 |
+
"sdtt_owt_hf": {
|
| 831 |
+
"backend": "huggingface",
|
| 832 |
+
"destination": "checkpoints/official/mdlm_sdtt/owt",
|
| 833 |
+
"files": [
|
| 834 |
+
{
|
| 835 |
+
"path": "checkpoints/official/mdlm_sdtt/owt/.cache/huggingface/.gitignore",
|
| 836 |
+
"sha256": "684888c0ebb17f374298b65ee2807526c066094c701bcc7ebbe1c1095f494fc1",
|
| 837 |
+
"size_bytes": 1
|
| 838 |
+
},
|
| 839 |
+
{
|
| 840 |
+
"path": "checkpoints/official/mdlm_sdtt/owt/.cache/huggingface/download/config.json.lock",
|
| 841 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 842 |
+
"size_bytes": 0
|
| 843 |
+
},
|
| 844 |
+
{
|
| 845 |
+
"path": "checkpoints/official/mdlm_sdtt/owt/.cache/huggingface/download/config.json.metadata",
|
| 846 |
+
"sha256": "1e0f3aee8e157cca53e07b747343b995b9b292319928273b9b7442b316a4f248",
|
| 847 |
+
"size_bytes": 101
|
| 848 |
+
},
|
| 849 |
+
{
|
| 850 |
+
"path": "checkpoints/official/mdlm_sdtt/owt/.cache/huggingface/download/model.safetensors.lock",
|
| 851 |
+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 852 |
+
"size_bytes": 0
|
| 853 |
+
},
|
| 854 |
+
{
|
| 855 |
+
"path": "checkpoints/official/mdlm_sdtt/owt/.cache/huggingface/download/model.safetensors.metadata",
|
| 856 |
+
"sha256": "6c0d35153b218696940f35f8243c579e2fd36c61d37b2191c376da401b6502ef",
|
| 857 |
+
"size_bytes": 125
|
| 858 |
+
},
|
| 859 |
+
{
|
| 860 |
+
"path": "checkpoints/official/mdlm_sdtt/owt/config.json",
|
| 861 |
+
"sha256": "8a62a25b9f44b2476d561becd36d60febda7d7d4217d81db9a6987c29ffb21ae",
|
| 862 |
+
"size_bytes": 5402
|
| 863 |
+
},
|
| 864 |
+
{
|
| 865 |
+
"path": "checkpoints/official/mdlm_sdtt/owt/model.safetensors",
|
| 866 |
+
"sha256": "33338a0cebd25993fc455447248563d00fe3414b66ce29ca0d7f3529f4d6218d",
|
| 867 |
+
"size_bytes": 678522696
|
| 868 |
+
}
|
| 869 |
+
],
|
| 870 |
+
"provenance": "official",
|
| 871 |
+
"quarantined": null,
|
| 872 |
+
"status": "downloaded",
|
| 873 |
+
"teacher_family": "masked_mdlm"
|
| 874 |
+
}
|
| 875 |
+
},
|
| 876 |
+
"schema_version": 1
|
| 877 |
+
}
|
artifacts/sources.yaml
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
flm:
|
| 2 |
+
url: https://github.com/david3684/flm.git
|
| 3 |
+
commit: a1918d5164e5038e37d0b7a4fb2010ce75b863b3
|
| 4 |
+
langflow:
|
| 5 |
+
url: https://github.com/nealchen2003/LangFlow.git
|
| 6 |
+
commit: a712b08570c56c787d6ef24f8e906a2dcdf768f5
|
| 7 |
+
duo:
|
| 8 |
+
url: https://github.com/s-sahoo/duo.git
|
| 9 |
+
commit: 7c9b498f5b717de064d6fad7e2509c866e6cb620
|
| 10 |
+
mdlm:
|
| 11 |
+
url: https://github.com/kuleshov-group/mdlm.git
|
| 12 |
+
commit: c112c526d193436838c98d81455ee51f90309470
|
| 13 |
+
candi:
|
| 14 |
+
url: https://github.com/patrickpynadath1/candi-diffusion.git
|
| 15 |
+
commit: cd57ae9eec98d6ac71cd52bdc50eeec8dfd70f91
|
| 16 |
+
rdlm:
|
| 17 |
+
url: https://github.com/harryjo97/RDLM.git
|
| 18 |
+
commit: 67443aa6a2d0fa981eb7c6105f9cbc563e59c5c1
|
| 19 |
+
sdtt:
|
| 20 |
+
url: https://github.com/jdeschena/sdtt.git
|
| 21 |
+
commit: 1150985e90b8f2d5749e4469d5154eff9ec922c4
|
| 22 |
+
di4c:
|
| 23 |
+
url: https://github.com/sony/di4c.git
|
| 24 |
+
commit: ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3
|
checkpoints/quarantine/20260825T101647Z/official/rdlm/lm1b/LM1B/config.yaml
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
seed: 0
|
| 2 |
+
ngpus: 8
|
| 3 |
+
use_wandb: true
|
| 4 |
+
run_mode: train
|
| 5 |
+
num_workers: 32
|
| 6 |
+
paths:
|
| 7 |
+
experiments: exps
|
| 8 |
+
n_jobs: 2
|
| 9 |
+
model:
|
| 10 |
+
_target_: model.dit.DiT
|
| 11 |
+
hidden_size: 768
|
| 12 |
+
n_heads: 12
|
| 13 |
+
cond_dim: 128
|
| 14 |
+
dropout: 0.0
|
| 15 |
+
n_blocks: 14
|
| 16 |
+
length: 128
|
| 17 |
+
input_dim: ${tokens}
|
| 18 |
+
output_dim: ${tokens}
|
| 19 |
+
name: lm1b
|
| 20 |
+
original_tokens: 30522
|
| 21 |
+
tokens: 3
|
| 22 |
+
training:
|
| 23 |
+
batch_size: 512
|
| 24 |
+
accum: 1
|
| 25 |
+
n_iters: 1000001
|
| 26 |
+
snapshot_freq: 50000
|
| 27 |
+
log_freq: 50
|
| 28 |
+
eval_freq: 100
|
| 29 |
+
snapshot_freq_for_preemption: 5000
|
| 30 |
+
snapshot_sampling: true
|
| 31 |
+
ema: 0.9999
|
| 32 |
+
loss_type: ce
|
| 33 |
+
data:
|
| 34 |
+
train: lm1b
|
| 35 |
+
valid: lm1b
|
| 36 |
+
cache_dir: data
|
| 37 |
+
sampling:
|
| 38 |
+
steps: 1000
|
| 39 |
+
predictor: grw
|
| 40 |
+
eval:
|
| 41 |
+
batch_size: ${training.batch_size}
|
| 42 |
+
n_iters: 200
|
| 43 |
+
entropy: true
|
| 44 |
+
gen_ppl: true
|
| 45 |
+
eval_model: gpt2
|
| 46 |
+
nll: true
|
| 47 |
+
simul_steps: 500
|
| 48 |
+
sde:
|
| 49 |
+
_target_: sde.LogBridge_Mixture
|
| 50 |
+
preprocess_steps: 10000
|
| 51 |
+
mix_type: step
|
| 52 |
+
rho_scale: 1.14
|
| 53 |
+
step_thr: 0.38
|
| 54 |
+
prior_dist:
|
| 55 |
+
_target_: distribution.Mixture
|
| 56 |
+
add_mask_token: true
|
| 57 |
+
scheduler:
|
| 58 |
+
_target_: scheduler_lib.Geometric
|
| 59 |
+
beta_0: 0.001
|
| 60 |
+
beta_f: 0.2
|
| 61 |
+
weight_type: step
|
| 62 |
+
left: 0.3
|
| 63 |
+
right: 0.75
|
| 64 |
+
lb: 0.0001
|
| 65 |
+
ub: 1.0
|
| 66 |
+
optim:
|
| 67 |
+
name: adamw
|
| 68 |
+
optimizer: AdamW
|
| 69 |
+
weight_decay: 0
|
| 70 |
+
lr: 0.0003
|
| 71 |
+
beta1: 0.9
|
| 72 |
+
beta2: 0.999
|
| 73 |
+
eps: 1.0e-08
|
| 74 |
+
warmup: 2500
|
| 75 |
+
grad_clip: 1.0
|
patches/langflow/README.md
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# LangFlow adapter audit
|
| 2 |
+
|
| 3 |
+
The adapter targets the read-only LangFlow checkout at commit
|
| 4 |
+
`a712b08570c56c787d6ef24f8e906a2dcdf768f5`. It does not patch or copy model
|
| 5 |
+
code. The supported registry cells are LangFlow/LM1B and LangFlow/OWT; each
|
| 6 |
+
cell binds its own official Hugging Face checkpoint and tokenizer contract.
|
| 7 |
+
|
| 8 |
+
The pinned release's public `upstreams/langflow/inference.py` hard-codes its
|
| 9 |
+
bundled config directory, which is not dataset-safe for the LM1B and OWT
|
| 10 |
+
checkpoint pair. The adapter therefore enters through
|
| 11 |
+
`adapters/sample_langflow.py`: it imports the pinned upstream `LangFlow` model
|
| 12 |
+
code, loads `LangFlowConfig` from the selected checkpoint directory, and keeps
|
| 13 |
+
the upstream underscore-style argparse contract. The exact mapping is:
|
| 14 |
+
|
| 15 |
+
- `--checkpoint`: immutable canonical
|
| 16 |
+
`checkpoints/official/langflow/<dataset>/model.safetensors`
|
| 17 |
+
- `--num_samples`: requested sample count, with upstream remainder batching
|
| 18 |
+
- `--num_steps`: requested many-step grid value
|
| 19 |
+
- `--seq_length`: canonical dataset length, 128 for LM1B and 1024 for OWT
|
| 20 |
+
- `--seed`: request seed
|
| 21 |
+
- `--output`: canonical `upstream_samples.txt` in the run directory
|
| 22 |
+
- `--tokenizer`: canonical dataset tokenizer from `artifacts/data.yaml`
|
| 23 |
+
|
| 24 |
+
The shared `dlb.adapters.capture` process observes the token tensor returned by
|
| 25 |
+
the real `LangFlow.generate_samples` method and the text returned by the real
|
| 26 |
+
tokenizer. It writes `upstream_token_ids.json` using
|
| 27 |
+
`dlb-upstream-token-capture-v1`; conversion does not parse the upstream text
|
| 28 |
+
file's human-oriented sample delimiters. Before upstream execution, the
|
| 29 |
+
wrapper cross-checks the dataset tokenizer and revision in `artifacts/data.yaml`
|
| 30 |
+
against `data/manifests/downloads.json`, requires that manifest's immutable
|
| 31 |
+
local snapshot directory, and replaces the upstream movable tokenizer request
|
| 32 |
+
with that path plus `local_files_only=True` under Hugging Face offline mode.
|
| 33 |
+
There is no network or moving-revision fallback. Conversion requires exactly the
|
| 34 |
+
requested number of fixed-length token rows and validates Task 7's runner-resolved
|
| 35 |
+
checkpoint digest, lock ID, selection, and `continuous_langflow` family before
|
| 36 |
+
reading the capture.
|
| 37 |
+
|
| 38 |
+
The upstream does not report per-sample latency. Canonical records therefore
|
| 39 |
+
use schema-permitted `0.0` only as an excluded sentinel; conversion metadata
|
| 40 |
+
states `unavailable_excluded_sentinel` and `exclude_from_latency: true`.
|
results/conditional/metrics/lm1b/duo_dcd/steps_4/metrics.json
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "lm1b",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 209.32037280649857,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 811914.9311523438,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 151934
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.03587051089962172,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [
|
| 41 |
+
0
|
| 42 |
+
],
|
| 43 |
+
"logarithm": "natural",
|
| 44 |
+
"mean_entropy": 3.7426513131748633,
|
| 45 |
+
"sample_count": 2048,
|
| 46 |
+
"token_count": 131072,
|
| 47 |
+
"unit": "nats"
|
| 48 |
+
},
|
| 49 |
+
"generated_minus_reference": -0.07450128603615136,
|
| 50 |
+
"reference": {
|
| 51 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 52 |
+
"excluded_token_ids": [
|
| 53 |
+
0
|
| 54 |
+
],
|
| 55 |
+
"logarithm": "natural",
|
| 56 |
+
"mean_entropy": 3.8171525992110147,
|
| 57 |
+
"sample_count": 2048,
|
| 58 |
+
"token_count": 131072,
|
| 59 |
+
"unit": "nats"
|
| 60 |
+
},
|
| 61 |
+
"slice": "source_tokens_64_128_only"
|
| 62 |
+
},
|
| 63 |
+
"self_bleu": {
|
| 64 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 65 |
+
"completions_per_prompt": 5,
|
| 66 |
+
"ngram_order": 4,
|
| 67 |
+
"prompt_count": 256,
|
| 68 |
+
"sample_count": 1280,
|
| 69 |
+
"score": 0.14268675432242428,
|
| 70 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 71 |
+
"smoothing": "chen_cherry_method1",
|
| 72 |
+
"tokenization": "gpt2",
|
| 73 |
+
"weights": [
|
| 74 |
+
0.25,
|
| 75 |
+
0.25,
|
| 76 |
+
0.25,
|
| 77 |
+
0.25
|
| 78 |
+
]
|
| 79 |
+
}
|
| 80 |
+
},
|
| 81 |
+
"prefix_length": 64,
|
| 82 |
+
"production_sample_count": 2048,
|
| 83 |
+
"protocol": "c64_zs_v1",
|
| 84 |
+
"requested_metrics": [
|
| 85 |
+
"conditional_gen_ppl",
|
| 86 |
+
"mauve_suffix",
|
| 87 |
+
"entropy",
|
| 88 |
+
"self_bleu",
|
| 89 |
+
"prefix_exact_match"
|
| 90 |
+
],
|
| 91 |
+
"sample_count": 2048,
|
| 92 |
+
"samples": "results/conditional/samples/lm1b/duo_dcd/steps_4/samples.jsonl",
|
| 93 |
+
"samples_sha256": "2fd697c7368ec40aa9d1a5b0b67e0078657acee3cf12f74935f1e09e70b1c0a4",
|
| 94 |
+
"schema_version": 1
|
| 95 |
+
}
|
results/conditional/metrics/lm1b/duo_dcd/steps_8/metrics.json
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "lm1b",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 155.4102272275335,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 766533.0893554688,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 151907
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.08763089060698975,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [
|
| 41 |
+
0
|
| 42 |
+
],
|
| 43 |
+
"logarithm": "natural",
|
| 44 |
+
"mean_entropy": 3.7630187834968547,
|
| 45 |
+
"sample_count": 2048,
|
| 46 |
+
"token_count": 131072,
|
| 47 |
+
"unit": "nats"
|
| 48 |
+
},
|
| 49 |
+
"generated_minus_reference": -0.05413381571415998,
|
| 50 |
+
"reference": {
|
| 51 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 52 |
+
"excluded_token_ids": [
|
| 53 |
+
0
|
| 54 |
+
],
|
| 55 |
+
"logarithm": "natural",
|
| 56 |
+
"mean_entropy": 3.8171525992110147,
|
| 57 |
+
"sample_count": 2048,
|
| 58 |
+
"token_count": 131072,
|
| 59 |
+
"unit": "nats"
|
| 60 |
+
},
|
| 61 |
+
"slice": "source_tokens_64_128_only"
|
| 62 |
+
},
|
| 63 |
+
"self_bleu": {
|
| 64 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 65 |
+
"completions_per_prompt": 5,
|
| 66 |
+
"ngram_order": 4,
|
| 67 |
+
"prompt_count": 256,
|
| 68 |
+
"sample_count": 1280,
|
| 69 |
+
"score": 0.14312298725073697,
|
| 70 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 71 |
+
"smoothing": "chen_cherry_method1",
|
| 72 |
+
"tokenization": "gpt2",
|
| 73 |
+
"weights": [
|
| 74 |
+
0.25,
|
| 75 |
+
0.25,
|
| 76 |
+
0.25,
|
| 77 |
+
0.25
|
| 78 |
+
]
|
| 79 |
+
}
|
| 80 |
+
},
|
| 81 |
+
"prefix_length": 64,
|
| 82 |
+
"production_sample_count": 2048,
|
| 83 |
+
"protocol": "c64_zs_v1",
|
| 84 |
+
"requested_metrics": [
|
| 85 |
+
"conditional_gen_ppl",
|
| 86 |
+
"mauve_suffix",
|
| 87 |
+
"entropy",
|
| 88 |
+
"self_bleu",
|
| 89 |
+
"prefix_exact_match"
|
| 90 |
+
],
|
| 91 |
+
"sample_count": 2048,
|
| 92 |
+
"samples": "results/conditional/samples/lm1b/duo_dcd/steps_8/samples.jsonl",
|
| 93 |
+
"samples_sha256": "0f3afbf266259292f4aa47cec3628212ca2a880ee6e1f92bc2bfe4f88ee65c84",
|
| 94 |
+
"schema_version": 1
|
| 95 |
+
}
|
results/conditional/metrics/owt/duo/steps_16/metrics.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "owt",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 117.05452234697738,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 623067.5930175781,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 130824
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.48335512723867746,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [],
|
| 41 |
+
"logarithm": "natural",
|
| 42 |
+
"mean_entropy": 3.811669839133404,
|
| 43 |
+
"sample_count": 2048,
|
| 44 |
+
"token_count": 131072,
|
| 45 |
+
"unit": "nats"
|
| 46 |
+
},
|
| 47 |
+
"generated_minus_reference": 0.0026212807874803623,
|
| 48 |
+
"reference": {
|
| 49 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 50 |
+
"excluded_token_ids": [],
|
| 51 |
+
"logarithm": "natural",
|
| 52 |
+
"mean_entropy": 3.8090485583459235,
|
| 53 |
+
"sample_count": 2048,
|
| 54 |
+
"token_count": 131072,
|
| 55 |
+
"unit": "nats"
|
| 56 |
+
},
|
| 57 |
+
"slice": "source_tokens_64_128_only"
|
| 58 |
+
},
|
| 59 |
+
"self_bleu": {
|
| 60 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 61 |
+
"completions_per_prompt": 5,
|
| 62 |
+
"ngram_order": 4,
|
| 63 |
+
"prompt_count": 256,
|
| 64 |
+
"sample_count": 1280,
|
| 65 |
+
"score": 0.03880899098992529,
|
| 66 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 67 |
+
"smoothing": "chen_cherry_method1",
|
| 68 |
+
"tokenization": "gpt2",
|
| 69 |
+
"weights": [
|
| 70 |
+
0.25,
|
| 71 |
+
0.25,
|
| 72 |
+
0.25,
|
| 73 |
+
0.25
|
| 74 |
+
]
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"prefix_length": 64,
|
| 78 |
+
"production_sample_count": 2048,
|
| 79 |
+
"protocol": "c64_zs_v1",
|
| 80 |
+
"requested_metrics": [
|
| 81 |
+
"conditional_gen_ppl",
|
| 82 |
+
"mauve_suffix",
|
| 83 |
+
"entropy",
|
| 84 |
+
"self_bleu",
|
| 85 |
+
"prefix_exact_match"
|
| 86 |
+
],
|
| 87 |
+
"sample_count": 2048,
|
| 88 |
+
"samples": "results/conditional/samples/owt/duo/steps_16/samples.jsonl",
|
| 89 |
+
"samples_sha256": "5e0149f0d1312d76e863d31176b4fb56cdbbd1b492d8fba699ac0221b7f8541e",
|
| 90 |
+
"schema_version": 1
|
| 91 |
+
}
|
results/conditional/metrics/owt/duo/steps_8/metrics.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "owt",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 181.9072684133015,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 680273.982421875,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 130734
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.35676239466566545,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [],
|
| 41 |
+
"logarithm": "natural",
|
| 42 |
+
"mean_entropy": 3.806470914345162,
|
| 43 |
+
"sample_count": 2048,
|
| 44 |
+
"token_count": 131072,
|
| 45 |
+
"unit": "nats"
|
| 46 |
+
},
|
| 47 |
+
"generated_minus_reference": -0.0025776440007616586,
|
| 48 |
+
"reference": {
|
| 49 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 50 |
+
"excluded_token_ids": [],
|
| 51 |
+
"logarithm": "natural",
|
| 52 |
+
"mean_entropy": 3.8090485583459235,
|
| 53 |
+
"sample_count": 2048,
|
| 54 |
+
"token_count": 131072,
|
| 55 |
+
"unit": "nats"
|
| 56 |
+
},
|
| 57 |
+
"slice": "source_tokens_64_128_only"
|
| 58 |
+
},
|
| 59 |
+
"self_bleu": {
|
| 60 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 61 |
+
"completions_per_prompt": 5,
|
| 62 |
+
"ngram_order": 4,
|
| 63 |
+
"prompt_count": 256,
|
| 64 |
+
"sample_count": 1280,
|
| 65 |
+
"score": 0.03628265012812328,
|
| 66 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 67 |
+
"smoothing": "chen_cherry_method1",
|
| 68 |
+
"tokenization": "gpt2",
|
| 69 |
+
"weights": [
|
| 70 |
+
0.25,
|
| 71 |
+
0.25,
|
| 72 |
+
0.25,
|
| 73 |
+
0.25
|
| 74 |
+
]
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"prefix_length": 64,
|
| 78 |
+
"production_sample_count": 2048,
|
| 79 |
+
"protocol": "c64_zs_v1",
|
| 80 |
+
"requested_metrics": [
|
| 81 |
+
"conditional_gen_ppl",
|
| 82 |
+
"mauve_suffix",
|
| 83 |
+
"entropy",
|
| 84 |
+
"self_bleu",
|
| 85 |
+
"prefix_exact_match"
|
| 86 |
+
],
|
| 87 |
+
"sample_count": 2048,
|
| 88 |
+
"samples": "results/conditional/samples/owt/duo/steps_8/samples.jsonl",
|
| 89 |
+
"samples_sha256": "4e1e241cd5e3d57a2d874605fd07f712d2f16ad726688c1e749fd7fb4ec2b6bf",
|
| 90 |
+
"schema_version": 1
|
| 91 |
+
}
|
results/conditional/metrics/owt/duo_dcd/steps_32/metrics.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "owt",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 65.01050813050746,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 546252.2498779297,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 130853
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.5270928403565143,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [],
|
| 41 |
+
"logarithm": "natural",
|
| 42 |
+
"mean_entropy": 3.784345295641898,
|
| 43 |
+
"sample_count": 2048,
|
| 44 |
+
"token_count": 131072,
|
| 45 |
+
"unit": "nats"
|
| 46 |
+
},
|
| 47 |
+
"generated_minus_reference": -0.02470326270402534,
|
| 48 |
+
"reference": {
|
| 49 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 50 |
+
"excluded_token_ids": [],
|
| 51 |
+
"logarithm": "natural",
|
| 52 |
+
"mean_entropy": 3.8090485583459235,
|
| 53 |
+
"sample_count": 2048,
|
| 54 |
+
"token_count": 131072,
|
| 55 |
+
"unit": "nats"
|
| 56 |
+
},
|
| 57 |
+
"slice": "source_tokens_64_128_only"
|
| 58 |
+
},
|
| 59 |
+
"self_bleu": {
|
| 60 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 61 |
+
"completions_per_prompt": 5,
|
| 62 |
+
"ngram_order": 4,
|
| 63 |
+
"prompt_count": 256,
|
| 64 |
+
"sample_count": 1280,
|
| 65 |
+
"score": 0.04862075235280836,
|
| 66 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 67 |
+
"smoothing": "chen_cherry_method1",
|
| 68 |
+
"tokenization": "gpt2",
|
| 69 |
+
"weights": [
|
| 70 |
+
0.25,
|
| 71 |
+
0.25,
|
| 72 |
+
0.25,
|
| 73 |
+
0.25
|
| 74 |
+
]
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"prefix_length": 64,
|
| 78 |
+
"production_sample_count": 2048,
|
| 79 |
+
"protocol": "c64_zs_v1",
|
| 80 |
+
"requested_metrics": [
|
| 81 |
+
"conditional_gen_ppl",
|
| 82 |
+
"mauve_suffix",
|
| 83 |
+
"entropy",
|
| 84 |
+
"self_bleu",
|
| 85 |
+
"prefix_exact_match"
|
| 86 |
+
],
|
| 87 |
+
"sample_count": 2048,
|
| 88 |
+
"samples": "results/conditional/samples/owt/duo_dcd/steps_32/samples.jsonl",
|
| 89 |
+
"samples_sha256": "7761cbadfd1e6f8d7c7abfe685172847084c4b1023f24d28b34aa58c923139af",
|
| 90 |
+
"schema_version": 1
|
| 91 |
+
}
|
results/conditional/metrics/owt/duo_di4c/steps_16/metrics.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "owt",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 52.00960413502672,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 516838.9313964844,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 130798
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.5760369467315837,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [],
|
| 41 |
+
"logarithm": "natural",
|
| 42 |
+
"mean_entropy": 3.740452823563147,
|
| 43 |
+
"sample_count": 2048,
|
| 44 |
+
"token_count": 131072,
|
| 45 |
+
"unit": "nats"
|
| 46 |
+
},
|
| 47 |
+
"generated_minus_reference": -0.06859573478277659,
|
| 48 |
+
"reference": {
|
| 49 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 50 |
+
"excluded_token_ids": [],
|
| 51 |
+
"logarithm": "natural",
|
| 52 |
+
"mean_entropy": 3.8090485583459235,
|
| 53 |
+
"sample_count": 2048,
|
| 54 |
+
"token_count": 131072,
|
| 55 |
+
"unit": "nats"
|
| 56 |
+
},
|
| 57 |
+
"slice": "source_tokens_64_128_only"
|
| 58 |
+
},
|
| 59 |
+
"self_bleu": {
|
| 60 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 61 |
+
"completions_per_prompt": 5,
|
| 62 |
+
"ngram_order": 4,
|
| 63 |
+
"prompt_count": 256,
|
| 64 |
+
"sample_count": 1280,
|
| 65 |
+
"score": 0.04896837417641065,
|
| 66 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 67 |
+
"smoothing": "chen_cherry_method1",
|
| 68 |
+
"tokenization": "gpt2",
|
| 69 |
+
"weights": [
|
| 70 |
+
0.25,
|
| 71 |
+
0.25,
|
| 72 |
+
0.25,
|
| 73 |
+
0.25
|
| 74 |
+
]
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"prefix_length": 64,
|
| 78 |
+
"production_sample_count": 2048,
|
| 79 |
+
"protocol": "c64_zs_v1",
|
| 80 |
+
"requested_metrics": [
|
| 81 |
+
"conditional_gen_ppl",
|
| 82 |
+
"mauve_suffix",
|
| 83 |
+
"entropy",
|
| 84 |
+
"self_bleu",
|
| 85 |
+
"prefix_exact_match"
|
| 86 |
+
],
|
| 87 |
+
"sample_count": 2048,
|
| 88 |
+
"samples": "results/conditional/samples/owt/duo_di4c/steps_16/samples.jsonl",
|
| 89 |
+
"samples_sha256": "51efb06f10aa3f5fb01ed172780cf4d3b09b3774cd964352133d047bbb54030c",
|
| 90 |
+
"schema_version": 1
|
| 91 |
+
}
|
results/conditional/metrics/owt/duo_di4c/steps_2/metrics.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "owt",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 1007.8743879428796,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 897900.6047363281,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 129837
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.019035138620966317,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [],
|
| 41 |
+
"logarithm": "natural",
|
| 42 |
+
"mean_entropy": 3.748546430363911,
|
| 43 |
+
"sample_count": 2048,
|
| 44 |
+
"token_count": 131072,
|
| 45 |
+
"unit": "nats"
|
| 46 |
+
},
|
| 47 |
+
"generated_minus_reference": -0.0605021279820126,
|
| 48 |
+
"reference": {
|
| 49 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 50 |
+
"excluded_token_ids": [],
|
| 51 |
+
"logarithm": "natural",
|
| 52 |
+
"mean_entropy": 3.8090485583459235,
|
| 53 |
+
"sample_count": 2048,
|
| 54 |
+
"token_count": 131072,
|
| 55 |
+
"unit": "nats"
|
| 56 |
+
},
|
| 57 |
+
"slice": "source_tokens_64_128_only"
|
| 58 |
+
},
|
| 59 |
+
"self_bleu": {
|
| 60 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 61 |
+
"completions_per_prompt": 5,
|
| 62 |
+
"ngram_order": 4,
|
| 63 |
+
"prompt_count": 256,
|
| 64 |
+
"sample_count": 1280,
|
| 65 |
+
"score": 0.017747813877889027,
|
| 66 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 67 |
+
"smoothing": "chen_cherry_method1",
|
| 68 |
+
"tokenization": "gpt2",
|
| 69 |
+
"weights": [
|
| 70 |
+
0.25,
|
| 71 |
+
0.25,
|
| 72 |
+
0.25,
|
| 73 |
+
0.25
|
| 74 |
+
]
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"prefix_length": 64,
|
| 78 |
+
"production_sample_count": 2048,
|
| 79 |
+
"protocol": "c64_zs_v1",
|
| 80 |
+
"requested_metrics": [
|
| 81 |
+
"conditional_gen_ppl",
|
| 82 |
+
"mauve_suffix",
|
| 83 |
+
"entropy",
|
| 84 |
+
"self_bleu",
|
| 85 |
+
"prefix_exact_match"
|
| 86 |
+
],
|
| 87 |
+
"sample_count": 2048,
|
| 88 |
+
"samples": "results/conditional/samples/owt/duo_di4c/steps_2/samples.jsonl",
|
| 89 |
+
"samples_sha256": "8bc60789f9176d84a1d187c641dce386d19c483f04e2461b30d8b08b25118b99",
|
| 90 |
+
"schema_version": 1
|
| 91 |
+
}
|
results/conditional/metrics/owt/duo_di4c/steps_32/metrics.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "owt",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 39.42306499268451,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 480796.1837158203,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 130852
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.5667127550570545,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [],
|
| 41 |
+
"logarithm": "natural",
|
| 42 |
+
"mean_entropy": 3.730506689949983,
|
| 43 |
+
"sample_count": 2048,
|
| 44 |
+
"token_count": 131072,
|
| 45 |
+
"unit": "nats"
|
| 46 |
+
},
|
| 47 |
+
"generated_minus_reference": -0.07854186839594046,
|
| 48 |
+
"reference": {
|
| 49 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 50 |
+
"excluded_token_ids": [],
|
| 51 |
+
"logarithm": "natural",
|
| 52 |
+
"mean_entropy": 3.8090485583459235,
|
| 53 |
+
"sample_count": 2048,
|
| 54 |
+
"token_count": 131072,
|
| 55 |
+
"unit": "nats"
|
| 56 |
+
},
|
| 57 |
+
"slice": "source_tokens_64_128_only"
|
| 58 |
+
},
|
| 59 |
+
"self_bleu": {
|
| 60 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 61 |
+
"completions_per_prompt": 5,
|
| 62 |
+
"ngram_order": 4,
|
| 63 |
+
"prompt_count": 256,
|
| 64 |
+
"sample_count": 1280,
|
| 65 |
+
"score": 0.05645896733063417,
|
| 66 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 67 |
+
"smoothing": "chen_cherry_method1",
|
| 68 |
+
"tokenization": "gpt2",
|
| 69 |
+
"weights": [
|
| 70 |
+
0.25,
|
| 71 |
+
0.25,
|
| 72 |
+
0.25,
|
| 73 |
+
0.25
|
| 74 |
+
]
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"prefix_length": 64,
|
| 78 |
+
"production_sample_count": 2048,
|
| 79 |
+
"protocol": "c64_zs_v1",
|
| 80 |
+
"requested_metrics": [
|
| 81 |
+
"conditional_gen_ppl",
|
| 82 |
+
"mauve_suffix",
|
| 83 |
+
"entropy",
|
| 84 |
+
"self_bleu",
|
| 85 |
+
"prefix_exact_match"
|
| 86 |
+
],
|
| 87 |
+
"sample_count": 2048,
|
| 88 |
+
"samples": "results/conditional/samples/owt/duo_di4c/steps_32/samples.jsonl",
|
| 89 |
+
"samples_sha256": "5f2c9fc8ef3f556895b8f80b849f28466fdb29eeb08c33b104cf476968fdd742",
|
| 90 |
+
"schema_version": 1
|
| 91 |
+
}
|
results/conditional/metrics/owt/flm/steps_16/metrics.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "owt",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 423.30408349540033,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 775165.6528320312,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 128167
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.020319028723620396,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [],
|
| 41 |
+
"logarithm": "natural",
|
| 42 |
+
"mean_entropy": 3.6004865011575804,
|
| 43 |
+
"sample_count": 2048,
|
| 44 |
+
"token_count": 131072,
|
| 45 |
+
"unit": "nats"
|
| 46 |
+
},
|
| 47 |
+
"generated_minus_reference": -0.2085620571883431,
|
| 48 |
+
"reference": {
|
| 49 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 50 |
+
"excluded_token_ids": [],
|
| 51 |
+
"logarithm": "natural",
|
| 52 |
+
"mean_entropy": 3.8090485583459235,
|
| 53 |
+
"sample_count": 2048,
|
| 54 |
+
"token_count": 131072,
|
| 55 |
+
"unit": "nats"
|
| 56 |
+
},
|
| 57 |
+
"slice": "source_tokens_64_128_only"
|
| 58 |
+
},
|
| 59 |
+
"self_bleu": {
|
| 60 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 61 |
+
"completions_per_prompt": 5,
|
| 62 |
+
"ngram_order": 4,
|
| 63 |
+
"prompt_count": 256,
|
| 64 |
+
"sample_count": 1280,
|
| 65 |
+
"score": 0.05085475179130416,
|
| 66 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 67 |
+
"smoothing": "chen_cherry_method1",
|
| 68 |
+
"tokenization": "gpt2",
|
| 69 |
+
"weights": [
|
| 70 |
+
0.25,
|
| 71 |
+
0.25,
|
| 72 |
+
0.25,
|
| 73 |
+
0.25
|
| 74 |
+
]
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"prefix_length": 64,
|
| 78 |
+
"production_sample_count": 2048,
|
| 79 |
+
"protocol": "c64_zs_v1",
|
| 80 |
+
"requested_metrics": [
|
| 81 |
+
"conditional_gen_ppl",
|
| 82 |
+
"mauve_suffix",
|
| 83 |
+
"entropy",
|
| 84 |
+
"self_bleu",
|
| 85 |
+
"prefix_exact_match"
|
| 86 |
+
],
|
| 87 |
+
"sample_count": 2048,
|
| 88 |
+
"samples": "results/conditional/samples/owt/flm/steps_16/samples.jsonl",
|
| 89 |
+
"samples_sha256": "d119335fe47153127b94b79b6145d0febd595862dec53363d6e55660710c1623",
|
| 90 |
+
"schema_version": 1
|
| 91 |
+
}
|
results/conditional/metrics/owt/flm/steps_8/metrics.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "owt",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 522.8471247741838,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 801771.1208496094,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 128093
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.01871077764691273,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [],
|
| 41 |
+
"logarithm": "natural",
|
| 42 |
+
"mean_entropy": 3.6276873653209707,
|
| 43 |
+
"sample_count": 2048,
|
| 44 |
+
"token_count": 131072,
|
| 45 |
+
"unit": "nats"
|
| 46 |
+
},
|
| 47 |
+
"generated_minus_reference": -0.1813611930249528,
|
| 48 |
+
"reference": {
|
| 49 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 50 |
+
"excluded_token_ids": [],
|
| 51 |
+
"logarithm": "natural",
|
| 52 |
+
"mean_entropy": 3.8090485583459235,
|
| 53 |
+
"sample_count": 2048,
|
| 54 |
+
"token_count": 131072,
|
| 55 |
+
"unit": "nats"
|
| 56 |
+
},
|
| 57 |
+
"slice": "source_tokens_64_128_only"
|
| 58 |
+
},
|
| 59 |
+
"self_bleu": {
|
| 60 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 61 |
+
"completions_per_prompt": 5,
|
| 62 |
+
"ngram_order": 4,
|
| 63 |
+
"prompt_count": 256,
|
| 64 |
+
"sample_count": 1280,
|
| 65 |
+
"score": 0.04843368670472299,
|
| 66 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 67 |
+
"smoothing": "chen_cherry_method1",
|
| 68 |
+
"tokenization": "gpt2",
|
| 69 |
+
"weights": [
|
| 70 |
+
0.25,
|
| 71 |
+
0.25,
|
| 72 |
+
0.25,
|
| 73 |
+
0.25
|
| 74 |
+
]
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"prefix_length": 64,
|
| 78 |
+
"production_sample_count": 2048,
|
| 79 |
+
"protocol": "c64_zs_v1",
|
| 80 |
+
"requested_metrics": [
|
| 81 |
+
"conditional_gen_ppl",
|
| 82 |
+
"mauve_suffix",
|
| 83 |
+
"entropy",
|
| 84 |
+
"self_bleu",
|
| 85 |
+
"prefix_exact_match"
|
| 86 |
+
],
|
| 87 |
+
"sample_count": 2048,
|
| 88 |
+
"samples": "results/conditional/samples/owt/flm/steps_8/samples.jsonl",
|
| 89 |
+
"samples_sha256": "09d445c18a9b567e4efa5d09e31cfc6d8fb3d9e481d2b32b9d473401192f3541",
|
| 90 |
+
"schema_version": 1
|
| 91 |
+
}
|
results/conditional/metrics/owt/fmlm/steps_1/metrics.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "owt",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 418.3709069699339,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 763425.544921875,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 126471
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.013042388506846947,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [],
|
| 41 |
+
"logarithm": "natural",
|
| 42 |
+
"mean_entropy": 3.3226030113769056,
|
| 43 |
+
"sample_count": 2048,
|
| 44 |
+
"token_count": 131072,
|
| 45 |
+
"unit": "nats"
|
| 46 |
+
},
|
| 47 |
+
"generated_minus_reference": -0.4864455469690179,
|
| 48 |
+
"reference": {
|
| 49 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 50 |
+
"excluded_token_ids": [],
|
| 51 |
+
"logarithm": "natural",
|
| 52 |
+
"mean_entropy": 3.8090485583459235,
|
| 53 |
+
"sample_count": 2048,
|
| 54 |
+
"token_count": 131072,
|
| 55 |
+
"unit": "nats"
|
| 56 |
+
},
|
| 57 |
+
"slice": "source_tokens_64_128_only"
|
| 58 |
+
},
|
| 59 |
+
"self_bleu": {
|
| 60 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 61 |
+
"completions_per_prompt": 5,
|
| 62 |
+
"ngram_order": 4,
|
| 63 |
+
"prompt_count": 256,
|
| 64 |
+
"sample_count": 1280,
|
| 65 |
+
"score": 0.06931739115447043,
|
| 66 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 67 |
+
"smoothing": "chen_cherry_method1",
|
| 68 |
+
"tokenization": "gpt2",
|
| 69 |
+
"weights": [
|
| 70 |
+
0.25,
|
| 71 |
+
0.25,
|
| 72 |
+
0.25,
|
| 73 |
+
0.25
|
| 74 |
+
]
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"prefix_length": 64,
|
| 78 |
+
"production_sample_count": 2048,
|
| 79 |
+
"protocol": "c64_zs_v1",
|
| 80 |
+
"requested_metrics": [
|
| 81 |
+
"conditional_gen_ppl",
|
| 82 |
+
"mauve_suffix",
|
| 83 |
+
"entropy",
|
| 84 |
+
"self_bleu",
|
| 85 |
+
"prefix_exact_match"
|
| 86 |
+
],
|
| 87 |
+
"sample_count": 2048,
|
| 88 |
+
"samples": "results/conditional/samples/owt/fmlm/steps_1/samples.jsonl",
|
| 89 |
+
"samples_sha256": "c01081bdd3435157d99b4df5617d03a8daf7916132598b43b1c9bd3d2e809c29",
|
| 90 |
+
"schema_version": 1
|
| 91 |
+
}
|
results/conditional/metrics/owt/fmlm/steps_4/metrics.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "owt",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 320.2237749965977,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 737148.0747070312,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 127777
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.047327734851474734,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [],
|
| 41 |
+
"logarithm": "natural",
|
| 42 |
+
"mean_entropy": 3.5209037170241406,
|
| 43 |
+
"sample_count": 2048,
|
| 44 |
+
"token_count": 131072,
|
| 45 |
+
"unit": "nats"
|
| 46 |
+
},
|
| 47 |
+
"generated_minus_reference": -0.28814484132178286,
|
| 48 |
+
"reference": {
|
| 49 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 50 |
+
"excluded_token_ids": [],
|
| 51 |
+
"logarithm": "natural",
|
| 52 |
+
"mean_entropy": 3.8090485583459235,
|
| 53 |
+
"sample_count": 2048,
|
| 54 |
+
"token_count": 131072,
|
| 55 |
+
"unit": "nats"
|
| 56 |
+
},
|
| 57 |
+
"slice": "source_tokens_64_128_only"
|
| 58 |
+
},
|
| 59 |
+
"self_bleu": {
|
| 60 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 61 |
+
"completions_per_prompt": 5,
|
| 62 |
+
"ngram_order": 4,
|
| 63 |
+
"prompt_count": 256,
|
| 64 |
+
"sample_count": 1280,
|
| 65 |
+
"score": 0.06447811515658214,
|
| 66 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 67 |
+
"smoothing": "chen_cherry_method1",
|
| 68 |
+
"tokenization": "gpt2",
|
| 69 |
+
"weights": [
|
| 70 |
+
0.25,
|
| 71 |
+
0.25,
|
| 72 |
+
0.25,
|
| 73 |
+
0.25
|
| 74 |
+
]
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"prefix_length": 64,
|
| 78 |
+
"production_sample_count": 2048,
|
| 79 |
+
"protocol": "c64_zs_v1",
|
| 80 |
+
"requested_metrics": [
|
| 81 |
+
"conditional_gen_ppl",
|
| 82 |
+
"mauve_suffix",
|
| 83 |
+
"entropy",
|
| 84 |
+
"self_bleu",
|
| 85 |
+
"prefix_exact_match"
|
| 86 |
+
],
|
| 87 |
+
"sample_count": 2048,
|
| 88 |
+
"samples": "results/conditional/samples/owt/fmlm/steps_4/samples.jsonl",
|
| 89 |
+
"samples_sha256": "3605aeeb3f4203bc4feed061bebaffc79a312a19f02760214a70631e513b6c98",
|
| 90 |
+
"schema_version": 1
|
| 91 |
+
}
|
results/conditional/metrics/owt/langflow/steps_1024/metrics.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "owt",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 83.29713008618158,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 574887.2979736328,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 129994
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.17680724007174314,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [],
|
| 41 |
+
"logarithm": "natural",
|
| 42 |
+
"mean_entropy": 3.309865086379918,
|
| 43 |
+
"sample_count": 2048,
|
| 44 |
+
"token_count": 131072,
|
| 45 |
+
"unit": "nats"
|
| 46 |
+
},
|
| 47 |
+
"generated_minus_reference": -0.4991834719660053,
|
| 48 |
+
"reference": {
|
| 49 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 50 |
+
"excluded_token_ids": [],
|
| 51 |
+
"logarithm": "natural",
|
| 52 |
+
"mean_entropy": 3.8090485583459235,
|
| 53 |
+
"sample_count": 2048,
|
| 54 |
+
"token_count": 131072,
|
| 55 |
+
"unit": "nats"
|
| 56 |
+
},
|
| 57 |
+
"slice": "source_tokens_64_128_only"
|
| 58 |
+
},
|
| 59 |
+
"self_bleu": {
|
| 60 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 61 |
+
"completions_per_prompt": 5,
|
| 62 |
+
"ngram_order": 4,
|
| 63 |
+
"prompt_count": 256,
|
| 64 |
+
"sample_count": 1280,
|
| 65 |
+
"score": 0.17340824216884448,
|
| 66 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 67 |
+
"smoothing": "chen_cherry_method1",
|
| 68 |
+
"tokenization": "gpt2",
|
| 69 |
+
"weights": [
|
| 70 |
+
0.25,
|
| 71 |
+
0.25,
|
| 72 |
+
0.25,
|
| 73 |
+
0.25
|
| 74 |
+
]
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"prefix_length": 64,
|
| 78 |
+
"production_sample_count": 2048,
|
| 79 |
+
"protocol": "c64_zs_v1",
|
| 80 |
+
"requested_metrics": [
|
| 81 |
+
"conditional_gen_ppl",
|
| 82 |
+
"mauve_suffix",
|
| 83 |
+
"entropy",
|
| 84 |
+
"self_bleu",
|
| 85 |
+
"prefix_exact_match"
|
| 86 |
+
],
|
| 87 |
+
"sample_count": 2048,
|
| 88 |
+
"samples": "results/conditional/samples/owt/langflow/steps_1024/samples.jsonl",
|
| 89 |
+
"samples_sha256": "a50d8186da1185515f43add8856efb37eb72c05407d84452de916147dc929362",
|
| 90 |
+
"schema_version": 1
|
| 91 |
+
}
|
results/conditional/metrics/owt/langflow/steps_32/metrics.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "owt",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 196.0952636775597,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 687680.2478027344,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 130277
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.20243654503381087,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [],
|
| 41 |
+
"logarithm": "natural",
|
| 42 |
+
"mean_entropy": 3.5900837748286336,
|
| 43 |
+
"sample_count": 2048,
|
| 44 |
+
"token_count": 131072,
|
| 45 |
+
"unit": "nats"
|
| 46 |
+
},
|
| 47 |
+
"generated_minus_reference": -0.21896478351728987,
|
| 48 |
+
"reference": {
|
| 49 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 50 |
+
"excluded_token_ids": [],
|
| 51 |
+
"logarithm": "natural",
|
| 52 |
+
"mean_entropy": 3.8090485583459235,
|
| 53 |
+
"sample_count": 2048,
|
| 54 |
+
"token_count": 131072,
|
| 55 |
+
"unit": "nats"
|
| 56 |
+
},
|
| 57 |
+
"slice": "source_tokens_64_128_only"
|
| 58 |
+
},
|
| 59 |
+
"self_bleu": {
|
| 60 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 61 |
+
"completions_per_prompt": 5,
|
| 62 |
+
"ngram_order": 4,
|
| 63 |
+
"prompt_count": 256,
|
| 64 |
+
"sample_count": 1280,
|
| 65 |
+
"score": 0.14085150876527613,
|
| 66 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 67 |
+
"smoothing": "chen_cherry_method1",
|
| 68 |
+
"tokenization": "gpt2",
|
| 69 |
+
"weights": [
|
| 70 |
+
0.25,
|
| 71 |
+
0.25,
|
| 72 |
+
0.25,
|
| 73 |
+
0.25
|
| 74 |
+
]
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"prefix_length": 64,
|
| 78 |
+
"production_sample_count": 2048,
|
| 79 |
+
"protocol": "c64_zs_v1",
|
| 80 |
+
"requested_metrics": [
|
| 81 |
+
"conditional_gen_ppl",
|
| 82 |
+
"mauve_suffix",
|
| 83 |
+
"entropy",
|
| 84 |
+
"self_bleu",
|
| 85 |
+
"prefix_exact_match"
|
| 86 |
+
],
|
| 87 |
+
"sample_count": 2048,
|
| 88 |
+
"samples": "results/conditional/samples/owt/langflow/steps_32/samples.jsonl",
|
| 89 |
+
"samples_sha256": "8185eeb9bd08b6f339fdac32e2c1d9f894df91d816f70102085b6b47febc0e73",
|
| 90 |
+
"schema_version": 1
|
| 91 |
+
}
|
results/conditional/metrics/owt/langflow/steps_4/metrics.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "owt",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 3153.3235155262064,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 1052850.2678222656,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 130688
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.028891059821879644,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [],
|
| 41 |
+
"logarithm": "natural",
|
| 42 |
+
"mean_entropy": 3.934521166625495,
|
| 43 |
+
"sample_count": 2048,
|
| 44 |
+
"token_count": 131072,
|
| 45 |
+
"unit": "nats"
|
| 46 |
+
},
|
| 47 |
+
"generated_minus_reference": 0.1254726082795714,
|
| 48 |
+
"reference": {
|
| 49 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 50 |
+
"excluded_token_ids": [],
|
| 51 |
+
"logarithm": "natural",
|
| 52 |
+
"mean_entropy": 3.8090485583459235,
|
| 53 |
+
"sample_count": 2048,
|
| 54 |
+
"token_count": 131072,
|
| 55 |
+
"unit": "nats"
|
| 56 |
+
},
|
| 57 |
+
"slice": "source_tokens_64_128_only"
|
| 58 |
+
},
|
| 59 |
+
"self_bleu": {
|
| 60 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 61 |
+
"completions_per_prompt": 5,
|
| 62 |
+
"ngram_order": 4,
|
| 63 |
+
"prompt_count": 256,
|
| 64 |
+
"sample_count": 1280,
|
| 65 |
+
"score": 0.026804049050975772,
|
| 66 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 67 |
+
"smoothing": "chen_cherry_method1",
|
| 68 |
+
"tokenization": "gpt2",
|
| 69 |
+
"weights": [
|
| 70 |
+
0.25,
|
| 71 |
+
0.25,
|
| 72 |
+
0.25,
|
| 73 |
+
0.25
|
| 74 |
+
]
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"prefix_length": 64,
|
| 78 |
+
"production_sample_count": 2048,
|
| 79 |
+
"protocol": "c64_zs_v1",
|
| 80 |
+
"requested_metrics": [
|
| 81 |
+
"conditional_gen_ppl",
|
| 82 |
+
"mauve_suffix",
|
| 83 |
+
"entropy",
|
| 84 |
+
"self_bleu",
|
| 85 |
+
"prefix_exact_match"
|
| 86 |
+
],
|
| 87 |
+
"sample_count": 2048,
|
| 88 |
+
"samples": "results/conditional/samples/owt/langflow/steps_4/samples.jsonl",
|
| 89 |
+
"samples_sha256": "17b7f1492f4deaf55844af231f118101a5d056b33c940ac2d6f985f748bc6917",
|
| 90 |
+
"schema_version": 1
|
| 91 |
+
}
|
results/conditional/metrics/owt/langflow/steps_8/metrics.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "owt",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 743.7936147872143,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 862068.1848144531,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 130384
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.0809010875072321,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [],
|
| 41 |
+
"logarithm": "natural",
|
| 42 |
+
"mean_entropy": 3.779973570713923,
|
| 43 |
+
"sample_count": 2048,
|
| 44 |
+
"token_count": 131072,
|
| 45 |
+
"unit": "nats"
|
| 46 |
+
},
|
| 47 |
+
"generated_minus_reference": -0.02907498763200067,
|
| 48 |
+
"reference": {
|
| 49 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 50 |
+
"excluded_token_ids": [],
|
| 51 |
+
"logarithm": "natural",
|
| 52 |
+
"mean_entropy": 3.8090485583459235,
|
| 53 |
+
"sample_count": 2048,
|
| 54 |
+
"token_count": 131072,
|
| 55 |
+
"unit": "nats"
|
| 56 |
+
},
|
| 57 |
+
"slice": "source_tokens_64_128_only"
|
| 58 |
+
},
|
| 59 |
+
"self_bleu": {
|
| 60 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 61 |
+
"completions_per_prompt": 5,
|
| 62 |
+
"ngram_order": 4,
|
| 63 |
+
"prompt_count": 256,
|
| 64 |
+
"sample_count": 1280,
|
| 65 |
+
"score": 0.0741745508033062,
|
| 66 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 67 |
+
"smoothing": "chen_cherry_method1",
|
| 68 |
+
"tokenization": "gpt2",
|
| 69 |
+
"weights": [
|
| 70 |
+
0.25,
|
| 71 |
+
0.25,
|
| 72 |
+
0.25,
|
| 73 |
+
0.25
|
| 74 |
+
]
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"prefix_length": 64,
|
| 78 |
+
"production_sample_count": 2048,
|
| 79 |
+
"protocol": "c64_zs_v1",
|
| 80 |
+
"requested_metrics": [
|
| 81 |
+
"conditional_gen_ppl",
|
| 82 |
+
"mauve_suffix",
|
| 83 |
+
"entropy",
|
| 84 |
+
"self_bleu",
|
| 85 |
+
"prefix_exact_match"
|
| 86 |
+
],
|
| 87 |
+
"sample_count": 2048,
|
| 88 |
+
"samples": "results/conditional/samples/owt/langflow/steps_8/samples.jsonl",
|
| 89 |
+
"samples_sha256": "05cbdc27690e89bb2ea511dfecbc700bbda688cb62396de028062f6c01e8a1c7",
|
| 90 |
+
"schema_version": 1
|
| 91 |
+
}
|
results/conditional/metrics/owt/mdlm/steps_1/metrics.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "owt",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 6026.51649395429,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 1126757.8327636719,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 129454
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.007340604203843546,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [],
|
| 41 |
+
"logarithm": "natural",
|
| 42 |
+
"mean_entropy": 3.8483674418303027,
|
| 43 |
+
"sample_count": 2048,
|
| 44 |
+
"token_count": 131072,
|
| 45 |
+
"unit": "nats"
|
| 46 |
+
},
|
| 47 |
+
"generated_minus_reference": 0.039318883484379175,
|
| 48 |
+
"reference": {
|
| 49 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 50 |
+
"excluded_token_ids": [],
|
| 51 |
+
"logarithm": "natural",
|
| 52 |
+
"mean_entropy": 3.8090485583459235,
|
| 53 |
+
"sample_count": 2048,
|
| 54 |
+
"token_count": 131072,
|
| 55 |
+
"unit": "nats"
|
| 56 |
+
},
|
| 57 |
+
"slice": "source_tokens_64_128_only"
|
| 58 |
+
},
|
| 59 |
+
"self_bleu": {
|
| 60 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 61 |
+
"completions_per_prompt": 5,
|
| 62 |
+
"ngram_order": 4,
|
| 63 |
+
"prompt_count": 256,
|
| 64 |
+
"sample_count": 1280,
|
| 65 |
+
"score": 0.011627875089846163,
|
| 66 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 67 |
+
"smoothing": "chen_cherry_method1",
|
| 68 |
+
"tokenization": "gpt2",
|
| 69 |
+
"weights": [
|
| 70 |
+
0.25,
|
| 71 |
+
0.25,
|
| 72 |
+
0.25,
|
| 73 |
+
0.25
|
| 74 |
+
]
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"prefix_length": 64,
|
| 78 |
+
"production_sample_count": 2048,
|
| 79 |
+
"protocol": "c64_zs_v1",
|
| 80 |
+
"requested_metrics": [
|
| 81 |
+
"conditional_gen_ppl",
|
| 82 |
+
"mauve_suffix",
|
| 83 |
+
"entropy",
|
| 84 |
+
"self_bleu",
|
| 85 |
+
"prefix_exact_match"
|
| 86 |
+
],
|
| 87 |
+
"sample_count": 2048,
|
| 88 |
+
"samples": "results/conditional/samples/owt/mdlm/steps_1/samples.jsonl",
|
| 89 |
+
"samples_sha256": "8f5fd59121babc7f14f15843a43a1cfa49e328ff67510121a5aa1e9eb523350e",
|
| 90 |
+
"schema_version": 1
|
| 91 |
+
}
|
results/conditional/metrics/owt/mdlm/steps_16/metrics.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "owt",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 238.9792015614064,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 715302.3962402344,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 130616
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.3945638372513164,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [],
|
| 41 |
+
"logarithm": "natural",
|
| 42 |
+
"mean_entropy": 3.8493230625291517,
|
| 43 |
+
"sample_count": 2048,
|
| 44 |
+
"token_count": 131072,
|
| 45 |
+
"unit": "nats"
|
| 46 |
+
},
|
| 47 |
+
"generated_minus_reference": 0.0402745041832282,
|
| 48 |
+
"reference": {
|
| 49 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 50 |
+
"excluded_token_ids": [],
|
| 51 |
+
"logarithm": "natural",
|
| 52 |
+
"mean_entropy": 3.8090485583459235,
|
| 53 |
+
"sample_count": 2048,
|
| 54 |
+
"token_count": 131072,
|
| 55 |
+
"unit": "nats"
|
| 56 |
+
},
|
| 57 |
+
"slice": "source_tokens_64_128_only"
|
| 58 |
+
},
|
| 59 |
+
"self_bleu": {
|
| 60 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 61 |
+
"completions_per_prompt": 5,
|
| 62 |
+
"ngram_order": 4,
|
| 63 |
+
"prompt_count": 256,
|
| 64 |
+
"sample_count": 1280,
|
| 65 |
+
"score": 0.029201508236315274,
|
| 66 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 67 |
+
"smoothing": "chen_cherry_method1",
|
| 68 |
+
"tokenization": "gpt2",
|
| 69 |
+
"weights": [
|
| 70 |
+
0.25,
|
| 71 |
+
0.25,
|
| 72 |
+
0.25,
|
| 73 |
+
0.25
|
| 74 |
+
]
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"prefix_length": 64,
|
| 78 |
+
"production_sample_count": 2048,
|
| 79 |
+
"protocol": "c64_zs_v1",
|
| 80 |
+
"requested_metrics": [
|
| 81 |
+
"conditional_gen_ppl",
|
| 82 |
+
"mauve_suffix",
|
| 83 |
+
"entropy",
|
| 84 |
+
"self_bleu",
|
| 85 |
+
"prefix_exact_match"
|
| 86 |
+
],
|
| 87 |
+
"sample_count": 2048,
|
| 88 |
+
"samples": "results/conditional/samples/owt/mdlm/steps_16/samples.jsonl",
|
| 89 |
+
"samples_sha256": "a3ba577015ff584b890e962ef1d23347224b7016f2bb1aa76463f8cca74eb3ab",
|
| 90 |
+
"schema_version": 1
|
| 91 |
+
}
|
results/conditional/metrics/owt/mdlm/steps_8/metrics.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "owt",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 490.7398287005558,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 807916.2199707031,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 130395
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.2025568730657235,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [],
|
| 41 |
+
"logarithm": "natural",
|
| 42 |
+
"mean_entropy": 3.856649545786545,
|
| 43 |
+
"sample_count": 2048,
|
| 44 |
+
"token_count": 131072,
|
| 45 |
+
"unit": "nats"
|
| 46 |
+
},
|
| 47 |
+
"generated_minus_reference": 0.047600987440621356,
|
| 48 |
+
"reference": {
|
| 49 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 50 |
+
"excluded_token_ids": [],
|
| 51 |
+
"logarithm": "natural",
|
| 52 |
+
"mean_entropy": 3.8090485583459235,
|
| 53 |
+
"sample_count": 2048,
|
| 54 |
+
"token_count": 131072,
|
| 55 |
+
"unit": "nats"
|
| 56 |
+
},
|
| 57 |
+
"slice": "source_tokens_64_128_only"
|
| 58 |
+
},
|
| 59 |
+
"self_bleu": {
|
| 60 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 61 |
+
"completions_per_prompt": 5,
|
| 62 |
+
"ngram_order": 4,
|
| 63 |
+
"prompt_count": 256,
|
| 64 |
+
"sample_count": 1280,
|
| 65 |
+
"score": 0.022621931775717988,
|
| 66 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 67 |
+
"smoothing": "chen_cherry_method1",
|
| 68 |
+
"tokenization": "gpt2",
|
| 69 |
+
"weights": [
|
| 70 |
+
0.25,
|
| 71 |
+
0.25,
|
| 72 |
+
0.25,
|
| 73 |
+
0.25
|
| 74 |
+
]
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"prefix_length": 64,
|
| 78 |
+
"production_sample_count": 2048,
|
| 79 |
+
"protocol": "c64_zs_v1",
|
| 80 |
+
"requested_metrics": [
|
| 81 |
+
"conditional_gen_ppl",
|
| 82 |
+
"mauve_suffix",
|
| 83 |
+
"entropy",
|
| 84 |
+
"self_bleu",
|
| 85 |
+
"prefix_exact_match"
|
| 86 |
+
],
|
| 87 |
+
"sample_count": 2048,
|
| 88 |
+
"samples": "results/conditional/samples/owt/mdlm/steps_8/samples.jsonl",
|
| 89 |
+
"samples_sha256": "6c1a088434089774758c8dc098901b835f4028c3550c75a0cfbf02c9b4ead141",
|
| 90 |
+
"schema_version": 1
|
| 91 |
+
}
|
results/conditional/metrics/owt/mdlm_di4c/steps_1/metrics.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "owt",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 2791.4371557642853,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 1025216.2373046875,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 129213
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.008204959866683138,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [],
|
| 41 |
+
"logarithm": "natural",
|
| 42 |
+
"mean_entropy": 3.7424604010263938,
|
| 43 |
+
"sample_count": 2048,
|
| 44 |
+
"token_count": 131072,
|
| 45 |
+
"unit": "nats"
|
| 46 |
+
},
|
| 47 |
+
"generated_minus_reference": -0.06658815731952972,
|
| 48 |
+
"reference": {
|
| 49 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 50 |
+
"excluded_token_ids": [],
|
| 51 |
+
"logarithm": "natural",
|
| 52 |
+
"mean_entropy": 3.8090485583459235,
|
| 53 |
+
"sample_count": 2048,
|
| 54 |
+
"token_count": 131072,
|
| 55 |
+
"unit": "nats"
|
| 56 |
+
},
|
| 57 |
+
"slice": "source_tokens_64_128_only"
|
| 58 |
+
},
|
| 59 |
+
"self_bleu": {
|
| 60 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 61 |
+
"completions_per_prompt": 5,
|
| 62 |
+
"ngram_order": 4,
|
| 63 |
+
"prompt_count": 256,
|
| 64 |
+
"sample_count": 1280,
|
| 65 |
+
"score": 0.017216425389052357,
|
| 66 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 67 |
+
"smoothing": "chen_cherry_method1",
|
| 68 |
+
"tokenization": "gpt2",
|
| 69 |
+
"weights": [
|
| 70 |
+
0.25,
|
| 71 |
+
0.25,
|
| 72 |
+
0.25,
|
| 73 |
+
0.25
|
| 74 |
+
]
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"prefix_length": 64,
|
| 78 |
+
"production_sample_count": 2048,
|
| 79 |
+
"protocol": "c64_zs_v1",
|
| 80 |
+
"requested_metrics": [
|
| 81 |
+
"conditional_gen_ppl",
|
| 82 |
+
"mauve_suffix",
|
| 83 |
+
"entropy",
|
| 84 |
+
"self_bleu",
|
| 85 |
+
"prefix_exact_match"
|
| 86 |
+
],
|
| 87 |
+
"sample_count": 2048,
|
| 88 |
+
"samples": "results/conditional/samples/owt/mdlm_di4c/steps_1/samples.jsonl",
|
| 89 |
+
"samples_sha256": "a2304bde56c71ed6adbacb4870d1e98ef236a2f59ab3f9b92e58ce56a494e60d",
|
| 90 |
+
"schema_version": 1
|
| 91 |
+
}
|
results/conditional/metrics/owt/mdlm_di4c/steps_2/metrics.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "owt",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 944.5300897856437,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 889739.8955078125,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 129876
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.02081879824149785,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [],
|
| 41 |
+
"logarithm": "natural",
|
| 42 |
+
"mean_entropy": 3.750602440233099,
|
| 43 |
+
"sample_count": 2048,
|
| 44 |
+
"token_count": 131072,
|
| 45 |
+
"unit": "nats"
|
| 46 |
+
},
|
| 47 |
+
"generated_minus_reference": -0.05844611811282441,
|
| 48 |
+
"reference": {
|
| 49 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 50 |
+
"excluded_token_ids": [],
|
| 51 |
+
"logarithm": "natural",
|
| 52 |
+
"mean_entropy": 3.8090485583459235,
|
| 53 |
+
"sample_count": 2048,
|
| 54 |
+
"token_count": 131072,
|
| 55 |
+
"unit": "nats"
|
| 56 |
+
},
|
| 57 |
+
"slice": "source_tokens_64_128_only"
|
| 58 |
+
},
|
| 59 |
+
"self_bleu": {
|
| 60 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 61 |
+
"completions_per_prompt": 5,
|
| 62 |
+
"ngram_order": 4,
|
| 63 |
+
"prompt_count": 256,
|
| 64 |
+
"sample_count": 1280,
|
| 65 |
+
"score": 0.018530838581378082,
|
| 66 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 67 |
+
"smoothing": "chen_cherry_method1",
|
| 68 |
+
"tokenization": "gpt2",
|
| 69 |
+
"weights": [
|
| 70 |
+
0.25,
|
| 71 |
+
0.25,
|
| 72 |
+
0.25,
|
| 73 |
+
0.25
|
| 74 |
+
]
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"prefix_length": 64,
|
| 78 |
+
"production_sample_count": 2048,
|
| 79 |
+
"protocol": "c64_zs_v1",
|
| 80 |
+
"requested_metrics": [
|
| 81 |
+
"conditional_gen_ppl",
|
| 82 |
+
"mauve_suffix",
|
| 83 |
+
"entropy",
|
| 84 |
+
"self_bleu",
|
| 85 |
+
"prefix_exact_match"
|
| 86 |
+
],
|
| 87 |
+
"sample_count": 2048,
|
| 88 |
+
"samples": "results/conditional/samples/owt/mdlm_di4c/steps_2/samples.jsonl",
|
| 89 |
+
"samples_sha256": "8b3357cc2acf13dbd47cc4f3db9684af5d5e79a580a8ef9106a159c4dad1fb1c",
|
| 90 |
+
"schema_version": 1
|
| 91 |
+
}
|
results/conditional/metrics/owt/mdlm_sdtt/steps_1/metrics.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "owt",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 2770.0619448869115,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 1022851.6850585938,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 129040
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.0073559087882152305,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [],
|
| 41 |
+
"logarithm": "natural",
|
| 42 |
+
"mean_entropy": 3.737069846035564,
|
| 43 |
+
"sample_count": 2048,
|
| 44 |
+
"token_count": 131072,
|
| 45 |
+
"unit": "nats"
|
| 46 |
+
},
|
| 47 |
+
"generated_minus_reference": -0.07197871231035968,
|
| 48 |
+
"reference": {
|
| 49 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 50 |
+
"excluded_token_ids": [],
|
| 51 |
+
"logarithm": "natural",
|
| 52 |
+
"mean_entropy": 3.8090485583459235,
|
| 53 |
+
"sample_count": 2048,
|
| 54 |
+
"token_count": 131072,
|
| 55 |
+
"unit": "nats"
|
| 56 |
+
},
|
| 57 |
+
"slice": "source_tokens_64_128_only"
|
| 58 |
+
},
|
| 59 |
+
"self_bleu": {
|
| 60 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 61 |
+
"completions_per_prompt": 5,
|
| 62 |
+
"ngram_order": 4,
|
| 63 |
+
"prompt_count": 256,
|
| 64 |
+
"sample_count": 1280,
|
| 65 |
+
"score": 0.01637466504973282,
|
| 66 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 67 |
+
"smoothing": "chen_cherry_method1",
|
| 68 |
+
"tokenization": "gpt2",
|
| 69 |
+
"weights": [
|
| 70 |
+
0.25,
|
| 71 |
+
0.25,
|
| 72 |
+
0.25,
|
| 73 |
+
0.25
|
| 74 |
+
]
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"prefix_length": 64,
|
| 78 |
+
"production_sample_count": 2048,
|
| 79 |
+
"protocol": "c64_zs_v1",
|
| 80 |
+
"requested_metrics": [
|
| 81 |
+
"conditional_gen_ppl",
|
| 82 |
+
"mauve_suffix",
|
| 83 |
+
"entropy",
|
| 84 |
+
"self_bleu",
|
| 85 |
+
"prefix_exact_match"
|
| 86 |
+
],
|
| 87 |
+
"sample_count": 2048,
|
| 88 |
+
"samples": "results/conditional/samples/owt/mdlm_sdtt/steps_1/samples.jsonl",
|
| 89 |
+
"samples_sha256": "6a08d4b51b749692546e37cb4c05a6fb5eada2a2dbf59063fc97edeeb49e6d53",
|
| 90 |
+
"schema_version": 1
|
| 91 |
+
}
|
results/conditional/metrics/owt/mdlm_sdtt/steps_2/metrics.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "owt",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 1077.2700142678389,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 905889.6745605469,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 129743
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.013707248537549552,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [],
|
| 41 |
+
"logarithm": "natural",
|
| 42 |
+
"mean_entropy": 3.745571857497006,
|
| 43 |
+
"sample_count": 2048,
|
| 44 |
+
"token_count": 131072,
|
| 45 |
+
"unit": "nats"
|
| 46 |
+
},
|
| 47 |
+
"generated_minus_reference": -0.06347670084891766,
|
| 48 |
+
"reference": {
|
| 49 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 50 |
+
"excluded_token_ids": [],
|
| 51 |
+
"logarithm": "natural",
|
| 52 |
+
"mean_entropy": 3.8090485583459235,
|
| 53 |
+
"sample_count": 2048,
|
| 54 |
+
"token_count": 131072,
|
| 55 |
+
"unit": "nats"
|
| 56 |
+
},
|
| 57 |
+
"slice": "source_tokens_64_128_only"
|
| 58 |
+
},
|
| 59 |
+
"self_bleu": {
|
| 60 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 61 |
+
"completions_per_prompt": 5,
|
| 62 |
+
"ngram_order": 4,
|
| 63 |
+
"prompt_count": 256,
|
| 64 |
+
"sample_count": 1280,
|
| 65 |
+
"score": 0.016799458482196977,
|
| 66 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 67 |
+
"smoothing": "chen_cherry_method1",
|
| 68 |
+
"tokenization": "gpt2",
|
| 69 |
+
"weights": [
|
| 70 |
+
0.25,
|
| 71 |
+
0.25,
|
| 72 |
+
0.25,
|
| 73 |
+
0.25
|
| 74 |
+
]
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"prefix_length": 64,
|
| 78 |
+
"production_sample_count": 2048,
|
| 79 |
+
"protocol": "c64_zs_v1",
|
| 80 |
+
"requested_metrics": [
|
| 81 |
+
"conditional_gen_ppl",
|
| 82 |
+
"mauve_suffix",
|
| 83 |
+
"entropy",
|
| 84 |
+
"self_bleu",
|
| 85 |
+
"prefix_exact_match"
|
| 86 |
+
],
|
| 87 |
+
"sample_count": 2048,
|
| 88 |
+
"samples": "results/conditional/samples/owt/mdlm_sdtt/steps_2/samples.jsonl",
|
| 89 |
+
"samples_sha256": "f3653ac76b1576c5ab5f7b21d958de6c00e11a107a56082d564f052918296ca4",
|
| 90 |
+
"schema_version": 1
|
| 91 |
+
}
|
results/conditional/metrics/owt/mdlm_sdtt/steps_32/metrics.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "owt",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 43.943780683780034,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 495179.28076171875,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 130899
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.5207367362678341,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [],
|
| 41 |
+
"logarithm": "natural",
|
| 42 |
+
"mean_entropy": 3.737161209473734,
|
| 43 |
+
"sample_count": 2048,
|
| 44 |
+
"token_count": 131072,
|
| 45 |
+
"unit": "nats"
|
| 46 |
+
},
|
| 47 |
+
"generated_minus_reference": -0.07188734887218962,
|
| 48 |
+
"reference": {
|
| 49 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 50 |
+
"excluded_token_ids": [],
|
| 51 |
+
"logarithm": "natural",
|
| 52 |
+
"mean_entropy": 3.8090485583459235,
|
| 53 |
+
"sample_count": 2048,
|
| 54 |
+
"token_count": 131072,
|
| 55 |
+
"unit": "nats"
|
| 56 |
+
},
|
| 57 |
+
"slice": "source_tokens_64_128_only"
|
| 58 |
+
},
|
| 59 |
+
"self_bleu": {
|
| 60 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 61 |
+
"completions_per_prompt": 5,
|
| 62 |
+
"ngram_order": 4,
|
| 63 |
+
"prompt_count": 256,
|
| 64 |
+
"sample_count": 1280,
|
| 65 |
+
"score": 0.05084042935920519,
|
| 66 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 67 |
+
"smoothing": "chen_cherry_method1",
|
| 68 |
+
"tokenization": "gpt2",
|
| 69 |
+
"weights": [
|
| 70 |
+
0.25,
|
| 71 |
+
0.25,
|
| 72 |
+
0.25,
|
| 73 |
+
0.25
|
| 74 |
+
]
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"prefix_length": 64,
|
| 78 |
+
"production_sample_count": 2048,
|
| 79 |
+
"protocol": "c64_zs_v1",
|
| 80 |
+
"requested_metrics": [
|
| 81 |
+
"conditional_gen_ppl",
|
| 82 |
+
"mauve_suffix",
|
| 83 |
+
"entropy",
|
| 84 |
+
"self_bleu",
|
| 85 |
+
"prefix_exact_match"
|
| 86 |
+
],
|
| 87 |
+
"sample_count": 2048,
|
| 88 |
+
"samples": "results/conditional/samples/owt/mdlm_sdtt/steps_32/samples.jsonl",
|
| 89 |
+
"samples_sha256": "8e619930d2928cf4b94b9b5e3ca22c58193fd8041f73efbbe00ddd74a6688804",
|
| 90 |
+
"schema_version": 1
|
| 91 |
+
}
|
results/conditional/metrics/owt/mdlm_sdtt/steps_8/metrics.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "owt",
|
| 3 |
+
"evaluation_continuation_length": 64,
|
| 4 |
+
"metrics": {
|
| 5 |
+
"conditional_generative_perplexity": {
|
| 6 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"context_length": 1024,
|
| 9 |
+
"model_id": "gpt2-large",
|
| 10 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 11 |
+
"model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
|
| 12 |
+
"offline": true,
|
| 13 |
+
"perplexity": 102.52068892439462,
|
| 14 |
+
"prompt_loss_excluded": true,
|
| 15 |
+
"sample_count": 2048,
|
| 16 |
+
"scored_slice": "source_tokens_64_128_only",
|
| 17 |
+
"tokenizer_id": "gpt2",
|
| 18 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 19 |
+
"tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 20 |
+
"total_nll": 605038.3244628906,
|
| 21 |
+
"truncation": "right",
|
| 22 |
+
"valid_token_count": 130676
|
| 23 |
+
},
|
| 24 |
+
"mauve_suffix": {
|
| 25 |
+
"comparison": "generated_suffix_vs_reference_suffix",
|
| 26 |
+
"prefix_included": false,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"score": 0.4434890512136005,
|
| 29 |
+
"slice": "source_tokens_64_128_only"
|
| 30 |
+
},
|
| 31 |
+
"prefix_exact_match": {
|
| 32 |
+
"matched": 2048,
|
| 33 |
+
"rate": 1.0,
|
| 34 |
+
"required_rate": 1.0,
|
| 35 |
+
"sample_count": 2048
|
| 36 |
+
},
|
| 37 |
+
"sample_entropy": {
|
| 38 |
+
"generated": {
|
| 39 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 40 |
+
"excluded_token_ids": [],
|
| 41 |
+
"logarithm": "natural",
|
| 42 |
+
"mean_entropy": 3.752913234833009,
|
| 43 |
+
"sample_count": 2048,
|
| 44 |
+
"token_count": 131072,
|
| 45 |
+
"unit": "nats"
|
| 46 |
+
},
|
| 47 |
+
"generated_minus_reference": -0.0561353235129145,
|
| 48 |
+
"reference": {
|
| 49 |
+
"aggregation": "arithmetic_mean_of_per_sample_entropy",
|
| 50 |
+
"excluded_token_ids": [],
|
| 51 |
+
"logarithm": "natural",
|
| 52 |
+
"mean_entropy": 3.8090485583459235,
|
| 53 |
+
"sample_count": 2048,
|
| 54 |
+
"token_count": 131072,
|
| 55 |
+
"unit": "nats"
|
| 56 |
+
},
|
| 57 |
+
"slice": "source_tokens_64_128_only"
|
| 58 |
+
},
|
| 59 |
+
"self_bleu": {
|
| 60 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 61 |
+
"completions_per_prompt": 5,
|
| 62 |
+
"ngram_order": 4,
|
| 63 |
+
"prompt_count": 256,
|
| 64 |
+
"sample_count": 1280,
|
| 65 |
+
"score": 0.03526571529695085,
|
| 66 |
+
"slice": "generated_source_tokens_64_128_only",
|
| 67 |
+
"smoothing": "chen_cherry_method1",
|
| 68 |
+
"tokenization": "gpt2",
|
| 69 |
+
"weights": [
|
| 70 |
+
0.25,
|
| 71 |
+
0.25,
|
| 72 |
+
0.25,
|
| 73 |
+
0.25
|
| 74 |
+
]
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"prefix_length": 64,
|
| 78 |
+
"production_sample_count": 2048,
|
| 79 |
+
"protocol": "c64_zs_v1",
|
| 80 |
+
"requested_metrics": [
|
| 81 |
+
"conditional_gen_ppl",
|
| 82 |
+
"mauve_suffix",
|
| 83 |
+
"entropy",
|
| 84 |
+
"self_bleu",
|
| 85 |
+
"prefix_exact_match"
|
| 86 |
+
],
|
| 87 |
+
"sample_count": 2048,
|
| 88 |
+
"samples": "results/conditional/samples/owt/mdlm_sdtt/steps_8/samples.jsonl",
|
| 89 |
+
"samples_sha256": "63b33a5acbc361179d98a87a4a5c4a9aec81e0cd9abff99bde56a13531e6eb81",
|
| 90 |
+
"schema_version": 1
|
| 91 |
+
}
|
results/conditional/samples/lm1b/duo/steps_1/stdout.log
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
CONFIG
|
| 2 |
+
├── mode
|
| 3 |
+
│ └── sample_eval
|
| 4 |
+
├── seed
|
| 5 |
+
│ └── 42
|
| 6 |
+
├── loader
|
| 7 |
+
│ └── global_batch_size: 512
|
| 8 |
+
│ eval_global_batch_size: 512
|
| 9 |
+
│ batch_size: 2
|
| 10 |
+
│ eval_batch_size: 64
|
| 11 |
+
│ num_workers: 512
|
| 12 |
+
│ pin_memory: true
|
| 13 |
+
│
|
| 14 |
+
├── sampling
|
| 15 |
+
│ └── predictor: ancestral
|
| 16 |
+
│ steps: 1
|
| 17 |
+
│ noise_removal: ancestral
|
| 18 |
+
│ use_float64: true
|
| 19 |
+
│ p_nucleus: 1.0
|
| 20 |
+
│ num_sample_batches: 32
|
| 21 |
+
│ num_sample_log: 2
|
| 22 |
+
│ semi_ar: false
|
| 23 |
+
│ stride_length: 1
|
| 24 |
+
│ num_strides: 1
|
| 25 |
+
│ guid_weight: null
|
| 26 |
+
│ psi:
|
| 27 |
+
│ time_profile: linear
|
| 28 |
+
│ high_mode: pure-posterior
|
| 29 |
+
│ middle_mode: pure-posterior
|
| 30 |
+
│ low_mode: pure-posterior
|
| 31 |
+
│ high_frac: 0.2
|
| 32 |
+
│ middle_frac: 0.6
|
| 33 |
+
│
|
| 34 |
+
├── training
|
| 35 |
+
│ └── ema: 0.9999
|
| 36 |
+
│ antithetic_sampling: true
|
| 37 |
+
│ importance_sampling: false
|
| 38 |
+
│ sampling_eps: 0.001
|
| 39 |
+
│ change_of_variables: false
|
| 40 |
+
│ loss_precision: bf16
|
| 41 |
+
│ finetune_path: ''
|
| 42 |
+
│ class_dropout_p: 0.1
|
| 43 |
+
│
|
| 44 |
+
├── eval
|
| 45 |
+
│ └── checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference
|
| 46 |
+
│ disable_ema: false
|
| 47 |
+
│ compute_generative_perplexity: false
|
| 48 |
+
│ perplexity_batch_size: 8
|
| 49 |
+
│ compute_perplexity_on_sanity: false
|
| 50 |
+
│ gen_ppl_eval_model_name_or_path: gpt2-large
|
| 51 |
+
│ generate_samples: true
|
| 52 |
+
│ generated_samples_path: /home/ruizzhao/diffusion_baseline/results/condit
|
| 53 |
+
│
|
| 54 |
+
├── optim
|
| 55 |
+
│ └── weight_decay: 0
|
| 56 |
+
│ lr: 0.0003
|
| 57 |
+
│ beta1: 0.9
|
| 58 |
+
│ beta2: 0.999
|
| 59 |
+
│ eps: 1.0e-08
|
| 60 |
+
│
|
| 61 |
+
├── trainer
|
| 62 |
+
│ └── _target_: lightning.Trainer
|
| 63 |
+
│ accelerator: cuda
|
| 64 |
+
│ num_nodes: 1
|
| 65 |
+
│ devices: 1
|
| 66 |
+
│ accumulate_grad_batches: 256
|
| 67 |
+
│ gradient_clip_val: 1.0
|
| 68 |
+
│ precision: bf16
|
| 69 |
+
│ num_sanity_val_steps: 2
|
| 70 |
+
│ max_steps: 1000000
|
| 71 |
+
│ log_every_n_steps: 100
|
| 72 |
+
│ limit_train_batches: 1.0
|
| 73 |
+
│ limit_val_batches: 1.0
|
| 74 |
+
│ val_check_interval: 5000
|
| 75 |
+
│ check_val_every_n_epoch: null
|
| 76 |
+
│
|
| 77 |
+
├── wandb
|
| 78 |
+
│ └── project: duo
|
| 79 |
+
│ group: null
|
| 80 |
+
│ job_type: null
|
| 81 |
+
│ name: null
|
| 82 |
+
│ id: None_42
|
| 83 |
+
│ tags:
|
| 84 |
+
│ - log-linear
|
| 85 |
+
│ - lm1b
|
| 86 |
+
│ - lm1b
|
| 87 |
+
│ - duo_base
|
| 88 |
+
│ offline: true
|
| 89 |
+
│
|
| 90 |
+
├── checkpointing
|
| 91 |
+
│ └── save_dir: /home/ruizzhao/diffusion_baseline/results/conditional/samples/
|
| 92 |
+
│ resume_from_ckpt: true
|
| 93 |
+
│ resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results/conditional/
|
| 94 |
+
│
|
| 95 |
+
├── callbacks
|
| 96 |
+
│ └── checkpoint_every_n_steps:
|
| 97 |
+
│ _target_: lightning.pytorch.callbacks.ModelCheckpoint
|
| 98 |
+
│ save_top_k: -1
|
| 99 |
+
│ save_last: true
|
| 100 |
+
│ dirpath: /home/ruizzhao/diffusion_baseline/results/conditional/samples
|
| 101 |
+
│ verbose: true
|
| 102 |
+
│ auto_insert_metric_name: false
|
| 103 |
+
│ every_n_train_steps: 500
|
| 104 |
+
│ checkpoint_monitor:
|
| 105 |
+
│ _target_: lightning.pytorch.callbacks.ModelCheckpoint
|
| 106 |
+
│ monitor: val/nll
|
| 107 |
+
│ mode: min
|
| 108 |
+
│ save_top_k: 1
|
| 109 |
+
│ save_last: false
|
| 110 |
+
│ dirpath: /home/ruizzhao/diffusion_baseline/results/conditional/samples
|
| 111 |
+
│ filename: best
|
| 112 |
+
│ auto_insert_metric_name: false
|
| 113 |
+
│ verbose: true
|
| 114 |
+
│ learning_rate_monitor:
|
| 115 |
+
│ _target_: lightning.pytorch.callbacks.LearningRateMonitor
|
| 116 |
+
│ logging_interval: step
|
| 117 |
+
│
|
| 118 |
+
├── data
|
| 119 |
+
│ └── train: lm1b
|
| 120 |
+
│ valid: lm1b
|
| 121 |
+
│ modality: text
|
| 122 |
+
│ tokenizer_name_or_path: bert-base-uncased
|
| 123 |
+
│ cache_dir: /share/kuleshov/ssahoo/textdiffusion/data
|
| 124 |
+
│ wrap: true
|
| 125 |
+
│ streaming: false
|
| 126 |
+
│ insert_train_eos: true
|
| 127 |
+
│ insert_valid_eos: true
|
| 128 |
+
│
|
| 129 |
+
├── model
|
| 130 |
+
│ └── name: small
|
| 131 |
+
│ type: ddit
|
| 132 |
+
│ hidden_size: 768
|
| 133 |
+
│ cond_dim: 128
|
| 134 |
+
│ length: 128
|
| 135 |
+
│ n_blocks: 12
|
| 136 |
+
│ n_heads: 12
|
| 137 |
+
│ scale_by_sigma: true
|
| 138 |
+
│ dropout: 0.1
|
| 139 |
+
│ tie_word_embeddings: false
|
| 140 |
+
│ vocab_lookup: true
|
| 141 |
+
│
|
| 142 |
+
├── strategy
|
| 143 |
+
│ └── _target_: lightning.pytorch.strategies.DDPStrategy
|
| 144 |
+
│ find_unused_parameters: false
|
| 145 |
+
│
|
| 146 |
+
├── noise
|
| 147 |
+
│ └── type: log-linear
|
| 148 |
+
│ eps: 0.001
|
| 149 |
+
│
|
| 150 |
+
├── lr_scheduler
|
| 151 |
+
│ └── _target_: transformers.get_constant_schedule_with_warmup
|
| 152 |
+
│ num_warmup_steps: 2500
|
| 153 |
+
│
|
| 154 |
+
├── prior
|
| 155 |
+
│ └── type: none
|
| 156 |
+
│ latent_width: 0
|
| 157 |
+
│ latent_height: 0
|
| 158 |
+
│
|
| 159 |
+
└── algo
|
| 160 |
+
└── name: duo_base
|
| 161 |
+
backbone: dit
|
| 162 |
+
parameterization: mean
|
| 163 |
+
time_conditioning: true
|
| 164 |
+
T: 0
|
| 165 |
+
subs_masking: false
|
| 166 |
+
causal_attention: false
|
| 167 |
+
ignore_bos: false
|
| 168 |
+
loss_type: elbo
|
| 169 |
+
|
| 170 |
+
[2026-08-25 19:27:07,998][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
|
| 171 |
+
[2026-08-25 19:28:28,464][dlb_pinned_upstream_main][INFO] - Generative perplexity: 269.1194152832031
|
| 172 |
+
[2026-08-25 19:28:28,465][dlb_pinned_upstream_main][INFO] - Sample entropy: 4.3520827293396
|
| 173 |
+
[2026-08-25 19:28:28,468][dlb_pinned_upstream_main][INFO] - Samples saved at: /home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_1/upstream_samples.json
|
results/conditional/samples/lm1b/duo/steps_1/upstream_hydra/config_tree.txt
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
CONFIG
|
| 2 |
+
├── mode
|
| 3 |
+
│ └── sample_eval
|
| 4 |
+
├── seed
|
| 5 |
+
│ └── 42
|
| 6 |
+
├── loader
|
| 7 |
+
│ └── global_batch_size: 512
|
| 8 |
+
│ eval_global_batch_size: 512
|
| 9 |
+
│ batch_size: 2
|
| 10 |
+
│ eval_batch_size: 1
|
| 11 |
+
│ num_workers: 512
|
| 12 |
+
│ pin_memory: true
|
| 13 |
+
│
|
| 14 |
+
├── sampling
|
| 15 |
+
│ └── predictor: ancestral
|
| 16 |
+
│ steps: 1
|
| 17 |
+
│ noise_removal: ancestral
|
| 18 |
+
│ use_float64: true
|
| 19 |
+
│ p_nucleus: 1.0
|
| 20 |
+
│ num_sample_batches: 1
|
| 21 |
+
│ num_sample_log: 2
|
| 22 |
+
│ semi_ar: false
|
| 23 |
+
│ stride_length: 1
|
| 24 |
+
│ num_strides: 1
|
| 25 |
+
│ guid_weight: null
|
| 26 |
+
│ psi:
|
| 27 |
+
│ time_profile: linear
|
| 28 |
+
│ high_mode: pure-posterior
|
| 29 |
+
│ middle_mode: pure-posterior
|
| 30 |
+
│ low_mode: pure-posterior
|
| 31 |
+
│ high_frac: 0.2
|
| 32 |
+
│ middle_frac: 0.6
|
| 33 |
+
│
|
| 34 |
+
├── training
|
| 35 |
+
│ └── ema: 0.9999
|
| 36 |
+
│ antithetic_sampling: true
|
| 37 |
+
│ importance_sampling: false
|
| 38 |
+
│ sampling_eps: 0.001
|
| 39 |
+
│ change_of_variables: false
|
| 40 |
+
│ loss_precision: bf16
|
| 41 |
+
│ finetune_path: ''
|
| 42 |
+
│ class_dropout_p: 0.1
|
| 43 |
+
│
|
| 44 |
+
├── eval
|
| 45 |
+
│ └── checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference
|
| 46 |
+
│ disable_ema: false
|
| 47 |
+
│ compute_generative_perplexity: false
|
| 48 |
+
│ perplexity_batch_size: 8
|
| 49 |
+
│ compute_perplexity_on_sanity: false
|
| 50 |
+
│ gen_ppl_eval_model_name_or_path: gpt2-large
|
| 51 |
+
│ generate_samples: true
|
| 52 |
+
│ generated_samples_path: /home/ruizzhao/diffusion_baseline/results/condit
|
| 53 |
+
│
|
| 54 |
+
├── optim
|
| 55 |
+
│ └── weight_decay: 0
|
| 56 |
+
│ lr: 0.0003
|
| 57 |
+
│ beta1: 0.9
|
| 58 |
+
│ beta2: 0.999
|
| 59 |
+
│ eps: 1.0e-08
|
| 60 |
+
│
|
| 61 |
+
├── trainer
|
| 62 |
+
│ └── _target_: lightning.Trainer
|
| 63 |
+
│ accelerator: cuda
|
| 64 |
+
│ num_nodes: 1
|
| 65 |
+
│ devices: 1
|
| 66 |
+
│ accumulate_grad_batches: 256
|
| 67 |
+
│ gradient_clip_val: 1.0
|
| 68 |
+
│ precision: bf16
|
| 69 |
+
│ num_sanity_val_steps: 2
|
| 70 |
+
│ max_steps: 1000000
|
| 71 |
+
│ log_every_n_steps: 100
|
| 72 |
+
│ limit_train_batches: 1.0
|
| 73 |
+
│ limit_val_batches: 1.0
|
| 74 |
+
│ val_check_interval: 5000
|
| 75 |
+
│ check_val_every_n_epoch: null
|
| 76 |
+
│
|
| 77 |
+
├── wandb
|
| 78 |
+
│ └── project: duo
|
| 79 |
+
│ group: null
|
| 80 |
+
│ job_type: null
|
| 81 |
+
│ name: null
|
| 82 |
+
│ id: None_42
|
| 83 |
+
│ tags:
|
| 84 |
+
│ - log-linear
|
| 85 |
+
│ - lm1b
|
| 86 |
+
│ - lm1b
|
| 87 |
+
│ - duo_base
|
| 88 |
+
│ offline: true
|
| 89 |
+
│
|
| 90 |
+
├── checkpointing
|
| 91 |
+
│ └── save_dir: /home/ruizzhao/diffusion_baseline/results/conditional/samples/
|
| 92 |
+
│ resume_from_ckpt: true
|
| 93 |
+
│ resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results/conditional/
|
| 94 |
+
│
|
| 95 |
+
├── callbacks
|
| 96 |
+
│ └── checkpoint_every_n_steps:
|
| 97 |
+
│ _target_: lightning.pytorch.callbacks.ModelCheckpoint
|
| 98 |
+
│ save_top_k: -1
|
| 99 |
+
│ save_last: true
|
| 100 |
+
│ dirpath: /home/ruizzhao/diffusion_baseline/results/conditional/samples
|
| 101 |
+
│ verbose: true
|
| 102 |
+
│ auto_insert_metric_name: false
|
| 103 |
+
│ every_n_train_steps: 500
|
| 104 |
+
│ checkpoint_monitor:
|
| 105 |
+
│ _target_: lightning.pytorch.callbacks.ModelCheckpoint
|
| 106 |
+
│ monitor: val/nll
|
| 107 |
+
│ mode: min
|
| 108 |
+
│ save_top_k: 1
|
| 109 |
+
│ save_last: false
|
| 110 |
+
│ dirpath: /home/ruizzhao/diffusion_baseline/results/conditional/samples
|
| 111 |
+
│ filename: best
|
| 112 |
+
│ auto_insert_metric_name: false
|
| 113 |
+
│ verbose: true
|
| 114 |
+
│ learning_rate_monitor:
|
| 115 |
+
│ _target_: lightning.pytorch.callbacks.LearningRateMonitor
|
| 116 |
+
│ logging_interval: step
|
| 117 |
+
│
|
| 118 |
+
├── data
|
| 119 |
+
│ └── train: lm1b
|
| 120 |
+
│ valid: lm1b
|
| 121 |
+
│ modality: text
|
| 122 |
+
│ tokenizer_name_or_path: bert-base-uncased
|
| 123 |
+
│ cache_dir: /share/kuleshov/ssahoo/textdiffusion/data
|
| 124 |
+
│ wrap: true
|
| 125 |
+
│ streaming: false
|
| 126 |
+
│ insert_train_eos: true
|
| 127 |
+
│ insert_valid_eos: true
|
| 128 |
+
│
|
| 129 |
+
├── model
|
| 130 |
+
│ └── name: small
|
| 131 |
+
│ type: ddit
|
| 132 |
+
│ hidden_size: 768
|
| 133 |
+
│ cond_dim: 128
|
| 134 |
+
│ length: 128
|
| 135 |
+
│ n_blocks: 12
|
| 136 |
+
│ n_heads: 12
|
| 137 |
+
│ scale_by_sigma: true
|
| 138 |
+
│ dropout: 0.1
|
| 139 |
+
│ tie_word_embeddings: false
|
| 140 |
+
│ vocab_lookup: true
|
| 141 |
+
│
|
| 142 |
+
├── strategy
|
| 143 |
+
│ └── _target_: lightning.pytorch.strategies.DDPStrategy
|
| 144 |
+
│ find_unused_parameters: false
|
| 145 |
+
│
|
| 146 |
+
├── noise
|
| 147 |
+
│ └── type: log-linear
|
| 148 |
+
│ eps: 0.001
|
| 149 |
+
│
|
| 150 |
+
├── lr_scheduler
|
| 151 |
+
│ └── _target_: transformers.get_constant_schedule_with_warmup
|
| 152 |
+
│ num_warmup_steps: 2500
|
| 153 |
+
│
|
| 154 |
+
├── prior
|
| 155 |
+
│ └── type: none
|
| 156 |
+
│ latent_width: 0
|
| 157 |
+
│ latent_height: 0
|
| 158 |
+
│
|
| 159 |
+
└── algo
|
| 160 |
+
└── name: duo_base
|
| 161 |
+
backbone: dit
|
| 162 |
+
parameterization: mean
|
| 163 |
+
time_conditioning: true
|
| 164 |
+
T: 0
|
| 165 |
+
subs_masking: false
|
| 166 |
+
causal_attention: false
|
| 167 |
+
ignore_bos: false
|
| 168 |
+
loss_type: elbo
|
| 169 |
+
|
results/conditional/samples/lm1b/duo/steps_1024/request.json
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"command": [
|
| 3 |
+
"/opt/anaconda3/envs/dlb-duo/bin/python",
|
| 4 |
+
"-u",
|
| 5 |
+
"-m",
|
| 6 |
+
"dlb.adapters.capture",
|
| 7 |
+
"--upstream-entrypoint=/home/ruizzhao/diffusion_baseline/upstreams/duo/main.py",
|
| 8 |
+
"--capture-path=/home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_1024/upstream_token_ids.json",
|
| 9 |
+
"--generation-mode=conditional_prefix",
|
| 10 |
+
"--conditioning-manifest=/home/ruizzhao/diffusion_baseline/data/manifests/conditional-lm1b-c64.json",
|
| 11 |
+
"--conditioning-manifest-sha256=0e5c5bf5b8a1445b348a65045f680ac31713dbfa1e690bedb73e0d1e7472a039",
|
| 12 |
+
"--conditioning-config-sha256=89d70134c4fb1ce21c18e46cfb9753f6cdcd460b879d6e60e99b2c7b56ae5114",
|
| 13 |
+
"--prefix-length=64",
|
| 14 |
+
"--evaluation-continuation-length=64",
|
| 15 |
+
"--prompt-count=1024",
|
| 16 |
+
"--diversity-prompt-count=256",
|
| 17 |
+
"--completions-per-diversity-prompt=5",
|
| 18 |
+
"--completion-schedule=c0:p0-1023;c1-4:p0-255",
|
| 19 |
+
"--",
|
| 20 |
+
"mode=sample_eval",
|
| 21 |
+
"seed=42",
|
| 22 |
+
"data=lm1b-wrap",
|
| 23 |
+
"model=small",
|
| 24 |
+
"model.length=128",
|
| 25 |
+
"loader.batch_size=2",
|
| 26 |
+
"loader.eval_batch_size=64",
|
| 27 |
+
"sampling.num_sample_batches=32",
|
| 28 |
+
"sampling.steps=1024",
|
| 29 |
+
"eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/flm_baselines/lm1b/lm1b_Duo.ckpt",
|
| 30 |
+
"trainer.devices=1",
|
| 31 |
+
"hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_1024/upstream_hydra",
|
| 32 |
+
"checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_1024/upstream_hydra",
|
| 33 |
+
"+wandb.offline=true",
|
| 34 |
+
"eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_1024/upstream_samples.json",
|
| 35 |
+
"algo=duo_base",
|
| 36 |
+
"algo.backbone=dit",
|
| 37 |
+
"sampling.predictor=ancestral",
|
| 38 |
+
"sampling.noise_removal=ancestral",
|
| 39 |
+
"eval.disable_ema=False"
|
| 40 |
+
],
|
| 41 |
+
"identity": {
|
| 42 |
+
"adapter_identity": "dlb.adapters.duo:v1|dlb.adapters.duo:DuoAdapter|8878182eb10d7d1e3f18219cb62c2153084842fdb0ab532b1a3c7df0e10da3c6",
|
| 43 |
+
"checkpoint_lock_id": "flm_lm1b_reproductions:ce8ff6a327d9f1470fef9a7ac66813ac94d4cc50268bc2998627498dc6f3a152:lm1b_Duo.ckpt",
|
| 44 |
+
"checkpoint_selection": {
|
| 45 |
+
"path": "lm1b_Duo.ckpt",
|
| 46 |
+
"resource": "flm_lm1b_reproductions",
|
| 47 |
+
"teacher_family": "uniform_duo"
|
| 48 |
+
},
|
| 49 |
+
"checkpoint_sha256": "42694ec4a5d7aa1ca9d1f5e377a99a75dd7239b047236cdc644ee770b157245f",
|
| 50 |
+
"checkpoint_teacher_family": "uniform_duo",
|
| 51 |
+
"command_sha256": "c8b0b99bafae2ec8206538797d1fc8ba6abd3b0e02cedf5b3de498325a1c635f",
|
| 52 |
+
"completion_schedule": "c0:p0-1023;c1-4:p0-255",
|
| 53 |
+
"completions_per_diversity_prompt": 5,
|
| 54 |
+
"conditioning_config_sha256": "89d70134c4fb1ce21c18e46cfb9753f6cdcd460b879d6e60e99b2c7b56ae5114",
|
| 55 |
+
"conditioning_manifest": "/home/ruizzhao/diffusion_baseline/data/manifests/conditional-lm1b-c64.json",
|
| 56 |
+
"conditioning_manifest_sha256": "0e5c5bf5b8a1445b348a65045f680ac31713dbfa1e690bedb73e0d1e7472a039",
|
| 57 |
+
"config_sha256": "6ce963030186c79cab2bc15670737a77ebfad08a0608bc87ee5c54e489ce678f",
|
| 58 |
+
"dataset_id": "lm1b",
|
| 59 |
+
"device": null,
|
| 60 |
+
"diversity_prompt_count": 256,
|
| 61 |
+
"environment": "dlb-duo",
|
| 62 |
+
"evaluation_continuation_length": 64,
|
| 63 |
+
"generation_mode": "conditional_prefix",
|
| 64 |
+
"model_id": "duo",
|
| 65 |
+
"prefix_length": 64,
|
| 66 |
+
"prompt_count": 1024,
|
| 67 |
+
"results_root": "/home/ruizzhao/diffusion_baseline/results/conditional",
|
| 68 |
+
"run_id": "duo-lm1b-steps-1024",
|
| 69 |
+
"sample_count": 2048,
|
| 70 |
+
"seed": 42,
|
| 71 |
+
"source_sha256": "7c9b498f5b717de064d6fad7e2509c866e6cb620",
|
| 72 |
+
"step_count": 1024
|
| 73 |
+
},
|
| 74 |
+
"request": {
|
| 75 |
+
"adapter_identity": "dlb.adapters.duo:v1|dlb.adapters.duo:DuoAdapter|8878182eb10d7d1e3f18219cb62c2153084842fdb0ab532b1a3c7df0e10da3c6",
|
| 76 |
+
"checkpoint_lock_id": "flm_lm1b_reproductions:ce8ff6a327d9f1470fef9a7ac66813ac94d4cc50268bc2998627498dc6f3a152:lm1b_Duo.ckpt",
|
| 77 |
+
"checkpoint_selection": {
|
| 78 |
+
"path": "lm1b_Duo.ckpt",
|
| 79 |
+
"resource": "flm_lm1b_reproductions",
|
| 80 |
+
"teacher_family": "uniform_duo"
|
| 81 |
+
},
|
| 82 |
+
"checkpoint_sha256": "42694ec4a5d7aa1ca9d1f5e377a99a75dd7239b047236cdc644ee770b157245f",
|
| 83 |
+
"checkpoint_teacher_family": "uniform_duo",
|
| 84 |
+
"command": null,
|
| 85 |
+
"completion_schedule": "c0:p0-1023;c1-4:p0-255",
|
| 86 |
+
"completions_per_diversity_prompt": 5,
|
| 87 |
+
"conditioning_config_sha256": "89d70134c4fb1ce21c18e46cfb9753f6cdcd460b879d6e60e99b2c7b56ae5114",
|
| 88 |
+
"conditioning_manifest": "/home/ruizzhao/diffusion_baseline/data/manifests/conditional-lm1b-c64.json",
|
| 89 |
+
"conditioning_manifest_sha256": "0e5c5bf5b8a1445b348a65045f680ac31713dbfa1e690bedb73e0d1e7472a039",
|
| 90 |
+
"config_sha256": "6ce963030186c79cab2bc15670737a77ebfad08a0608bc87ee5c54e489ce678f",
|
| 91 |
+
"dataset_id": "lm1b",
|
| 92 |
+
"device": null,
|
| 93 |
+
"diversity_prompt_count": 256,
|
| 94 |
+
"environment": "dlb-duo",
|
| 95 |
+
"evaluation_continuation_length": 64,
|
| 96 |
+
"generation_mode": "conditional_prefix",
|
| 97 |
+
"model_id": "duo",
|
| 98 |
+
"prefix_length": 64,
|
| 99 |
+
"prompt_count": 1024,
|
| 100 |
+
"results_root": "/home/ruizzhao/diffusion_baseline/results/conditional",
|
| 101 |
+
"run_id": "duo-lm1b-steps-1024",
|
| 102 |
+
"sample_count": 2048,
|
| 103 |
+
"seed": 42,
|
| 104 |
+
"source_sha256": "7c9b498f5b717de064d6fad7e2509c866e6cb620",
|
| 105 |
+
"step_count": 1024
|
| 106 |
+
},
|
| 107 |
+
"started_at": "2026-08-25T11:37:10.819468Z"
|
| 108 |
+
}
|
results/conditional/samples/lm1b/duo/steps_16/conversion_metadata.json
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"checkpoint_lock_id": "flm_lm1b_reproductions:ce8ff6a327d9f1470fef9a7ac66813ac94d4cc50268bc2998627498dc6f3a152:lm1b_Duo.ckpt",
|
| 3 |
+
"checkpoint_selection": {
|
| 4 |
+
"path": "lm1b_Duo.ckpt",
|
| 5 |
+
"resource": "flm_lm1b_reproductions",
|
| 6 |
+
"teacher_family": "uniform_duo"
|
| 7 |
+
},
|
| 8 |
+
"checkpoint_sha256": "42694ec4a5d7aa1ca9d1f5e377a99a75dd7239b047236cdc644ee770b157245f",
|
| 9 |
+
"conditioning_implementation": "zero_shot_runtime_projection",
|
| 10 |
+
"exclude_from_latency": true,
|
| 11 |
+
"format": "dlb-conditional-token-capture-v1",
|
| 12 |
+
"generated_samples": 2048,
|
| 13 |
+
"generation_seconds_sentinel": 0.0,
|
| 14 |
+
"generation_seconds_source": "unavailable_excluded_sentinel",
|
| 15 |
+
"paper_conditional_training_reproduced": false,
|
| 16 |
+
"requested_samples": 2048,
|
| 17 |
+
"teacher_family": "uniform_duo",
|
| 18 |
+
"token_ids_source": "upstream_conditional_capture",
|
| 19 |
+
"tokenizer": "bert-base-uncased",
|
| 20 |
+
"tokenizer_revision": "86b5e0934494bd15c9632b12f734a8a67f723594",
|
| 21 |
+
"trim_policy": "none_exact_conditional_schedule",
|
| 22 |
+
"trimmed_samples": 0,
|
| 23 |
+
"upstream_output": "/home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_16/upstream_token_ids.json"
|
| 24 |
+
}
|
results/conditional/samples/lm1b/duo/steps_16/samples.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
results/conditional/samples/lm1b/duo/steps_16/stderr.log
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 0 |
0%| | 0/32 [00:00<?, ?it/s]
|
| 1 |
3%|▎ | 1/32 [00:03<01:34, 3.06s/it]
|
| 2 |
6%|▋ | 2/32 [00:05<01:25, 2.87s/it]
|
| 3 |
9%|▉ | 3/32 [00:08<01:22, 2.84s/it]
|
| 4 |
12%|█▎ | 4/32 [00:11<01:18, 2.81s/it]
|
| 5 |
16%|█▌ | 5/32 [00:14<01:18, 2.89s/it]
|
| 6 |
19%|█▉ | 6/32 [00:17<01:16, 2.94s/it]
|
| 7 |
22%|██▏ | 7/32 [00:20<01:11, 2.85s/it]
|
| 8 |
25%|██▌ | 8/32 [00:22<01:08, 2.85s/it]
|
| 9 |
28%|██▊ | 9/32 [00:26<01:08, 3.00s/it]
|
| 10 |
31%|███▏ | 10/32 [00:29<01:04, 2.94s/it]
|
| 11 |
34%|███▍ | 11/32 [00:31<01:00, 2.89s/it]
|
| 12 |
38%|███▊ | 12/32 [00:34<00:58, 2.91s/it]
|
| 13 |
41%|████ | 13/32 [00:37<00:54, 2.87s/it]
|
| 14 |
44%|████▍ | 14/32 [00:40<00:51, 2.89s/it]
|
| 15 |
47%|████▋ | 15/32 [00:43<00:48, 2.83s/it]
|
| 16 |
50%|█████ | 16/32 [00:46<00:46, 2.89s/it]
|
| 17 |
53%|█████▎ | 17/32 [00:49<00:44, 2.96s/it]
|
| 18 |
56%|█████▋ | 18/32 [00:52<00:42, 3.07s/it]
|
| 19 |
59%|█████▉ | 19/32 [00:55<00:40, 3.08s/it]
|
| 20 |
62%|██████▎ | 20/32 [00:58<00:35, 2.95s/it]
|
| 21 |
66%|██████▌ | 21/32 [01:01<00:31, 2.87s/it]
|
| 22 |
69%|██████▉ | 22/32 [01:03<00:28, 2.85s/it]
|
| 23 |
72%|███████▏ | 23/32 [01:06<00:25, 2.80s/it]
|
| 24 |
75%|███████▌ | 24/32 [01:09<00:22, 2.82s/it]
|
| 25 |
78%|███████▊ | 25/32 [01:12<00:19, 2.76s/it]
|
| 26 |
81%|████████▏ | 26/32 [01:15<00:17, 2.96s/it]
|
| 27 |
84%|████████▍ | 27/32 [01:18<00:14, 2.86s/it]
|
| 28 |
88%|████████▊ | 28/32 [01:20<00:11, 2.85s/it]
|
| 29 |
91%|█████████ | 29/32 [01:23<00:08, 2.86s/it]
|
| 30 |
94%|█████████▍| 30/32 [01:27<00:06, 3.03s/it]
|
| 31 |
97%|█████████▋| 31/32 [01:29<00:02, 2.90s/it]
|
|
|
|
| 1 |
+
Seed set to 42
|
| 2 |
+
/opt/anaconda3/envs/dlb-duo/lib/python3.12/site-packages/huggingface_hub/file_download.py:949: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.
|
| 3 |
+
warnings.warn(
|
| 4 |
+
/opt/anaconda3/envs/dlb-duo/lib/python3.12/site-packages/lightning/fabric/utilities/cloud_io.py:57: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
|
| 5 |
+
|
| 6 |
0%| | 0/32 [00:00<?, ?it/s]
|
| 7 |
3%|▎ | 1/32 [00:03<01:34, 3.06s/it]
|
| 8 |
6%|▋ | 2/32 [00:05<01:25, 2.87s/it]
|
| 9 |
9%|▉ | 3/32 [00:08<01:22, 2.84s/it]
|
| 10 |
12%|█▎ | 4/32 [00:11<01:18, 2.81s/it]
|
| 11 |
16%|█▌ | 5/32 [00:14<01:18, 2.89s/it]
|
| 12 |
19%|█▉ | 6/32 [00:17<01:16, 2.94s/it]
|
| 13 |
22%|██▏ | 7/32 [00:20<01:11, 2.85s/it]
|
| 14 |
25%|██▌ | 8/32 [00:22<01:08, 2.85s/it]
|
| 15 |
28%|██▊ | 9/32 [00:26<01:08, 3.00s/it]
|
| 16 |
31%|███▏ | 10/32 [00:29<01:04, 2.94s/it]
|
| 17 |
34%|███▍ | 11/32 [00:31<01:00, 2.89s/it]
|
| 18 |
38%|███▊ | 12/32 [00:34<00:58, 2.91s/it]
|
| 19 |
41%|████ | 13/32 [00:37<00:54, 2.87s/it]
|
| 20 |
44%|████▍ | 14/32 [00:40<00:51, 2.89s/it]
|
| 21 |
47%|████▋ | 15/32 [00:43<00:48, 2.83s/it]
|
| 22 |
50%|█████ | 16/32 [00:46<00:46, 2.89s/it]
|
| 23 |
53%|█████▎ | 17/32 [00:49<00:44, 2.96s/it]
|
| 24 |
56%|█████▋ | 18/32 [00:52<00:42, 3.07s/it]
|
| 25 |
59%|█████▉ | 19/32 [00:55<00:40, 3.08s/it]
|
| 26 |
62%|██████▎ | 20/32 [00:58<00:35, 2.95s/it]
|
| 27 |
66%|██████▌ | 21/32 [01:01<00:31, 2.87s/it]
|
| 28 |
69%|██████▉ | 22/32 [01:03<00:28, 2.85s/it]
|
| 29 |
72%|███████▏ | 23/32 [01:06<00:25, 2.80s/it]
|
| 30 |
75%|███████▌ | 24/32 [01:09<00:22, 2.82s/it]
|
| 31 |
78%|███████▊ | 25/32 [01:12<00:19, 2.76s/it]
|
| 32 |
81%|████████▏ | 26/32 [01:15<00:17, 2.96s/it]
|
| 33 |
84%|████████▍ | 27/32 [01:18<00:14, 2.86s/it]
|
| 34 |
88%|████████▊ | 28/32 [01:20<00:11, 2.85s/it]
|
| 35 |
91%|█████████ | 29/32 [01:23<00:08, 2.86s/it]
|
| 36 |
94%|█████████▍| 30/32 [01:27<00:06, 3.03s/it]
|
| 37 |
97%|█████████▋| 31/32 [01:29<00:02, 2.90s/it]
|
results/conditional/samples/lm1b/duo/steps_16/upstream_hydra/config_tree.txt
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
CONFIG
|
| 2 |
+
├── mode
|
| 3 |
+
│ └── sample_eval
|
| 4 |
+
├── seed
|
| 5 |
+
│ └── 42
|
| 6 |
+
├── loader
|
| 7 |
+
│ └── global_batch_size: 512
|
| 8 |
+
│ eval_global_batch_size: 512
|
| 9 |
+
│ batch_size: 2
|
| 10 |
+
│ eval_batch_size: 1
|
| 11 |
+
│ num_workers: 512
|
| 12 |
+
│ pin_memory: true
|
| 13 |
+
│
|
| 14 |
+
├── sampling
|
| 15 |
+
│ └── predictor: ancestral
|
| 16 |
+
│ steps: 16
|
| 17 |
+
│ noise_removal: ancestral
|
| 18 |
+
│ use_float64: true
|
| 19 |
+
│ p_nucleus: 1.0
|
| 20 |
+
│ num_sample_batches: 1
|
| 21 |
+
│ num_sample_log: 2
|
| 22 |
+
│ semi_ar: false
|
| 23 |
+
│ stride_length: 1
|
| 24 |
+
│ num_strides: 1
|
| 25 |
+
│ guid_weight: null
|
| 26 |
+
│ psi:
|
| 27 |
+
│ time_profile: linear
|
| 28 |
+
│ high_mode: pure-posterior
|
| 29 |
+
│ middle_mode: pure-posterior
|
| 30 |
+
│ low_mode: pure-posterior
|
| 31 |
+
│ high_frac: 0.2
|
| 32 |
+
│ middle_frac: 0.6
|
| 33 |
+
│
|
| 34 |
+
├── training
|
| 35 |
+
│ └── ema: 0.9999
|
| 36 |
+
│ antithetic_sampling: true
|
| 37 |
+
│ importance_sampling: false
|
| 38 |
+
│ sampling_eps: 0.001
|
| 39 |
+
│ change_of_variables: false
|
| 40 |
+
│ loss_precision: bf16
|
| 41 |
+
│ finetune_path: ''
|
| 42 |
+
│ class_dropout_p: 0.1
|
| 43 |
+
│
|
| 44 |
+
├── eval
|
| 45 |
+
│ └── checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference
|
| 46 |
+
│ disable_ema: false
|
| 47 |
+
│ compute_generative_perplexity: false
|
| 48 |
+
│ perplexity_batch_size: 8
|
| 49 |
+
│ compute_perplexity_on_sanity: false
|
| 50 |
+
│ gen_ppl_eval_model_name_or_path: gpt2-large
|
| 51 |
+
│ generate_samples: true
|
| 52 |
+
│ generated_samples_path: /home/ruizzhao/diffusion_baseline/results/condit
|
| 53 |
+
│
|
| 54 |
+
├── optim
|
| 55 |
+
│ └── weight_decay: 0
|
| 56 |
+
│ lr: 0.0003
|
| 57 |
+
│ beta1: 0.9
|
| 58 |
+
│ beta2: 0.999
|
| 59 |
+
│ eps: 1.0e-08
|
| 60 |
+
│
|
| 61 |
+
├── trainer
|
| 62 |
+
│ └── _target_: lightning.Trainer
|
| 63 |
+
│ accelerator: cuda
|
| 64 |
+
│ num_nodes: 1
|
| 65 |
+
│ devices: 1
|
| 66 |
+
│ accumulate_grad_batches: 256
|
| 67 |
+
│ gradient_clip_val: 1.0
|
| 68 |
+
│ precision: bf16
|
| 69 |
+
│ num_sanity_val_steps: 2
|
| 70 |
+
│ max_steps: 1000000
|
| 71 |
+
│ log_every_n_steps: 100
|
| 72 |
+
│ limit_train_batches: 1.0
|
| 73 |
+
│ limit_val_batches: 1.0
|
| 74 |
+
│ val_check_interval: 5000
|
| 75 |
+
│ check_val_every_n_epoch: null
|
| 76 |
+
│
|
| 77 |
+
├── wandb
|
| 78 |
+
│ └── project: duo
|
| 79 |
+
│ group: null
|
| 80 |
+
│ job_type: null
|
| 81 |
+
│ name: null
|
| 82 |
+
│ id: None_42
|
| 83 |
+
│ tags:
|
| 84 |
+
│ - log-linear
|
| 85 |
+
│ - lm1b
|
| 86 |
+
│ - lm1b
|
| 87 |
+
│ - duo_base
|
| 88 |
+
│ offline: true
|
| 89 |
+
│
|
| 90 |
+
├── checkpointing
|
| 91 |
+
│ └── save_dir: /home/ruizzhao/diffusion_baseline/results/conditional/samples/
|
| 92 |
+
│ resume_from_ckpt: true
|
| 93 |
+
│ resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results/conditional/
|
| 94 |
+
│
|
| 95 |
+
├── callbacks
|
| 96 |
+
│ └── checkpoint_every_n_steps:
|
| 97 |
+
│ _target_: lightning.pytorch.callbacks.ModelCheckpoint
|
| 98 |
+
│ save_top_k: -1
|
| 99 |
+
│ save_last: true
|
| 100 |
+
│ dirpath: /home/ruizzhao/diffusion_baseline/results/conditional/samples
|
| 101 |
+
│ verbose: true
|
| 102 |
+
│ auto_insert_metric_name: false
|
| 103 |
+
│ every_n_train_steps: 500
|
| 104 |
+
│ checkpoint_monitor:
|
| 105 |
+
│ _target_: lightning.pytorch.callbacks.ModelCheckpoint
|
| 106 |
+
│ monitor: val/nll
|
| 107 |
+
│ mode: min
|
| 108 |
+
│ save_top_k: 1
|
| 109 |
+
│ save_last: false
|
| 110 |
+
│ dirpath: /home/ruizzhao/diffusion_baseline/results/conditional/samples
|
| 111 |
+
│ filename: best
|
| 112 |
+
│ auto_insert_metric_name: false
|
| 113 |
+
│ verbose: true
|
| 114 |
+
│ learning_rate_monitor:
|
| 115 |
+
│ _target_: lightning.pytorch.callbacks.LearningRateMonitor
|
| 116 |
+
│ logging_interval: step
|
| 117 |
+
│
|
| 118 |
+
├── data
|
| 119 |
+
│ └── train: lm1b
|
| 120 |
+
│ valid: lm1b
|
| 121 |
+
│ modality: text
|
| 122 |
+
│ tokenizer_name_or_path: bert-base-uncased
|
| 123 |
+
│ cache_dir: /share/kuleshov/ssahoo/textdiffusion/data
|
| 124 |
+
│ wrap: true
|
| 125 |
+
│ streaming: false
|
| 126 |
+
│ insert_train_eos: true
|
| 127 |
+
│ insert_valid_eos: true
|
| 128 |
+
│
|
| 129 |
+
├── model
|
| 130 |
+
│ └── name: small
|
| 131 |
+
│ type: ddit
|
| 132 |
+
│ hidden_size: 768
|
| 133 |
+
│ cond_dim: 128
|
| 134 |
+
│ length: 128
|
| 135 |
+
│ n_blocks: 12
|
| 136 |
+
│ n_heads: 12
|
| 137 |
+
│ scale_by_sigma: true
|
| 138 |
+
│ dropout: 0.1
|
| 139 |
+
│ tie_word_embeddings: false
|
| 140 |
+
│ vocab_lookup: true
|
| 141 |
+
│
|
| 142 |
+
├── strategy
|
| 143 |
+
│ └── _target_: lightning.pytorch.strategies.DDPStrategy
|
| 144 |
+
│ find_unused_parameters: false
|
| 145 |
+
│
|
| 146 |
+
├── noise
|
| 147 |
+
│ └── type: log-linear
|
| 148 |
+
│ eps: 0.001
|
| 149 |
+
│
|
| 150 |
+
├── lr_scheduler
|
| 151 |
+
│ └── _target_: transformers.get_constant_schedule_with_warmup
|
| 152 |
+
│ num_warmup_steps: 2500
|
| 153 |
+
│
|
| 154 |
+
├── prior
|
| 155 |
+
│ └── type: none
|
| 156 |
+
│ latent_width: 0
|
| 157 |
+
│ latent_height: 0
|
| 158 |
+
│
|
| 159 |
+
└── algo
|
| 160 |
+
└── name: duo_base
|
| 161 |
+
backbone: dit
|
| 162 |
+
parameterization: mean
|
| 163 |
+
time_conditioning: true
|
| 164 |
+
T: 0
|
| 165 |
+
subs_masking: false
|
| 166 |
+
causal_attention: false
|
| 167 |
+
ignore_bos: false
|
| 168 |
+
loss_type: elbo
|
| 169 |
+
|
results/conditional/samples/lm1b/duo/steps_2/upstream_hydra/dlb_pinned_upstream_main.log
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[2026-08-25 19:28:37,027][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
|
| 2 |
+
[2026-08-25 19:30:00,496][dlb_pinned_upstream_main][INFO] - Generative perplexity: 168.81248474121094
|
| 3 |
+
[2026-08-25 19:30:00,496][dlb_pinned_upstream_main][INFO] - Sample entropy: 4.308840274810791
|
| 4 |
+
[2026-08-25 19:30:00,499][dlb_pinned_upstream_main][INFO] - Samples saved at: /home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_2/upstream_samples.json
|
| 5 |
+
[2026-08-26 01:18:35,569][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
|
| 6 |
+
[2026-08-26 01:18:45,231][dlb_pinned_upstream_main][INFO] - Generative perplexity: 372.2198181152344
|
| 7 |
+
[2026-08-26 01:18:45,231][dlb_pinned_upstream_main][INFO] - Sample entropy: 4.198279857635498
|
| 8 |
+
[2026-08-26 01:18:45,232][dlb_pinned_upstream_main][INFO] - Samples saved at: /home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_2/upstream_samples.json
|
results/conditional/samples/lm1b/duo/steps_2/upstream_samples.json
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"generative_ppl": 372.2198181152344,
|
| 3 |
+
"entropy": 4.198279857635498,
|
| 4 |
+
"generated_seqs": [
|
| 5 |
+
"[CLS] central telegraph just submitted ( dip the car muslim weak to a in the the sprawling circuit. [CLS] he wishes using obama, not mccain and again. the example did 7 percent who voted in? [CLS] memphis used to the care of those becoming a'out \" i did say then but of course they suggested that consistent not that not is the va sar thousandsrs, not just arrived for the reason the late drink decided -ctive one was written by the al and or - - i think d south upcoming. [CLS] [CLS] a atrocities are in the us but witnessed. [CLS] the worstable trust is available for this bad estate ira - - - [CLS]"
|
| 6 |
+
]
|
| 7 |
+
}
|
results/conditional/samples/lm1b/duo/steps_32/request.json
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"command": [
|
| 3 |
+
"/opt/anaconda3/envs/dlb-duo/bin/python",
|
| 4 |
+
"-u",
|
| 5 |
+
"-m",
|
| 6 |
+
"dlb.adapters.capture",
|
| 7 |
+
"--upstream-entrypoint=/home/ruizzhao/diffusion_baseline/upstreams/duo/main.py",
|
| 8 |
+
"--capture-path=/home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_32/upstream_token_ids.json",
|
| 9 |
+
"--generation-mode=conditional_prefix",
|
| 10 |
+
"--conditioning-manifest=/home/ruizzhao/diffusion_baseline/data/manifests/conditional-lm1b-c64.json",
|
| 11 |
+
"--conditioning-manifest-sha256=0e5c5bf5b8a1445b348a65045f680ac31713dbfa1e690bedb73e0d1e7472a039",
|
| 12 |
+
"--conditioning-config-sha256=89d70134c4fb1ce21c18e46cfb9753f6cdcd460b879d6e60e99b2c7b56ae5114",
|
| 13 |
+
"--prefix-length=64",
|
| 14 |
+
"--evaluation-continuation-length=64",
|
| 15 |
+
"--prompt-count=1024",
|
| 16 |
+
"--diversity-prompt-count=256",
|
| 17 |
+
"--completions-per-diversity-prompt=5",
|
| 18 |
+
"--completion-schedule=c0:p0-1023;c1-4:p0-255",
|
| 19 |
+
"--",
|
| 20 |
+
"mode=sample_eval",
|
| 21 |
+
"seed=42",
|
| 22 |
+
"data=lm1b-wrap",
|
| 23 |
+
"model=small",
|
| 24 |
+
"model.length=128",
|
| 25 |
+
"loader.batch_size=2",
|
| 26 |
+
"loader.eval_batch_size=64",
|
| 27 |
+
"sampling.num_sample_batches=32",
|
| 28 |
+
"sampling.steps=32",
|
| 29 |
+
"eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/flm_baselines/lm1b/lm1b_Duo.ckpt",
|
| 30 |
+
"trainer.devices=1",
|
| 31 |
+
"hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_32/upstream_hydra",
|
| 32 |
+
"checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_32/upstream_hydra",
|
| 33 |
+
"+wandb.offline=true",
|
| 34 |
+
"eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_32/upstream_samples.json",
|
| 35 |
+
"algo=duo_base",
|
| 36 |
+
"algo.backbone=dit",
|
| 37 |
+
"sampling.predictor=ancestral",
|
| 38 |
+
"sampling.noise_removal=ancestral",
|
| 39 |
+
"eval.disable_ema=False"
|
| 40 |
+
],
|
| 41 |
+
"identity": {
|
| 42 |
+
"adapter_identity": "dlb.adapters.duo:v1|dlb.adapters.duo:DuoAdapter|8878182eb10d7d1e3f18219cb62c2153084842fdb0ab532b1a3c7df0e10da3c6",
|
| 43 |
+
"checkpoint_lock_id": "flm_lm1b_reproductions:ce8ff6a327d9f1470fef9a7ac66813ac94d4cc50268bc2998627498dc6f3a152:lm1b_Duo.ckpt",
|
| 44 |
+
"checkpoint_selection": {
|
| 45 |
+
"path": "lm1b_Duo.ckpt",
|
| 46 |
+
"resource": "flm_lm1b_reproductions",
|
| 47 |
+
"teacher_family": "uniform_duo"
|
| 48 |
+
},
|
| 49 |
+
"checkpoint_sha256": "42694ec4a5d7aa1ca9d1f5e377a99a75dd7239b047236cdc644ee770b157245f",
|
| 50 |
+
"checkpoint_teacher_family": "uniform_duo",
|
| 51 |
+
"command_sha256": "f41778b2eeb39203a92454e0ce1da052aa2a6991ce4e46b2b8f97b87e8cb04f9",
|
| 52 |
+
"completion_schedule": "c0:p0-1023;c1-4:p0-255",
|
| 53 |
+
"completions_per_diversity_prompt": 5,
|
| 54 |
+
"conditioning_config_sha256": "89d70134c4fb1ce21c18e46cfb9753f6cdcd460b879d6e60e99b2c7b56ae5114",
|
| 55 |
+
"conditioning_manifest": "/home/ruizzhao/diffusion_baseline/data/manifests/conditional-lm1b-c64.json",
|
| 56 |
+
"conditioning_manifest_sha256": "0e5c5bf5b8a1445b348a65045f680ac31713dbfa1e690bedb73e0d1e7472a039",
|
| 57 |
+
"config_sha256": "6ce963030186c79cab2bc15670737a77ebfad08a0608bc87ee5c54e489ce678f",
|
| 58 |
+
"dataset_id": "lm1b",
|
| 59 |
+
"device": null,
|
| 60 |
+
"diversity_prompt_count": 256,
|
| 61 |
+
"environment": "dlb-duo",
|
| 62 |
+
"evaluation_continuation_length": 64,
|
| 63 |
+
"generation_mode": "conditional_prefix",
|
| 64 |
+
"model_id": "duo",
|
| 65 |
+
"prefix_length": 64,
|
| 66 |
+
"prompt_count": 1024,
|
| 67 |
+
"results_root": "/home/ruizzhao/diffusion_baseline/results/conditional",
|
| 68 |
+
"run_id": "duo-lm1b-steps-32",
|
| 69 |
+
"sample_count": 2048,
|
| 70 |
+
"seed": 42,
|
| 71 |
+
"source_sha256": "7c9b498f5b717de064d6fad7e2509c866e6cb620",
|
| 72 |
+
"step_count": 32
|
| 73 |
+
},
|
| 74 |
+
"request": {
|
| 75 |
+
"adapter_identity": "dlb.adapters.duo:v1|dlb.adapters.duo:DuoAdapter|8878182eb10d7d1e3f18219cb62c2153084842fdb0ab532b1a3c7df0e10da3c6",
|
| 76 |
+
"checkpoint_lock_id": "flm_lm1b_reproductions:ce8ff6a327d9f1470fef9a7ac66813ac94d4cc50268bc2998627498dc6f3a152:lm1b_Duo.ckpt",
|
| 77 |
+
"checkpoint_selection": {
|
| 78 |
+
"path": "lm1b_Duo.ckpt",
|
| 79 |
+
"resource": "flm_lm1b_reproductions",
|
| 80 |
+
"teacher_family": "uniform_duo"
|
| 81 |
+
},
|
| 82 |
+
"checkpoint_sha256": "42694ec4a5d7aa1ca9d1f5e377a99a75dd7239b047236cdc644ee770b157245f",
|
| 83 |
+
"checkpoint_teacher_family": "uniform_duo",
|
| 84 |
+
"command": null,
|
| 85 |
+
"completion_schedule": "c0:p0-1023;c1-4:p0-255",
|
| 86 |
+
"completions_per_diversity_prompt": 5,
|
| 87 |
+
"conditioning_config_sha256": "89d70134c4fb1ce21c18e46cfb9753f6cdcd460b879d6e60e99b2c7b56ae5114",
|
| 88 |
+
"conditioning_manifest": "/home/ruizzhao/diffusion_baseline/data/manifests/conditional-lm1b-c64.json",
|
| 89 |
+
"conditioning_manifest_sha256": "0e5c5bf5b8a1445b348a65045f680ac31713dbfa1e690bedb73e0d1e7472a039",
|
| 90 |
+
"config_sha256": "6ce963030186c79cab2bc15670737a77ebfad08a0608bc87ee5c54e489ce678f",
|
| 91 |
+
"dataset_id": "lm1b",
|
| 92 |
+
"device": null,
|
| 93 |
+
"diversity_prompt_count": 256,
|
| 94 |
+
"environment": "dlb-duo",
|
| 95 |
+
"evaluation_continuation_length": 64,
|
| 96 |
+
"generation_mode": "conditional_prefix",
|
| 97 |
+
"model_id": "duo",
|
| 98 |
+
"prefix_length": 64,
|
| 99 |
+
"prompt_count": 1024,
|
| 100 |
+
"results_root": "/home/ruizzhao/diffusion_baseline/results/conditional",
|
| 101 |
+
"run_id": "duo-lm1b-steps-32",
|
| 102 |
+
"sample_count": 2048,
|
| 103 |
+
"seed": 42,
|
| 104 |
+
"source_sha256": "7c9b498f5b717de064d6fad7e2509c866e6cb620",
|
| 105 |
+
"step_count": 32
|
| 106 |
+
},
|
| 107 |
+
"started_at": "2026-08-25T11:35:03.023171Z"
|
| 108 |
+
}
|
results/conditional/samples/lm1b/duo/steps_32/samples.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
results/conditional/samples/lm1b/duo/steps_32/upstream_hydra/dlb_pinned_upstream_main.log
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[2026-08-25 19:35:07,648][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
|
| 2 |
+
[2026-08-25 19:37:06,579][dlb_pinned_upstream_main][INFO] - Generative perplexity: 80.59013366699219
|
| 3 |
+
[2026-08-25 19:37:06,580][dlb_pinned_upstream_main][INFO] - Sample entropy: 4.331864833831787
|
| 4 |
+
[2026-08-25 19:37:06,587][dlb_pinned_upstream_main][INFO] - Samples saved at: /home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_32/upstream_samples.json
|
| 5 |
+
[2026-08-26 01:19:54,416][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
|
| 6 |
+
[2026-08-26 01:20:11,545][dlb_pinned_upstream_main][INFO] - Generative perplexity: 60.40050506591797
|
| 7 |
+
[2026-08-26 01:20:11,546][dlb_pinned_upstream_main][INFO] - Sample entropy: 4.089200019836426
|
| 8 |
+
[2026-08-26 01:20:11,547][dlb_pinned_upstream_main][INFO] - Samples saved at: /home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_32/upstream_samples.json
|
results/conditional/samples/lm1b/duo/steps_32/upstream_samples.json
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"generative_ppl": 60.40050506591797,
|
| 3 |
+
"entropy": 4.089200019836426,
|
| 4 |
+
"generated_seqs": [
|
| 5 |
+
"[CLS] west as an executive background / analysis. [CLS] the bae case is widely expected to leave the u. s. supreme court in coming weeks. [CLS] the suspect was originally from wisconsin but was on a flight headed to arvaux. [CLS] what works very well is the big big corporations and big, middle americans - like their unions and the shareholders'investments - for the healthcare companies and technology companies. [CLS] he said it was a transportation program, and an environment was needed, but so was the cost of the project. [CLS] catering for the guests'meals included link managers for the smoothest restaurants around, mr. gerard said. [CLS] [CLS]"
|
| 6 |
+
]
|
| 7 |
+
}
|
results/conditional/samples/lm1b/duo/steps_4/upstream_hydra/dlb_pinned_upstream_main.log
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[2026-08-25 19:30:08,744][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
|
| 2 |
+
[2026-08-25 19:31:31,861][dlb_pinned_upstream_main][INFO] - Generative perplexity: 112.44278717041016
|
| 3 |
+
[2026-08-25 19:31:31,862][dlb_pinned_upstream_main][INFO] - Sample entropy: 4.307894229888916
|
| 4 |
+
[2026-08-25 19:31:31,868][dlb_pinned_upstream_main][INFO] - Samples saved at: /home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_4/upstream_samples.json
|
| 5 |
+
[2026-08-26 01:18:53,356][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
|
| 6 |
+
[2026-08-26 01:19:03,389][dlb_pinned_upstream_main][INFO] - Generative perplexity: 340.0912780761719
|
| 7 |
+
[2026-08-26 01:19:03,390][dlb_pinned_upstream_main][INFO] - Sample entropy: 4.253403663635254
|
| 8 |
+
[2026-08-26 01:19:03,391][dlb_pinned_upstream_main][INFO] - Samples saved at: /home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_4/upstream_samples.json
|
results/conditional/samples/lm1b/duo/steps_8/run_metadata.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"checkpoint_lock_id":"flm_lm1b_reproductions:ce8ff6a327d9f1470fef9a7ac66813ac94d4cc50268bc2998627498dc6f3a152:lm1b_Duo.ckpt","checkpoint_selection":{"path":"lm1b_Duo.ckpt","resource":"flm_lm1b_reproductions","teacher_family":"uniform_duo"},"checkpoint_sha256":"42694ec4a5d7aa1ca9d1f5e377a99a75dd7239b047236cdc644ee770b157245f","checkpoint_teacher_family":"uniform_duo","command":["/opt/anaconda3/envs/dlb-duo/bin/python","-u","-m","dlb.adapters.capture","--upstream-entrypoint=/home/ruizzhao/diffusion_baseline/upstreams/duo/main.py","--capture-path=/home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_8/upstream_token_ids.json","--generation-mode=conditional_prefix","--conditioning-manifest=/home/ruizzhao/diffusion_baseline/data/manifests/conditional-lm1b-c64.json","--conditioning-manifest-sha256=0e5c5bf5b8a1445b348a65045f680ac31713dbfa1e690bedb73e0d1e7472a039","--conditioning-config-sha256=89d70134c4fb1ce21c18e46cfb9753f6cdcd460b879d6e60e99b2c7b56ae5114","--prefix-length=64","--evaluation-continuation-length=64","--prompt-count=1024","--diversity-prompt-count=256","--completions-per-diversity-prompt=5","--completion-schedule=c0:p0-1023;c1-4:p0-255","--","mode=sample_eval","seed=42","data=lm1b-wrap","model=small","model.length=128","loader.batch_size=2","loader.eval_batch_size=64","sampling.num_sample_batches=32","sampling.steps=8","eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/flm_baselines/lm1b/lm1b_Duo.ckpt","trainer.devices=1","hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_8/upstream_hydra","checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_8/upstream_hydra","+wandb.offline=true","eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_8/upstream_samples.json","algo=duo_base","algo.backbone=dit","sampling.predictor=ancestral","sampling.noise_removal=ancestral","eval.disable_ema=False"],"command_sha256":"b8fa30b138d8fb508ed791becaffbc41afe91e06f72e849700b8e03beb6d694f","config_sha256":"6ce963030186c79cab2bc15670737a77ebfad08a0608bc87ee5c54e489ce678f","environment":{"cuda_version":null,"cuda_visible_devices":"4","hostname":"DS210029","platform":"Linux-5.15.0-186-generic-x86_64-with-glibc2.35","python":"3.12.13","torch_version":"2.5.1"},"finished_at":"2026-08-25T11:33:12.467828Z","identity":{"adapter_identity":"dlb.adapters.duo:v1|dlb.adapters.duo:DuoAdapter|8878182eb10d7d1e3f18219cb62c2153084842fdb0ab532b1a3c7df0e10da3c6","checkpoint_lock_id":"flm_lm1b_reproductions:ce8ff6a327d9f1470fef9a7ac66813ac94d4cc50268bc2998627498dc6f3a152:lm1b_Duo.ckpt","checkpoint_selection":{"path":"lm1b_Duo.ckpt","resource":"flm_lm1b_reproductions","teacher_family":"uniform_duo"},"checkpoint_sha256":"42694ec4a5d7aa1ca9d1f5e377a99a75dd7239b047236cdc644ee770b157245f","checkpoint_teacher_family":"uniform_duo","command_sha256":"b8fa30b138d8fb508ed791becaffbc41afe91e06f72e849700b8e03beb6d694f","completion_schedule":"c0:p0-1023;c1-4:p0-255","completions_per_diversity_prompt":5,"conditioning_config_sha256":"89d70134c4fb1ce21c18e46cfb9753f6cdcd460b879d6e60e99b2c7b56ae5114","conditioning_manifest":"/home/ruizzhao/diffusion_baseline/data/manifests/conditional-lm1b-c64.json","conditioning_manifest_sha256":"0e5c5bf5b8a1445b348a65045f680ac31713dbfa1e690bedb73e0d1e7472a039","config_sha256":"6ce963030186c79cab2bc15670737a77ebfad08a0608bc87ee5c54e489ce678f","dataset_id":"lm1b","device":null,"diversity_prompt_count":256,"environment":"dlb-duo","evaluation_continuation_length":64,"generation_mode":"conditional_prefix","model_id":"duo","prefix_length":64,"prompt_count":1024,"results_root":"/home/ruizzhao/diffusion_baseline/results/conditional","run_id":"duo-lm1b-steps-8","sample_count":2048,"seed":42,"source_sha256":"7c9b498f5b717de064d6fad7e2509c866e6cb620","step_count":8},"request":{"sample_count":2048},"sample_count":2048,"source_sha256":"7c9b498f5b717de064d6fad7e2509c866e6cb620","started_at":"2026-08-25T11:31:36.170053Z","status":"succeeded","stderr_log":"/home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_8/stderr.log","stdout_log":"/home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_8/stdout.log"}
|
results/conditional/samples/lm1b/duo/steps_8/stdout.log
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
CONFIG
|
| 2 |
+
├── mode
|
| 3 |
+
│ └── sample_eval
|
| 4 |
+
├── seed
|
| 5 |
+
│ └── 42
|
| 6 |
+
├── loader
|
| 7 |
+
│ └── global_batch_size: 512
|
| 8 |
+
│ eval_global_batch_size: 512
|
| 9 |
+
│ batch_size: 2
|
| 10 |
+
│ eval_batch_size: 64
|
| 11 |
+
│ num_workers: 512
|
| 12 |
+
│ pin_memory: true
|
| 13 |
+
│
|
| 14 |
+
├── sampling
|
| 15 |
+
│ └── predictor: ancestral
|
| 16 |
+
│ steps: 8
|
| 17 |
+
│ noise_removal: ancestral
|
| 18 |
+
│ use_float64: true
|
| 19 |
+
│ p_nucleus: 1.0
|
| 20 |
+
│ num_sample_batches: 32
|
| 21 |
+
│ num_sample_log: 2
|
| 22 |
+
│ semi_ar: false
|
| 23 |
+
│ stride_length: 1
|
| 24 |
+
│ num_strides: 1
|
| 25 |
+
│ guid_weight: null
|
| 26 |
+
│ psi:
|
| 27 |
+
│ time_profile: linear
|
| 28 |
+
│ high_mode: pure-posterior
|
| 29 |
+
│ middle_mode: pure-posterior
|
| 30 |
+
│ low_mode: pure-posterior
|
| 31 |
+
│ high_frac: 0.2
|
| 32 |
+
│ middle_frac: 0.6
|
| 33 |
+
│
|
| 34 |
+
├── training
|
| 35 |
+
│ └── ema: 0.9999
|
| 36 |
+
│ antithetic_sampling: true
|
| 37 |
+
│ importance_sampling: false
|
| 38 |
+
│ sampling_eps: 0.001
|
| 39 |
+
│ change_of_variables: false
|
| 40 |
+
│ loss_precision: bf16
|
| 41 |
+
│ finetune_path: ''
|
| 42 |
+
│ class_dropout_p: 0.1
|
| 43 |
+
│
|
| 44 |
+
├── eval
|
| 45 |
+
│ └── checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference
|
| 46 |
+
│ disable_ema: false
|
| 47 |
+
│ compute_generative_perplexity: false
|
| 48 |
+
│ perplexity_batch_size: 8
|
| 49 |
+
│ compute_perplexity_on_sanity: false
|
| 50 |
+
│ gen_ppl_eval_model_name_or_path: gpt2-large
|
| 51 |
+
│ generate_samples: true
|
| 52 |
+
│ generated_samples_path: /home/ruizzhao/diffusion_baseline/results/condit
|
| 53 |
+
│
|
| 54 |
+
├── optim
|
| 55 |
+
│ └── weight_decay: 0
|
| 56 |
+
│ lr: 0.0003
|
| 57 |
+
│ beta1: 0.9
|
| 58 |
+
│ beta2: 0.999
|
| 59 |
+
│ eps: 1.0e-08
|
| 60 |
+
│
|
| 61 |
+
├── trainer
|
| 62 |
+
│ └── _target_: lightning.Trainer
|
| 63 |
+
│ accelerator: cuda
|
| 64 |
+
│ num_nodes: 1
|
| 65 |
+
│ devices: 1
|
| 66 |
+
│ accumulate_grad_batches: 256
|
| 67 |
+
│ gradient_clip_val: 1.0
|
| 68 |
+
│ precision: bf16
|
| 69 |
+
│ num_sanity_val_steps: 2
|
| 70 |
+
│ max_steps: 1000000
|
| 71 |
+
│ log_every_n_steps: 100
|
| 72 |
+
│ limit_train_batches: 1.0
|
| 73 |
+
│ limit_val_batches: 1.0
|
| 74 |
+
│ val_check_interval: 5000
|
| 75 |
+
│ check_val_every_n_epoch: null
|
| 76 |
+
│
|
| 77 |
+
├── wandb
|
| 78 |
+
│ └── project: duo
|
| 79 |
+
│ group: null
|
| 80 |
+
│ job_type: null
|
| 81 |
+
│ name: null
|
| 82 |
+
│ id: None_42
|
| 83 |
+
│ tags:
|
| 84 |
+
│ - log-linear
|
| 85 |
+
│ - lm1b
|
| 86 |
+
│ - lm1b
|
| 87 |
+
│ - duo_base
|
| 88 |
+
│ offline: true
|
| 89 |
+
│
|
| 90 |
+
├── checkpointing
|
| 91 |
+
│ └── save_dir: /home/ruizzhao/diffusion_baseline/results/conditional/samples/
|
| 92 |
+
│ resume_from_ckpt: true
|
| 93 |
+
│ resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results/conditional/
|
| 94 |
+
│
|
| 95 |
+
├── callbacks
|
| 96 |
+
│ └── checkpoint_every_n_steps:
|
| 97 |
+
│ _target_: lightning.pytorch.callbacks.ModelCheckpoint
|
| 98 |
+
│ save_top_k: -1
|
| 99 |
+
│ save_last: true
|
| 100 |
+
│ dirpath: /home/ruizzhao/diffusion_baseline/results/conditional/samples
|
| 101 |
+
│ verbose: true
|
| 102 |
+
│ auto_insert_metric_name: false
|
| 103 |
+
│ every_n_train_steps: 500
|
| 104 |
+
│ checkpoint_monitor:
|
| 105 |
+
│ _target_: lightning.pytorch.callbacks.ModelCheckpoint
|
| 106 |
+
│ monitor: val/nll
|
| 107 |
+
│ mode: min
|
| 108 |
+
│ save_top_k: 1
|
| 109 |
+
│ save_last: false
|
| 110 |
+
│ dirpath: /home/ruizzhao/diffusion_baseline/results/conditional/samples
|
| 111 |
+
│ filename: best
|
| 112 |
+
│ auto_insert_metric_name: false
|
| 113 |
+
│ verbose: true
|
| 114 |
+
│ learning_rate_monitor:
|
| 115 |
+
│ _target_: lightning.pytorch.callbacks.LearningRateMonitor
|
| 116 |
+
│ logging_interval: step
|
| 117 |
+
│
|
| 118 |
+
├── data
|
| 119 |
+
│ └── train: lm1b
|
| 120 |
+
│ valid: lm1b
|
| 121 |
+
│ modality: text
|
| 122 |
+
│ tokenizer_name_or_path: bert-base-uncased
|
| 123 |
+
│ cache_dir: /share/kuleshov/ssahoo/textdiffusion/data
|
| 124 |
+
│ wrap: true
|
| 125 |
+
│ streaming: false
|
| 126 |
+
│ insert_train_eos: true
|
| 127 |
+
│ insert_valid_eos: true
|
| 128 |
+
│
|
| 129 |
+
├── model
|
| 130 |
+
│ └── name: small
|
| 131 |
+
│ type: ddit
|
| 132 |
+
│ hidden_size: 768
|
| 133 |
+
│ cond_dim: 128
|
| 134 |
+
│ length: 128
|
| 135 |
+
│ n_blocks: 12
|
| 136 |
+
│ n_heads: 12
|
| 137 |
+
│ scale_by_sigma: true
|
| 138 |
+
│ dropout: 0.1
|
| 139 |
+
│ tie_word_embeddings: false
|
| 140 |
+
│ vocab_lookup: true
|
| 141 |
+
│
|
| 142 |
+
├── strategy
|
| 143 |
+
│ └── _target_: lightning.pytorch.strategies.DDPStrategy
|
| 144 |
+
│ find_unused_parameters: false
|
| 145 |
+
│
|
| 146 |
+
├── noise
|
| 147 |
+
│ └── type: log-linear
|
| 148 |
+
│ eps: 0.001
|
| 149 |
+
│
|
| 150 |
+
├── lr_scheduler
|
| 151 |
+
│ └── _target_: transformers.get_constant_schedule_with_warmup
|
| 152 |
+
│ num_warmup_steps: 2500
|
| 153 |
+
│
|
| 154 |
+
├── prior
|
| 155 |
+
│ └── type: none
|
| 156 |
+
│ latent_width: 0
|
| 157 |
+
│ latent_height: 0
|
| 158 |
+
│
|
| 159 |
+
└── algo
|
| 160 |
+
└── name: duo_base
|
| 161 |
+
backbone: dit
|
| 162 |
+
parameterization: mean
|
| 163 |
+
time_conditioning: true
|
| 164 |
+
T: 0
|
| 165 |
+
subs_masking: false
|
| 166 |
+
causal_attention: false
|
| 167 |
+
ignore_bos: false
|
| 168 |
+
loss_type: elbo
|
| 169 |
+
|
| 170 |
+
[2026-08-25 19:31:40,547][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
|
| 171 |
+
[2026-08-25 19:33:10,333][dlb_pinned_upstream_main][INFO] - Generative perplexity: 90.41707611083984
|
| 172 |
+
[2026-08-25 19:33:10,334][dlb_pinned_upstream_main][INFO] - Sample entropy: 4.319861888885498
|
| 173 |
+
[2026-08-25 19:33:10,340][dlb_pinned_upstream_main][INFO] - Samples saved at: /home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_8/upstream_samples.json
|
results/conditional/samples/lm1b/duo/steps_8/upstream_hydra/.hydra/hydra.yaml
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
hydra:
|
| 2 |
+
run:
|
| 3 |
+
dir: /home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_8/upstream_hydra
|
| 4 |
+
sweep:
|
| 5 |
+
dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
|
| 6 |
+
subdir: ${hydra.job.num}
|
| 7 |
+
launcher:
|
| 8 |
+
_target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
|
| 9 |
+
sweeper:
|
| 10 |
+
_target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
|
| 11 |
+
max_batch_size: null
|
| 12 |
+
params: null
|
| 13 |
+
help:
|
| 14 |
+
app_name: ${hydra.job.name}
|
| 15 |
+
header: '${hydra.help.app_name} is powered by Hydra.
|
| 16 |
+
|
| 17 |
+
'
|
| 18 |
+
footer: 'Powered by Hydra (https://hydra.cc)
|
| 19 |
+
|
| 20 |
+
Use --hydra-help to view Hydra specific help
|
| 21 |
+
|
| 22 |
+
'
|
| 23 |
+
template: '${hydra.help.header}
|
| 24 |
+
|
| 25 |
+
== Configuration groups ==
|
| 26 |
+
|
| 27 |
+
Compose your configuration from those groups (group=option)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
$APP_CONFIG_GROUPS
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
== Config ==
|
| 34 |
+
|
| 35 |
+
Override anything in the config (foo.bar=value)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
$CONFIG
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
${hydra.help.footer}
|
| 42 |
+
|
| 43 |
+
'
|
| 44 |
+
hydra_help:
|
| 45 |
+
template: 'Hydra (${hydra.runtime.version})
|
| 46 |
+
|
| 47 |
+
See https://hydra.cc for more info.
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
== Flags ==
|
| 51 |
+
|
| 52 |
+
$FLAGS_HELP
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
== Configuration groups ==
|
| 56 |
+
|
| 57 |
+
Compose your configuration from those groups (For example, append hydra/job_logging=disabled
|
| 58 |
+
to command line)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
$HYDRA_CONFIG_GROUPS
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
Use ''--cfg hydra'' to Show the Hydra config.
|
| 65 |
+
|
| 66 |
+
'
|
| 67 |
+
hydra_help: ???
|
| 68 |
+
hydra_logging:
|
| 69 |
+
version: 1
|
| 70 |
+
formatters:
|
| 71 |
+
simple:
|
| 72 |
+
format: '[%(asctime)s][HYDRA] %(message)s'
|
| 73 |
+
handlers:
|
| 74 |
+
console:
|
| 75 |
+
class: logging.StreamHandler
|
| 76 |
+
formatter: simple
|
| 77 |
+
stream: ext://sys.stdout
|
| 78 |
+
root:
|
| 79 |
+
level: INFO
|
| 80 |
+
handlers:
|
| 81 |
+
- console
|
| 82 |
+
loggers:
|
| 83 |
+
logging_example:
|
| 84 |
+
level: DEBUG
|
| 85 |
+
disable_existing_loggers: false
|
| 86 |
+
job_logging:
|
| 87 |
+
version: 1
|
| 88 |
+
formatters:
|
| 89 |
+
simple:
|
| 90 |
+
format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
|
| 91 |
+
handlers:
|
| 92 |
+
console:
|
| 93 |
+
class: logging.StreamHandler
|
| 94 |
+
formatter: simple
|
| 95 |
+
stream: ext://sys.stdout
|
| 96 |
+
file:
|
| 97 |
+
class: logging.FileHandler
|
| 98 |
+
formatter: simple
|
| 99 |
+
filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
|
| 100 |
+
root:
|
| 101 |
+
level: INFO
|
| 102 |
+
handlers:
|
| 103 |
+
- console
|
| 104 |
+
- file
|
| 105 |
+
disable_existing_loggers: false
|
| 106 |
+
env: {}
|
| 107 |
+
mode: RUN
|
| 108 |
+
searchpath: []
|
| 109 |
+
callbacks: {}
|
| 110 |
+
output_subdir: .hydra
|
| 111 |
+
overrides:
|
| 112 |
+
hydra:
|
| 113 |
+
- hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_8/upstream_hydra
|
| 114 |
+
- hydra.mode=RUN
|
| 115 |
+
task:
|
| 116 |
+
- mode=sample_eval
|
| 117 |
+
- seed=42
|
| 118 |
+
- data=lm1b-wrap
|
| 119 |
+
- model=small
|
| 120 |
+
- model.length=128
|
| 121 |
+
- loader.batch_size=2
|
| 122 |
+
- loader.eval_batch_size=1
|
| 123 |
+
- sampling.num_sample_batches=1
|
| 124 |
+
- sampling.steps=8
|
| 125 |
+
- eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/flm_baselines/lm1b/lm1b_Duo.ckpt
|
| 126 |
+
- trainer.devices=1
|
| 127 |
+
- checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_8/upstream_hydra
|
| 128 |
+
- +wandb.offline=true
|
| 129 |
+
- eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_8/upstream_samples.json
|
| 130 |
+
- algo=duo_base
|
| 131 |
+
- algo.backbone=dit
|
| 132 |
+
- sampling.predictor=ancestral
|
| 133 |
+
- sampling.noise_removal=ancestral
|
| 134 |
+
- eval.disable_ema=False
|
| 135 |
+
job:
|
| 136 |
+
name: dlb_pinned_upstream_main
|
| 137 |
+
chdir: true
|
| 138 |
+
override_dirname: +wandb.offline=true,algo.backbone=dit,algo=duo_base,checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_8/upstream_hydra,data=lm1b-wrap,eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/flm_baselines/lm1b/lm1b_Duo.ckpt,eval.disable_ema=False,eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_8/upstream_samples.json,loader.batch_size=2,loader.eval_batch_size=1,mode=sample_eval,model.length=128,model=small,sampling.noise_removal=ancestral,sampling.num_sample_batches=1,sampling.predictor=ancestral,sampling.steps=8,seed=42,trainer.devices=1
|
| 139 |
+
id: ???
|
| 140 |
+
num: ???
|
| 141 |
+
config_name: config
|
| 142 |
+
env_set: {}
|
| 143 |
+
env_copy: []
|
| 144 |
+
config:
|
| 145 |
+
override_dirname:
|
| 146 |
+
kv_sep: '='
|
| 147 |
+
item_sep: ','
|
| 148 |
+
exclude_keys: []
|
| 149 |
+
runtime:
|
| 150 |
+
version: 1.3.2
|
| 151 |
+
version_base: '1.3'
|
| 152 |
+
cwd: /home/ruizzhao/diffusion_baseline
|
| 153 |
+
config_sources:
|
| 154 |
+
- path: hydra.conf
|
| 155 |
+
schema: pkg
|
| 156 |
+
provider: hydra
|
| 157 |
+
- path: /home/ruizzhao/diffusion_baseline/upstreams/duo/configs
|
| 158 |
+
schema: file
|
| 159 |
+
provider: main
|
| 160 |
+
- path: ''
|
| 161 |
+
schema: structured
|
| 162 |
+
provider: schema
|
| 163 |
+
output_dir: /home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/duo/steps_8/upstream_hydra
|
| 164 |
+
choices:
|
| 165 |
+
algo: duo_base
|
| 166 |
+
prior: none
|
| 167 |
+
lr_scheduler: constant_warmup
|
| 168 |
+
noise: log-linear
|
| 169 |
+
strategy: ddp
|
| 170 |
+
model: small
|
| 171 |
+
data: lm1b-wrap
|
| 172 |
+
hydra/env: default
|
| 173 |
+
hydra/callbacks: null
|
| 174 |
+
hydra/job_logging: default
|
| 175 |
+
hydra/hydra_logging: default
|
| 176 |
+
hydra/hydra_help: default
|
| 177 |
+
hydra/help: default
|
| 178 |
+
hydra/sweeper: basic
|
| 179 |
+
hydra/launcher: basic
|
| 180 |
+
hydra/output: default
|
| 181 |
+
verbose: false
|
results/conditional/samples/lm1b/duo/steps_8/upstream_samples.json
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"generative_ppl": 174.0432891845703,
|
| 3 |
+
"entropy": 4.258587837219238,
|
| 4 |
+
"generated_seqs": [
|
| 5 |
+
"[CLS] [CLS] its brush between myth and territory frequently can add to the colors phrasation al meaning byries. [CLS] director : cary is chief content officer, mclaughlin systems. [CLS] a sexual first. [CLS] she assumed the operatives of a huge day in the low teens on the early afternoon of wednesday morning, where she often stood adrift next to a small delegation of dancing rather than carrying on. [CLS] at the goldie development arena, the local promoters hope venue will mark the link's 1, 400 anniversary. [CLS] the stadium boasts a classy dock that seats over 83, 000 people, from august to september. [CLS] even with the [CLS]"
|
| 6 |
+
]
|
| 7 |
+
}
|
results/conditional/samples/lm1b/duo_di4c/steps_1/samples.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
results/conditional/samples/lm1b/duo_di4c/steps_1/upstream_token_ids.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
results/conditional/samples/lm1b/flm/steps_1024/upstream_token_ids.json
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"samples": [],
|
| 3 |
+
"schema": "dlb-upstream-token-capture-v1"
|
| 4 |
+
}
|