Add files using upload-large-folder tool
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- artifacts/checkpoints.yaml +411 -0
- artifacts/elf_lock.json +21 -0
- artifacts/source_lock.json +37 -0
- data/processed/.builder-cache/lm1b/train/_home_ruizzhao_diffusion_baseline_data_processed_.builder-cache_lm1b_train_generator_default-3a58fdf789a93b0f_0.0.0.lock +0 -0
- data/processed/.builder-cache/lm1b/validation/generator/default-62c652dc08a03c0b/0.0.0.incomplete_info.lock +0 -0
- data/processed/.builder-cache/owt/train/_home_ruizzhao_diffusion_baseline_data_processed_.builder-cache_owt_train_generator_default-b8b56016fa13f36d_0.0.0.lock +0 -0
- data/processed/.builder-cache/owt/train/generator/default-b8b56016fa13f36d/0.0.0.incomplete_info.lock +0 -0
- data/processed/.builder-cache/owt/validation/generator/default-0c41487195ccfdac/0.0.0.incomplete_info.lock +0 -0
- data/processed/.builder-cache/owt/validation/generator/default-0c41487195ccfdac/0.0.0_builder.lock +0 -0
- data/processed/.builder-cache/owt/validation/generator/default-bec011984d977746/0.0.0.incomplete_info.lock +0 -0
- data/processed/.builder-cache/owt/validation/generator/default-bec011984d977746/0.0.0/dataset_info.json +1 -0
- data/processed/.builder-cache/owt/validation/generator/default-bec011984d977746/0.0.0_builder.lock +0 -0
- data/processed/owt-gpt2-1024/dataset_dict.json +1 -0
- docs/01_sources_and_availability.md +35 -0
- docs/02_datasets.md +36 -0
- docs/06_generation_and_evaluation.md +79 -0
- docs/07_slurm.md +39 -0
- docs/09_four_gpu_local.md +75 -0
- docs/12_elf_condition_recovery.md +146 -0
- docs/13_elf_eos_fix.md +178 -0
- docs/19_elf_short_response_rescore.md +75 -0
- docs/EXPERIMENT_MATRIX.md +43 -0
- docs/superpowers/plans/2026-08-25-conditional-generation.md +1116 -0
- results/_invalid/old_di4c_shape_skip_20260825_095622/samples_duo_di4c/steps_32/run_metadata.json +103 -0
- results/_invalid/old_di4c_shape_skip_20260825_095622/samples_mdlm_di4c/steps_32/samples.jsonl +0 -0
- results/_invalid/old_di4c_shape_skip_20260825_095622/samples_mdlm_di4c/steps_4/run_metadata.json +103 -0
- results/_invalid/old_di4c_shape_skip_20260825_095622/timing_mdlm_di4c/steps_4/timing.json +1 -0
- results/conditional/samples/lm1b/duo_di4c/steps_4/upstream_token_ids.json +0 -0
- results/conditional/samples/lm1b/langflow/steps_32/samples.jsonl +0 -0
- results/conditional/samples/lm1b/mdlm_di4c/steps_1/upstream_token_ids.json +0 -0
- results/conditional/samples/lm1b/mdlm_di4c/steps_2/upstream_token_ids.json +0 -0
- results/conditional/samples/lm1b/mdlm_di4c/steps_4/samples.jsonl +0 -0
- results/conditional/samples/lm1b/mdlm_sdtt/steps_16/samples.jsonl +0 -0
- results/elf-gpu01/logs/20260913T065838-1485984/owt-prefix__c64-text-t5__steps_1__seed_42__compile_0__generate.log +259 -0
- results/elf-gpu01/quality/owt-prefix/c64-text-t5/steps_1/seed_42/compile_0/request.json +51 -0
- results/elf-gpu01/quality/owt/unconditional/steps_1024/seed_42/compile_0/samples.jsonl +0 -0
- results/elf-gpu01/quality/wmt14/validation/steps_2/seed_42/compile_0/samples.jsonl +0 -0
- results/elf-gpu01/quality/wmt14/validation/steps_4/seed_42/compile_0/samples.jsonl +0 -0
- results/elf-gpu01/quality/xsum/validation/steps_4/seed_42/compile_0/request.json +60 -0
- results/elf-h200-gpu2/logs/smoke_xsum.log +2 -0
- results/elf-h200-gpu2/quality/wmt14/validation/steps_16/seed_42/compile_0/samples.jsonl +0 -0
- results/elf-owt-baseline-eval-owt-prefix/steps_32/metrics.json +70 -0
- results/elf-timing-gpu0-20260913T083438-1652790/environment/owt-prefix_validation_s32-after.txt +20 -0
- results/elf-timing-gpu1-20260915T091449-470600/environment/owt-prefix_validation_s1024-after.txt +20 -0
- results/elf-timing-gpu1-20260915T091449-470600/environment/owt_validation_s1024-after.txt +20 -0
- results/logs/4gpu-benchmark-candi-lm1b-steps-1.err +14 -0
- results/logs/4gpu-benchmark-candi-lm1b-steps-16.err +14 -0
- results/logs/4gpu-benchmark-candi-lm1b-steps-2.out +6 -0
- results/logs/4gpu-benchmark-candi-owt-steps-16.out +0 -0
- results/logs/4gpu-benchmark-candi-owt-steps-32.err +33 -0
artifacts/checkpoints.yaml
ADDED
|
@@ -0,0 +1,411 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
schema_version: 1
|
| 2 |
+
resources:
|
| 3 |
+
flm_lm1b_hf:
|
| 4 |
+
id: flm_lm1b_hf
|
| 5 |
+
backend: huggingface
|
| 6 |
+
provenance: official
|
| 7 |
+
teacher_family: continuous_flm
|
| 8 |
+
destination: official/flm/lm1b
|
| 9 |
+
license: apache-2.0
|
| 10 |
+
terms_url: https://huggingface.co/david3684/FLM-B-LM1B
|
| 11 |
+
digest: {policy: capture_after_download}
|
| 12 |
+
required_files: [config.json, model.safetensors]
|
| 13 |
+
source:
|
| 14 |
+
repo_id: david3684/FLM-B-LM1B
|
| 15 |
+
revision: cd0ca433d11d9392494de8122f3b63cc996fd4ce
|
| 16 |
+
allow_patterns: [README.md, config.json, model.safetensors]
|
| 17 |
+
fmlm_lm1b_hf:
|
| 18 |
+
id: fmlm_lm1b_hf
|
| 19 |
+
backend: huggingface
|
| 20 |
+
provenance: official
|
| 21 |
+
teacher_family: continuous_flm
|
| 22 |
+
destination: official/fmlm/lm1b
|
| 23 |
+
license: apache-2.0
|
| 24 |
+
terms_url: https://huggingface.co/david3684/FMLM-B-LM1B
|
| 25 |
+
digest: {policy: capture_after_download}
|
| 26 |
+
required_files: [config.json, model.safetensors]
|
| 27 |
+
source:
|
| 28 |
+
repo_id: david3684/FMLM-B-LM1B
|
| 29 |
+
revision: 356592ce2d9af4adbf9d7b55e8e00bbfb37eb4c8
|
| 30 |
+
allow_patterns: [README.md, config.json, model.safetensors]
|
| 31 |
+
flm_owt_hf:
|
| 32 |
+
id: flm_owt_hf
|
| 33 |
+
backend: huggingface
|
| 34 |
+
provenance: official
|
| 35 |
+
teacher_family: continuous_flm
|
| 36 |
+
destination: official/flm/owt
|
| 37 |
+
license: apache-2.0
|
| 38 |
+
terms_url: https://huggingface.co/david3684/FLM-B-OWT
|
| 39 |
+
digest: {policy: capture_after_download}
|
| 40 |
+
required_files: [config.json, model.safetensors]
|
| 41 |
+
source:
|
| 42 |
+
repo_id: david3684/FLM-B-OWT
|
| 43 |
+
revision: 624471b934fdd0421757d62290f7e639f32566d3
|
| 44 |
+
allow_patterns: [README.md, config.json, model.safetensors]
|
| 45 |
+
fmlm_owt_hf:
|
| 46 |
+
id: fmlm_owt_hf
|
| 47 |
+
backend: huggingface
|
| 48 |
+
provenance: official
|
| 49 |
+
teacher_family: continuous_flm
|
| 50 |
+
destination: official/fmlm/owt
|
| 51 |
+
license: apache-2.0
|
| 52 |
+
terms_url: https://huggingface.co/david3684/FMLM-B-OWT
|
| 53 |
+
digest: {policy: capture_after_download}
|
| 54 |
+
required_files: [config.json, model.safetensors]
|
| 55 |
+
source:
|
| 56 |
+
repo_id: david3684/FMLM-B-OWT
|
| 57 |
+
revision: 483ea1b38bba56632cd40dc5a3c70a2340bb4946
|
| 58 |
+
allow_patterns: [README.md, config.json, model.safetensors]
|
| 59 |
+
langflow_owt_hf:
|
| 60 |
+
id: langflow_owt_hf
|
| 61 |
+
backend: huggingface
|
| 62 |
+
provenance: official
|
| 63 |
+
teacher_family: continuous_langflow
|
| 64 |
+
destination: official/langflow/owt
|
| 65 |
+
license: apache-2.0
|
| 66 |
+
terms_url: https://huggingface.co/Continuous-Rivals-Discrete/langflow-owt
|
| 67 |
+
digest: {policy: capture_after_download}
|
| 68 |
+
required_files: [config.json, config.py, model.py, model.safetensors]
|
| 69 |
+
source:
|
| 70 |
+
repo_id: Continuous-Rivals-Discrete/langflow-owt
|
| 71 |
+
revision: a08f933dd337d52762fec5ef7d60c131896cc341
|
| 72 |
+
allow_patterns: [README.md, __init__.py, config.json, config.py, model.py, model.safetensors]
|
| 73 |
+
langflow_lm1b_hf:
|
| 74 |
+
id: langflow_lm1b_hf
|
| 75 |
+
backend: huggingface
|
| 76 |
+
provenance: official
|
| 77 |
+
teacher_family: continuous_langflow
|
| 78 |
+
destination: official/langflow/lm1b
|
| 79 |
+
license: apache-2.0
|
| 80 |
+
terms_url: https://huggingface.co/Continuous-Rivals-Discrete/langflow-lm1b
|
| 81 |
+
digest: {policy: capture_after_download}
|
| 82 |
+
required_files: [config.json, config.py, model.py, model.safetensors]
|
| 83 |
+
source:
|
| 84 |
+
repo_id: Continuous-Rivals-Discrete/langflow-lm1b
|
| 85 |
+
revision: d5d395d9bdf01f697ba8830110e3e78c63d77cb5
|
| 86 |
+
allow_patterns: [README.md, __init__.py, config.json, config.py, model.py, model.safetensors]
|
| 87 |
+
duo_owt_hf:
|
| 88 |
+
id: duo_owt_hf
|
| 89 |
+
backend: huggingface
|
| 90 |
+
provenance: official
|
| 91 |
+
teacher_family: uniform_duo
|
| 92 |
+
destination: official/duo/owt
|
| 93 |
+
license: apache-2.0
|
| 94 |
+
terms_url: https://huggingface.co/s-sahoo/duo
|
| 95 |
+
digest: {policy: capture_after_download}
|
| 96 |
+
required_files: [config.json, config.py, model.py, model.safetensors]
|
| 97 |
+
source:
|
| 98 |
+
repo_id: s-sahoo/duo
|
| 99 |
+
revision: ef8ad7a25a4bf9624306230913dd7ae5cb2a00bd
|
| 100 |
+
allow_patterns: [README.md, config.json, config.py, model.py, model.safetensors]
|
| 101 |
+
duo_dcd_owt_hf:
|
| 102 |
+
id: duo_dcd_owt_hf
|
| 103 |
+
backend: huggingface
|
| 104 |
+
provenance: official
|
| 105 |
+
teacher_family: uniform_duo
|
| 106 |
+
destination: official/duo_dcd/owt
|
| 107 |
+
license: apache-2.0
|
| 108 |
+
terms_url: https://huggingface.co/s-sahoo/duo-distilled
|
| 109 |
+
digest: {policy: capture_after_download}
|
| 110 |
+
required_files: [config.json, config.py, model.py, model.safetensors]
|
| 111 |
+
source:
|
| 112 |
+
repo_id: s-sahoo/duo-distilled
|
| 113 |
+
revision: 08ee71faabfc0e2eaaa6cb1833ed3a486931ef2b
|
| 114 |
+
allow_patterns: [README.md, config.json, config.py, model.py, model.safetensors]
|
| 115 |
+
mdlm_owt_hf:
|
| 116 |
+
id: mdlm_owt_hf
|
| 117 |
+
backend: huggingface
|
| 118 |
+
provenance: official
|
| 119 |
+
teacher_family: masked_mdlm
|
| 120 |
+
destination: official/mdlm/owt
|
| 121 |
+
license: apache-2.0
|
| 122 |
+
terms_url: https://huggingface.co/kuleshov-group/mdlm-owt
|
| 123 |
+
digest: {policy: capture_after_download}
|
| 124 |
+
required_files: [config.json, configuration_mdlm.py, modeling_mdlm.py, model.safetensors]
|
| 125 |
+
source:
|
| 126 |
+
repo_id: kuleshov-group/mdlm-owt
|
| 127 |
+
revision: d0958fa851335ece6c15260ce0025f030673c0fb
|
| 128 |
+
allow_patterns: [README.md, config.json, configuration_mdlm.py, modeling_mdlm.py, model.safetensors]
|
| 129 |
+
sdtt_owt_hf:
|
| 130 |
+
id: sdtt_owt_hf
|
| 131 |
+
backend: huggingface
|
| 132 |
+
provenance: official
|
| 133 |
+
teacher_family: masked_mdlm
|
| 134 |
+
destination: official/mdlm_sdtt/owt
|
| 135 |
+
license: apache-2.0
|
| 136 |
+
terms_url: https://huggingface.co/jdeschena/sdtt/tree/baselines_kld_step_70000
|
| 137 |
+
digest: {policy: capture_after_download}
|
| 138 |
+
required_files: [config.json, model.safetensors]
|
| 139 |
+
source:
|
| 140 |
+
repo_id: jdeschena/sdtt
|
| 141 |
+
revision: c84bdf45ce3a80f5a70bb09548093f1c8f2ac7a7
|
| 142 |
+
allow_patterns: [config.json, model.safetensors]
|
| 143 |
+
note: Final round-7 KLD baseline student from the masked MDLM teacher.
|
| 144 |
+
flm_lm1b_reproductions:
|
| 145 |
+
id: flm_lm1b_reproductions
|
| 146 |
+
backend: gdrive
|
| 147 |
+
provenance: reference_reproduction
|
| 148 |
+
teacher_family: mixed_baselines
|
| 149 |
+
destination: reference_reproduction/flm_baselines/lm1b
|
| 150 |
+
license: upstream-not-specified
|
| 151 |
+
terms_url: https://github.com/david3684/flm/blob/a1918d5164e5038e37d0b7a4fb2010ce75b863b3/README.md
|
| 152 |
+
digest: {policy: capture_after_download}
|
| 153 |
+
required_files: [lm1b_CANDI.ckpt, lm1b_Duo.ckpt, lm1b_MDLM_.ckpt]
|
| 154 |
+
source:
|
| 155 |
+
folder_id: 1TJO3aFWqI7ukbmjciZ6krAUFlAak1itl
|
| 156 |
+
expected_files:
|
| 157 |
+
lm1b_CANDI.ckpt: 19TbJu1VHWX5EHqE8Si4xWqBhJlTqaGjf
|
| 158 |
+
lm1b_Duo.ckpt: 1yHtaYrLX34rS5tu9V1SKF5NvwTVxPWoF
|
| 159 |
+
lm1b_MDLM_.ckpt: 1jp6Jka8tQJJ9zljihdLErokFp9TDvlsl
|
| 160 |
+
note: Supplemental reproduction folder; cells remain recipe-backed until server lock inspection proves exact filenames and teacher families.
|
| 161 |
+
duo_raw_owt_drive:
|
| 162 |
+
id: duo_raw_owt_drive
|
| 163 |
+
backend: gdrive
|
| 164 |
+
provenance: official
|
| 165 |
+
teacher_family: uniform_duo
|
| 166 |
+
destination: official/duo/raw_owt
|
| 167 |
+
license: apache-2.0-code-checkpoint-terms-upstream
|
| 168 |
+
terms_url: https://github.com/s-sahoo/duo/blob/7c9b498f5b717de064d6fad7e2509c866e6cb620/README.md
|
| 169 |
+
digest: {policy: capture_after_download}
|
| 170 |
+
required_files: [duo-distilled.ckpt, duo.ckpt]
|
| 171 |
+
source:
|
| 172 |
+
folder_id: 1JpqFM8XRvifwIkjWPfMyuDvu41r1yk0t
|
| 173 |
+
expected_files:
|
| 174 |
+
duo-distilled.ckpt: 1WaF4ITJ24rl6JhIYKfrA4FWvXaAPZjJL
|
| 175 |
+
duo.ckpt: 1SbslPVBrSWrN3V3TlngjcLapROa74o0_
|
| 176 |
+
note: Official finetunable raw OWT checkpoint folder; HF snapshots remain canonical for evaluation.
|
| 177 |
+
rdlm_lm1b_drive:
|
| 178 |
+
id: rdlm_lm1b_drive
|
| 179 |
+
backend: gdrive
|
| 180 |
+
provenance: official
|
| 181 |
+
teacher_family: continuous_rdlm
|
| 182 |
+
destination: official/rdlm/lm1b
|
| 183 |
+
license: upstream-not-specified
|
| 184 |
+
terms_url: https://github.com/harryjo97/RDLM/blob/67443aa6a2d0fa981eb7c6105f9cbc563e59c5c1/README.md
|
| 185 |
+
digest: {policy: capture_after_download}
|
| 186 |
+
required_files:
|
| 187 |
+
- LM1B/checkpoint.pth
|
| 188 |
+
- LM1B/config.yaml
|
| 189 |
+
- LM1B/sde.pkl
|
| 190 |
+
- Text8/checkpoint.pth
|
| 191 |
+
- Text8/config.yaml
|
| 192 |
+
- Text8/sde.pkl
|
| 193 |
+
source:
|
| 194 |
+
folder_id: 1aDTZtPIxAxQrkaRSahjuWbkxX1OYq9CC
|
| 195 |
+
expected_files:
|
| 196 |
+
LM1B/checkpoint.pth: 1zAqP-DtZigoiChpNfgJgJobKIZ9VOaro
|
| 197 |
+
LM1B/config.yaml: 1-rXPglhh3BqDQ0XlQqQ0wN8Gu2iNg5KU
|
| 198 |
+
LM1B/sde.pkl: 1I2yLxRSOk4jlBryFhwOfJ6L3UBmljwl_
|
| 199 |
+
Text8/checkpoint.pth: 17x9jd44hfUuhByh9XsZEeClwSnsCNQIY
|
| 200 |
+
Text8/config.yaml: 158JPIYFGm5cBRlQ2ZmY__9t4UYl9-CQl
|
| 201 |
+
Text8/sde.pkl: 1M3mzJFCOnXsQF9MKfrUGcn8sAbKtaLBh
|
| 202 |
+
note: Folder also contains Text8; server lock records the observed LM1B checkpoint, config, and SDE files.
|
| 203 |
+
di4c_mdlm_owt_zenodo:
|
| 204 |
+
id: di4c_mdlm_owt_zenodo
|
| 205 |
+
backend: zenodo
|
| 206 |
+
provenance: official
|
| 207 |
+
teacher_family: masked_mdlm
|
| 208 |
+
destination: official/mdlm_di4c/owt
|
| 209 |
+
license: zenodo-record-terms
|
| 210 |
+
terms_url: https://zenodo.org/records/15124163
|
| 211 |
+
digest: {policy: capture_after_download}
|
| 212 |
+
required_files: [sdtt7-di4c2.ckpt]
|
| 213 |
+
source:
|
| 214 |
+
record_id: 15124163
|
| 215 |
+
files: [sdtt7-di4c2.ckpt]
|
| 216 |
+
published_checksums:
|
| 217 |
+
sdtt7-di4c2.ckpt: md5:b55b961b1f0d7f2ed55539452875ec15
|
| 218 |
+
note: Masked SDTT/MDLM teacher release only; never valid for Duo/uniform Di4C cells.
|
| 219 |
+
recipes:
|
| 220 |
+
flm_lm1b_official_ckpt:
|
| 221 |
+
model: flm
|
| 222 |
+
dataset: lm1b
|
| 223 |
+
provenance: official
|
| 224 |
+
teacher_family: continuous_flm
|
| 225 |
+
source: flm
|
| 226 |
+
source_commit: a1918d5164e5038e37d0b7a4fb2010ce75b863b3
|
| 227 |
+
command: >-
|
| 228 |
+
gdown --folder https://drive.google.com/drive/folders/1fNAx4LP2RwPBdqDQFQ_gRrYZI9u3Vq15
|
| 229 |
+
-O downloads/flm_official_ckpt && mkdir -p checkpoints/official/flm_ckpt/lm1b
|
| 230 |
+
&& cp downloads/flm_official_ckpt/lm1b_flm.ckpt
|
| 231 |
+
checkpoints/official/flm_ckpt/lm1b/lm1b_flm.ckpt
|
| 232 |
+
output: checkpoints/official/flm_ckpt/lm1b
|
| 233 |
+
sampling_checkpoint: lm1b_flm.ckpt
|
| 234 |
+
note: Official FLM Lightning checkpoint staged from the author's public Drive release.
|
| 235 |
+
flm_owt_official_ckpt:
|
| 236 |
+
model: flm
|
| 237 |
+
dataset: owt
|
| 238 |
+
provenance: official
|
| 239 |
+
teacher_family: continuous_flm
|
| 240 |
+
source: flm
|
| 241 |
+
source_commit: a1918d5164e5038e37d0b7a4fb2010ce75b863b3
|
| 242 |
+
command: >-
|
| 243 |
+
gdown --folder https://drive.google.com/drive/folders/1fNAx4LP2RwPBdqDQFQ_gRrYZI9u3Vq15
|
| 244 |
+
-O downloads/flm_official_ckpt && mkdir -p checkpoints/official/flm_ckpt/owt
|
| 245 |
+
&& cp downloads/flm_official_ckpt/owt_flm.ckpt
|
| 246 |
+
checkpoints/official/flm_ckpt/owt/owt_flm.ckpt
|
| 247 |
+
output: checkpoints/official/flm_ckpt/owt
|
| 248 |
+
sampling_checkpoint: owt_flm.ckpt
|
| 249 |
+
note: Official FLM Lightning checkpoint staged from the author's public Drive release.
|
| 250 |
+
fmlm_lm1b_official_ckpt:
|
| 251 |
+
model: fmlm
|
| 252 |
+
dataset: lm1b
|
| 253 |
+
provenance: official
|
| 254 |
+
teacher_family: continuous_flm
|
| 255 |
+
source: flm
|
| 256 |
+
source_commit: a1918d5164e5038e37d0b7a4fb2010ce75b863b3
|
| 257 |
+
command: >-
|
| 258 |
+
gdown --folder https://drive.google.com/drive/folders/1fNAx4LP2RwPBdqDQFQ_gRrYZI9u3Vq15
|
| 259 |
+
-O downloads/flm_official_ckpt && mkdir -p checkpoints/official/fmlm_ckpt/lm1b
|
| 260 |
+
&& cp downloads/flm_official_ckpt/lm1b_fmlm.ckpt
|
| 261 |
+
checkpoints/official/fmlm_ckpt/lm1b/lm1b_fmlm.ckpt
|
| 262 |
+
output: checkpoints/official/fmlm_ckpt/lm1b
|
| 263 |
+
sampling_checkpoint: lm1b_fmlm.ckpt
|
| 264 |
+
note: Official FMLM Lightning checkpoint staged from the author's public Drive release.
|
| 265 |
+
fmlm_owt_official_ckpt:
|
| 266 |
+
model: fmlm
|
| 267 |
+
dataset: owt
|
| 268 |
+
provenance: official
|
| 269 |
+
teacher_family: continuous_flm
|
| 270 |
+
source: flm
|
| 271 |
+
source_commit: a1918d5164e5038e37d0b7a4fb2010ce75b863b3
|
| 272 |
+
command: >-
|
| 273 |
+
gdown --folder https://drive.google.com/drive/folders/1fNAx4LP2RwPBdqDQFQ_gRrYZI9u3Vq15
|
| 274 |
+
-O downloads/flm_official_ckpt && mkdir -p checkpoints/official/fmlm_ckpt/owt
|
| 275 |
+
&& cp downloads/flm_official_ckpt/owt_fmlm.ckpt
|
| 276 |
+
checkpoints/official/fmlm_ckpt/owt/owt_fmlm.ckpt
|
| 277 |
+
output: checkpoints/official/fmlm_ckpt/owt
|
| 278 |
+
sampling_checkpoint: owt_fmlm.ckpt
|
| 279 |
+
note: Official FMLM Lightning checkpoint staged from the author's public Drive release.
|
| 280 |
+
candi_owt:
|
| 281 |
+
model: candi
|
| 282 |
+
dataset: owt
|
| 283 |
+
provenance: reference_reproduction
|
| 284 |
+
teacher_family: hybrid_candi
|
| 285 |
+
source: candi
|
| 286 |
+
source_commit: cd57ae9eec98d6ac71cd52bdc50eeec8dfd70f91
|
| 287 |
+
command: >-
|
| 288 |
+
bash scripts/train/candi.sh --source upstreams/candi --dataset owt
|
| 289 |
+
--output checkpoints/reference_reproduction/candi/owt
|
| 290 |
+
output: checkpoints/reference_reproduction/candi/owt
|
| 291 |
+
sampling_checkpoint: model.ckpt
|
| 292 |
+
sampling_config: config.yaml
|
| 293 |
+
note: Explicit prerequisite for the project-owned Task 12 wrapper; the upstream site-specific Slurm job is not invoked directly.
|
| 294 |
+
duo_dcd_lm1b:
|
| 295 |
+
model: duo_dcd
|
| 296 |
+
dataset: lm1b
|
| 297 |
+
provenance: reference_reproduction
|
| 298 |
+
teacher_family: uniform_duo
|
| 299 |
+
source: duo
|
| 300 |
+
source_commit: 7c9b498f5b717de064d6fad7e2509c866e6cb620
|
| 301 |
+
command: >-
|
| 302 |
+
bash scripts/distill/duo_dcd.sh --source upstreams/duo --dataset lm1b
|
| 303 |
+
--teacher checkpoints/reference_reproduction/flm_baselines/lm1b/lm1b_Duo.ckpt
|
| 304 |
+
--output checkpoints/reference_reproduction/duo_dcd/lm1b
|
| 305 |
+
--rounds 8 --steps-per-round 10000 --global-batch-size 128
|
| 306 |
+
--learning-rate 6e-5
|
| 307 |
+
output: checkpoints/reference_reproduction/duo_dcd/lm1b
|
| 308 |
+
sampling_checkpoint: model.ckpt
|
| 309 |
+
sampling_config: config.yaml
|
| 310 |
+
prerequisites: [flm_lm1b_reproductions]
|
| 311 |
+
note: Explicit prerequisite for the project-owned Task 12 wrapper; do not substitute the OWT distilled snapshot or alter the reference schedule.
|
| 312 |
+
duo_di4c_lm1b:
|
| 313 |
+
model: duo_di4c
|
| 314 |
+
dataset: lm1b
|
| 315 |
+
provenance: reference_reproduction
|
| 316 |
+
teacher_family: uniform_duo
|
| 317 |
+
source: di4c
|
| 318 |
+
source_commit: ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3
|
| 319 |
+
teacher_adapter: uniform_to_absorbing
|
| 320 |
+
command: >-
|
| 321 |
+
bash scripts/distill/di4c.sh --model duo_di4c --dataset lm1b
|
| 322 |
+
--teacher-family uniform_duo
|
| 323 |
+
--teacher-checkpoint checkpoints/reference_reproduction/flm_baselines/lm1b/lm1b_Duo.ckpt
|
| 324 |
+
--output checkpoints/reference_reproduction/duo_di4c/lm1b
|
| 325 |
+
--student-init teacher
|
| 326 |
+
--upstream-override is_di4c=true
|
| 327 |
+
output: checkpoints/reference_reproduction/duo_di4c/lm1b
|
| 328 |
+
sampling_checkpoint: student_checkpoints/20000.ckpt
|
| 329 |
+
sampling_config: config.yaml
|
| 330 |
+
prerequisites: [flm_lm1b_reproductions]
|
| 331 |
+
note: Task 12's repository wrapper must adapt the uniform Duo vocabulary/head to the absorbing Di4C shape; Zenodo SDTT files are forbidden substitutes.
|
| 332 |
+
duo_di4c_owt:
|
| 333 |
+
model: duo_di4c
|
| 334 |
+
dataset: owt
|
| 335 |
+
provenance: reference_reproduction
|
| 336 |
+
teacher_family: uniform_duo
|
| 337 |
+
source: di4c
|
| 338 |
+
source_commit: ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3
|
| 339 |
+
teacher_adapter: uniform_to_absorbing
|
| 340 |
+
command: >-
|
| 341 |
+
bash scripts/distill/di4c.sh --model duo_di4c --dataset owt
|
| 342 |
+
--teacher-family uniform_duo
|
| 343 |
+
--teacher-checkpoint checkpoints/official/duo/raw_owt/duo.ckpt
|
| 344 |
+
--output checkpoints/reference_reproduction/duo_di4c/owt
|
| 345 |
+
--student-init hf_small
|
| 346 |
+
--upstream-override is_di4c=true
|
| 347 |
+
output: checkpoints/reference_reproduction/duo_di4c/owt
|
| 348 |
+
sampling_checkpoint: student_checkpoints/50000.ckpt
|
| 349 |
+
sampling_config: config.yaml
|
| 350 |
+
prerequisites: [duo_raw_owt_drive]
|
| 351 |
+
note: Task 12's repository wrapper must adapt the uniform Duo vocabulary/head to the absorbing Di4C shape; never relabel the masked-teacher Zenodo checkpoint.
|
| 352 |
+
mdlm_sdtt_lm1b:
|
| 353 |
+
model: mdlm_sdtt
|
| 354 |
+
dataset: lm1b
|
| 355 |
+
provenance: reference_reproduction
|
| 356 |
+
teacher_family: masked_mdlm
|
| 357 |
+
source: sdtt
|
| 358 |
+
source_commit: 1150985e90b8f2d5749e4469d5154eff9ec922c4
|
| 359 |
+
teacher_adapter: masked_to_absorbing
|
| 360 |
+
command: >-
|
| 361 |
+
bash scripts/distill/mdlm_sdtt.sh --model mdlm_sdtt --dataset lm1b
|
| 362 |
+
--teacher-family masked_mdlm
|
| 363 |
+
--teacher-checkpoint checkpoints/reference_reproduction/flm_baselines/lm1b/lm1b_MDLM_.ckpt
|
| 364 |
+
--output checkpoints/reference_reproduction/mdlm_sdtt/lm1b
|
| 365 |
+
output: checkpoints/reference_reproduction/mdlm_sdtt/lm1b
|
| 366 |
+
sampling_checkpoint: student_checkpoints/70000.ckpt
|
| 367 |
+
sampling_config: config.yaml
|
| 368 |
+
prerequisites: [flm_lm1b_reproductions]
|
| 369 |
+
note: Task 12's repository wrapper must adapt the BERT masked teacher to the absorbing SDTT shape; the upstream loader is not shape-compatible directly.
|
| 370 |
+
mdlm_di4c_lm1b:
|
| 371 |
+
model: mdlm_di4c
|
| 372 |
+
dataset: lm1b
|
| 373 |
+
provenance: reference_reproduction
|
| 374 |
+
teacher_family: masked_mdlm
|
| 375 |
+
source: di4c
|
| 376 |
+
source_commit: ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3
|
| 377 |
+
teacher_adapter: masked_to_absorbing
|
| 378 |
+
command: >-
|
| 379 |
+
bash scripts/distill/di4c.sh --model mdlm_di4c --dataset lm1b
|
| 380 |
+
--teacher-family masked_mdlm
|
| 381 |
+
--teacher-checkpoint checkpoints/reference_reproduction/flm_baselines/lm1b/lm1b_MDLM_.ckpt
|
| 382 |
+
--output checkpoints/reference_reproduction/mdlm_di4c/lm1b
|
| 383 |
+
--student-init teacher
|
| 384 |
+
--upstream-override is_di4c=true
|
| 385 |
+
output: checkpoints/reference_reproduction/mdlm_di4c/lm1b
|
| 386 |
+
sampling_checkpoint: student_checkpoints/20000.ckpt
|
| 387 |
+
sampling_config: config.yaml
|
| 388 |
+
prerequisites: [flm_lm1b_reproductions]
|
| 389 |
+
note: Task 12's repository wrapper must adapt the BERT masked teacher to the absorbing Di4C shape; Zenodo contains OWT students only.
|
| 390 |
+
coverage:
|
| 391 |
+
- {model: langflow, dataset: lm1b, resource: langflow_lm1b_hf}
|
| 392 |
+
- {model: langflow, dataset: owt, resource: langflow_owt_hf}
|
| 393 |
+
- {model: duo, dataset: lm1b, resource: flm_lm1b_reproductions, path: lm1b_Duo.ckpt, teacher_family: uniform_duo}
|
| 394 |
+
- {model: duo, dataset: owt, resource: duo_owt_hf}
|
| 395 |
+
- {model: duo_dcd, dataset: owt, resource: duo_dcd_owt_hf}
|
| 396 |
+
- {model: mdlm, dataset: lm1b, resource: flm_lm1b_reproductions, path: lm1b_MDLM_.ckpt, teacher_family: masked_mdlm}
|
| 397 |
+
- {model: mdlm, dataset: owt, resource: mdlm_owt_hf}
|
| 398 |
+
- {model: candi, dataset: lm1b, resource: flm_lm1b_reproductions, path: lm1b_CANDI.ckpt, teacher_family: hybrid_candi}
|
| 399 |
+
- {model: rdlm, dataset: lm1b, resource: rdlm_lm1b_drive}
|
| 400 |
+
- model: mdlm_sdtt
|
| 401 |
+
dataset: owt
|
| 402 |
+
resource: sdtt_owt_hf
|
| 403 |
+
sampling_config: config.json
|
| 404 |
+
sampling_config_source: resource
|
| 405 |
+
- model: mdlm_di4c
|
| 406 |
+
dataset: owt
|
| 407 |
+
resource: di4c_mdlm_owt_zenodo
|
| 408 |
+
sampling_config: configs/sampling/di4c_mdlm_owt.yaml
|
| 409 |
+
sampling_config_source: project
|
| 410 |
+
sampling_config_sha256: cf627f89b8dede9b25ac8e10407c3b7de203a76c4ba84100723287e571dcec34
|
| 411 |
+
sampling_config_source_commit: ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3
|
artifacts/elf_lock.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 1,
|
| 3 |
+
"verified_at": "2026-09-10",
|
| 4 |
+
"source": {
|
| 5 |
+
"url": "https://github.com/lillian039/ELF.git",
|
| 6 |
+
"branch": "pytorch_elf",
|
| 7 |
+
"commit": "b29d8833609e9ab7f67cd9da39435ac5cea04837"
|
| 8 |
+
},
|
| 9 |
+
"assets": {
|
| 10 |
+
"owt": {"repo_id": "embedded-language-flows/ELF-B-owt-torch", "revision": "146f84133c1389bfd4ef47f14ec7a955da22faa7", "repo_type": "model", "patterns": ["checkpoint_95085", "config.yml"], "checkpoint": "checkpoint_95085"},
|
| 11 |
+
"wmt14": {"repo_id": "embedded-language-flows/ELF-B-de-en-torch", "revision": "93ce98315a2dec985cdefb8d8e62ab618e896ec4", "repo_type": "model", "patterns": ["checkpoint_880600", "config.yml"], "checkpoint": "checkpoint_880600"},
|
| 12 |
+
"xsum": {"repo_id": "embedded-language-flows/ELF-B-xsum-torch", "revision": "aca4abd1c7ff5f841f0e2a3ec8a18b0a927927f1", "repo_type": "model", "patterns": ["checkpoint_39800", "config.yml"], "checkpoint": "checkpoint_39800"},
|
| 13 |
+
"t5": {"repo_id": "google-t5/t5-small", "revision": "df1b051c49625cf57a3d0d8d3863ed4d13564fe4", "repo_type": "model", "patterns": ["config.json", "model.safetensors", "spiece.model", "tokenizer.json", "tokenizer_config.json"]},
|
| 14 |
+
"gpt2": {"repo_id": "gpt2", "revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e", "repo_type": "model", "patterns": ["config.json", "tokenizer.json", "tokenizer_config.json", "vocab.json", "merges.txt"]},
|
| 15 |
+
"gpt2-large": {"repo_id": "gpt2-large", "revision": "32b71b12589c2f8d625668d2335a01cac3249519", "repo_type": "model", "patterns": ["config.json", "model.safetensors", "tokenizer.json", "tokenizer_config.json", "vocab.json", "merges.txt"]},
|
| 16 |
+
"wmt14-official-validation": {"repo_id": "embedded-language-flows/wmt14_de-en_validation_t5", "revision": "bcc92bda4b07c329354843724c9d9ea740f30685", "repo_type": "dataset", "patterns": ["*.arrow", "*.json"]},
|
| 17 |
+
"xsum-official-validation": {"repo_id": "embedded-language-flows/xsum_validation_t5", "revision": "b0b4f5de92434d308eceeb9e38fd3f8d7d2b6a9e", "repo_type": "dataset", "patterns": ["*.arrow", "*.json"]},
|
| 18 |
+
"wmt14-raw": {"repo_id": "wmt/wmt14", "revision": "b199e406369ec1b7634206d3ded5ba45de2fe696", "repo_type": "dataset", "patterns": ["de-en/validation-*.parquet", "de-en/test-*.parquet"]},
|
| 19 |
+
"xsum-raw": {"repo_id": "EdinburghNLP/xsum", "revision": "7d4d486c2f8ef850b1a11aead99b894ff3dd7da9", "repo_type": "dataset", "patterns": ["data/validation-*.parquet", "data/test-*.parquet"]}
|
| 20 |
+
}
|
| 21 |
+
}
|
artifacts/source_lock.json
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"manifest_sha256": "b7b0b94f1f7dfcd037110fca3d3b7f0163b551985a542efc0d014d689fde6a4d",
|
| 3 |
+
"sources": {
|
| 4 |
+
"candi": {
|
| 5 |
+
"commit": "cd57ae9eec98d6ac71cd52bdc50eeec8dfd70f91",
|
| 6 |
+
"url": "https://github.com/patrickpynadath1/candi-diffusion.git"
|
| 7 |
+
},
|
| 8 |
+
"di4c": {
|
| 9 |
+
"commit": "ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3",
|
| 10 |
+
"url": "https://github.com/sony/di4c.git"
|
| 11 |
+
},
|
| 12 |
+
"duo": {
|
| 13 |
+
"commit": "7c9b498f5b717de064d6fad7e2509c866e6cb620",
|
| 14 |
+
"url": "https://github.com/s-sahoo/duo.git"
|
| 15 |
+
},
|
| 16 |
+
"flm": {
|
| 17 |
+
"commit": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
|
| 18 |
+
"url": "https://github.com/david3684/flm.git"
|
| 19 |
+
},
|
| 20 |
+
"langflow": {
|
| 21 |
+
"commit": "a712b08570c56c787d6ef24f8e906a2dcdf768f5",
|
| 22 |
+
"url": "https://github.com/nealchen2003/LangFlow.git"
|
| 23 |
+
},
|
| 24 |
+
"mdlm": {
|
| 25 |
+
"commit": "c112c526d193436838c98d81455ee51f90309470",
|
| 26 |
+
"url": "https://github.com/kuleshov-group/mdlm.git"
|
| 27 |
+
},
|
| 28 |
+
"rdlm": {
|
| 29 |
+
"commit": "67443aa6a2d0fa981eb7c6105f9cbc563e59c5c1",
|
| 30 |
+
"url": "https://github.com/harryjo97/RDLM.git"
|
| 31 |
+
},
|
| 32 |
+
"sdtt": {
|
| 33 |
+
"commit": "1150985e90b8f2d5749e4469d5154eff9ec922c4",
|
| 34 |
+
"url": "https://github.com/jdeschena/sdtt.git"
|
| 35 |
+
}
|
| 36 |
+
}
|
| 37 |
+
}
|
data/processed/.builder-cache/lm1b/train/_home_ruizzhao_diffusion_baseline_data_processed_.builder-cache_lm1b_train_generator_default-3a58fdf789a93b0f_0.0.0.lock
ADDED
|
File without changes
|
data/processed/.builder-cache/lm1b/validation/generator/default-62c652dc08a03c0b/0.0.0.incomplete_info.lock
ADDED
|
File without changes
|
data/processed/.builder-cache/owt/train/_home_ruizzhao_diffusion_baseline_data_processed_.builder-cache_owt_train_generator_default-b8b56016fa13f36d_0.0.0.lock
ADDED
|
File without changes
|
data/processed/.builder-cache/owt/train/generator/default-b8b56016fa13f36d/0.0.0.incomplete_info.lock
ADDED
|
File without changes
|
data/processed/.builder-cache/owt/validation/generator/default-0c41487195ccfdac/0.0.0.incomplete_info.lock
ADDED
|
File without changes
|
data/processed/.builder-cache/owt/validation/generator/default-0c41487195ccfdac/0.0.0_builder.lock
ADDED
|
File without changes
|
data/processed/.builder-cache/owt/validation/generator/default-bec011984d977746/0.0.0.incomplete_info.lock
ADDED
|
File without changes
|
data/processed/.builder-cache/owt/validation/generator/default-bec011984d977746/0.0.0/dataset_info.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"description": "", "citation": "", "homepage": "", "license": "", "features": {"input_ids": {"feature": {"dtype": "uint16", "_type": "Value"}, "length": 1024, "_type": "Sequence"}}, "builder_name": "generator", "dataset_name": "generator", "config_name": "default", "version": {"version_str": "0.0.0", "major": 0, "minor": 0, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 226344960, "num_examples": 110520, "dataset_name": "generator"}}, "download_checksums": {}, "download_size": 0, "dataset_size": 226344960, "size_in_bytes": 226344960}
|
data/processed/.builder-cache/owt/validation/generator/default-bec011984d977746/0.0.0_builder.lock
ADDED
|
File without changes
|
data/processed/owt-gpt2-1024/dataset_dict.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"splits": ["train", "validation"]}
|
docs/01_sources_and_availability.md
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 01. 源码与可用性
|
| 2 |
+
|
| 3 |
+
## 服务器端下载
|
| 4 |
+
|
| 5 |
+
源码仓库由 `artifacts/sources.yaml` 固定 URL 和 40 位 commit。服务器执行:
|
| 6 |
+
|
| 7 |
+
```bash
|
| 8 |
+
export DLB_ROOT="$PWD"
|
| 9 |
+
conda activate dlb-bootstrap
|
| 10 |
+
export DLB_PYTHON="$CONDA_PREFIX/bin/python"
|
| 11 |
+
bash scripts/fetch_sources.sh
|
| 12 |
+
"$DLB_PYTHON" scripts/verify_sources.py --root "$DLB_ROOT"
|
| 13 |
+
```
|
| 14 |
+
|
| 15 |
+
`fetch_sources.sh` 优先使用 `DLB_PYTHON`,否则使用 `PYTHON_BIN`、当前 Conda 环境的 `python`、PATH 中的 `python` 或 `python3`。脚本会拒绝 origin 不一致、脏工作树和未处于锁定 commit 的 checkout。`upstreams/` 是只读依赖;必要兼容修复只能放入 `patches/<source>/`,不能直接改上游目录。
|
| 16 |
+
|
| 17 |
+
## 公开可用性
|
| 18 |
+
|
| 19 |
+
注册表 `configs/experiments.yaml` 是唯一的 coverage 来源:
|
| 20 |
+
|
| 21 |
+
- `supported` cell 才会进入 generation matrix。
|
| 22 |
+
- `langflow/lm1b` 和 `langflow/owt` 使用官方 Hugging Face checkpoint,并进入 many-step generation matrix。
|
| 23 |
+
- `rdlm/owt` unsupported:官方 RDLM release 没有 OWT model。
|
| 24 |
+
|
| 25 |
+
unsupported 不是失败,也不能使用另一个数据集或另一个 teacher 替代。矩阵生成后再次确认:
|
| 26 |
+
|
| 27 |
+
```bash
|
| 28 |
+
python -m dlb.matrix --root "$DLB_ROOT" \
|
| 29 |
+
--output "$DLB_ROOT/results/matrix/generation.tsv"
|
| 30 |
+
cat "$DLB_ROOT/results/matrix/unsupported.tsv"
|
| 31 |
+
```
|
| 32 |
+
|
| 33 |
+
## 不在本地做的事
|
| 34 |
+
|
| 35 |
+
本地 worktree 不运行 `fetch_sources.sh`、`fetch_data.py`、`fetch_checkpoints.py` 的真实下载,不创建 GPU 环境,不训练,不生成 1,024 样本。提交 GitHub 后,在服务器 clone 同一 commit 执行上述命令。
|
docs/02_datasets.md
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 02. 数据下载与预处理
|
| 2 |
+
|
| 3 |
+
## 固定数据契约
|
| 4 |
+
|
| 5 |
+
| 数据集 | Hub revision | tokenizer | 长度 | split |
|
| 6 |
+
|---|---|---|---:|---|
|
| 7 |
+
| LM1B | `billion-word-benchmark/lm1b@35161838ea9e05371a25a8db037f94fcae4c2064` | `bert-base-uncased` | 128 | `train` 与 `test` |
|
| 8 |
+
| OWT | `Skylion007/openwebtext@79d93d786212f7344586290adb811d4ae6a1762c` | `gpt2` | 1024 | 最后 100,000 篇为 validation |
|
| 9 |
+
|
| 10 |
+
LM1B 的 parquet revision、tokenizer revision、词表大小和文件校验均在 `artifacts/data.yaml`。OWT 约 8,013,769 篇文档,开始前需要至少 55 GiB 可用空间;只有明确使用 `--allow-low-disk` 才会绕过预检,并在 manifest 中留下证据。
|
| 11 |
+
|
| 12 |
+
## 下载、预处理和校验
|
| 13 |
+
|
| 14 |
+
以下命令只在服务器执行:
|
| 15 |
+
|
| 16 |
+
```bash
|
| 17 |
+
python scripts/fetch_data.py --root "$DLB_ROOT"
|
| 18 |
+
python scripts/preprocess_data.py --root "$DLB_ROOT" --dataset all
|
| 19 |
+
python scripts/verify_data.py --root "$DLB_ROOT" --dataset all
|
| 20 |
+
```
|
| 21 |
+
|
| 22 |
+
下载使用固定 Hub revision 和服务器本地 `data/raw/huggingface/` cache,可中断后重跑。预处理结果位于:
|
| 23 |
+
|
| 24 |
+
```text
|
| 25 |
+
data/processed/lm1b-bert-128/
|
| 26 |
+
data/processed/owt-gpt2-1024/
|
| 27 |
+
data/manifests/downloads.json
|
| 28 |
+
data/manifests/lm1b.json
|
| 29 |
+
data/manifests/owt.json
|
| 30 |
+
```
|
| 31 |
+
|
| 32 |
+
预处理会添加对应 tokenizer 的 BOS/EOS 语义并保证每个 packed sequence 长度严格一致。不要手动改 split、长度、tokenizer 或 OWT validation 边界;否则训练 recipe 和指标不再可比。
|
| 33 |
+
|
| 34 |
+
## 失败恢复
|
| 35 |
+
|
| 36 |
+
若 `verify_data.py` 报错,保留 `data/manifests/` 和日志,先检查磁盘、Hub cache 和 revision,再重新执行对应阶段。不要把不完整的 `data/processed/*` 改名为完成目录。
|
docs/06_generation_and_evaluation.md
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 06. 生成、评测与 timing
|
| 2 |
+
|
| 3 |
+
## 生成矩阵
|
| 4 |
+
|
| 5 |
+
先生成并检查 TSV:
|
| 6 |
+
|
| 7 |
+
```bash
|
| 8 |
+
python -m dlb.matrix --root "$DLB_ROOT" \
|
| 9 |
+
--output "$DLB_ROOT/results/matrix/generation.tsv" \
|
| 10 |
+
--sample-count 1024 --seed 42
|
| 11 |
+
python -m dlb.matrix --root "$DLB_ROOT" \
|
| 12 |
+
--output "$DLB_ROOT/results/matrix/generation.tsv" --validate
|
| 13 |
+
```
|
| 14 |
+
|
| 15 |
+
完整生成:
|
| 16 |
+
|
| 17 |
+
```bash
|
| 18 |
+
bash scripts/run_all.sh --root "$DLB_ROOT"
|
| 19 |
+
```
|
| 20 |
+
|
| 21 |
+
它会逐行继续执行,即使一个 cell 失败也会记录 `results/logs/generation_failures.tsv`。每个成功目录包含 `samples.jsonl`、`run_metadata.json`、`request.json` 和 stdout/stderr;runner 只有在样本 schema 和数量均通过后才写成功标记。
|
| 22 |
+
|
| 23 |
+
单个 cell:
|
| 24 |
+
|
| 25 |
+
```bash
|
| 26 |
+
bash scripts/run_one.sh --model mdlm --dataset owt --steps 32 \
|
| 27 |
+
--num-samples 1024 --seed 42
|
| 28 |
+
```
|
| 29 |
+
|
| 30 |
+
smoke:
|
| 31 |
+
|
| 32 |
+
```bash
|
| 33 |
+
bash scripts/smoke_all.sh
|
| 34 |
+
```
|
| 35 |
+
|
| 36 |
+
smoke 输出在 `results/smoke/`,sample_count=1 的 identity 不会被正式聚合接受。
|
| 37 |
+
|
| 38 |
+
## 三项质量指标
|
| 39 |
+
|
| 40 |
+
```bash
|
| 41 |
+
bash scripts/evaluate_all.sh --root "$DLB_ROOT"
|
| 42 |
+
```
|
| 43 |
+
|
| 44 |
+
评测环境必须能读取固定 revision 的 `gpt2-large`。正式评测不允许 `--allow-partial` 或自定义 special-id;LM1B 只排除约定的 padding,保留 BOS/EOS 语义。输出为每个 cell 的 `results/metrics/.../metrics.json`,包含样本 SHA、sample_count、PPL、unigram entropy 和 Self-BLEU。
|
| 45 |
+
|
| 46 |
+
## 独立 latency
|
| 47 |
+
|
| 48 |
+
主 latency 协议固定为 batch size 1、5 warm-up、32 repetitions、CUDA synchronize;加载、首次编译、decode、指标计算和落盘不计入。单个 cell:
|
| 49 |
+
|
| 50 |
+
```bash
|
| 51 |
+
bash scripts/benchmark_one.sh --model flm --dataset lm1b --steps 32 \
|
| 52 |
+
--seed 42 --precision author
|
| 53 |
+
```
|
| 54 |
+
|
| 55 |
+
批量执行:
|
| 56 |
+
|
| 57 |
+
```bash
|
| 58 |
+
bash scripts/benchmark_all.sh --root "$DLB_ROOT"
|
| 59 |
+
```
|
| 60 |
+
|
| 61 |
+
timing 输出在 `results/timing/.../timing.json`,`seconds_per_sample` 是主汇总字段;不要使用样本里的 `generation_seconds` 替代它。
|
| 62 |
+
|
| 63 |
+
## 严格汇总
|
| 64 |
+
|
| 65 |
+
```bash
|
| 66 |
+
python scripts/aggregate_results.py --root "$DLB_ROOT"
|
| 67 |
+
```
|
| 68 |
+
|
| 69 |
+
严格模式要求全部 132 个 supported task 都有 1,024 样本、三项质量指标、独立 timing 和匹配 provenance,输出:
|
| 70 |
+
|
| 71 |
+
```text
|
| 72 |
+
results/summary/results_long.csv
|
| 73 |
+
results/summary/results_wide.csv
|
| 74 |
+
results/summary/failures.csv
|
| 75 |
+
results/summary/unsupported.csv
|
| 76 |
+
results/summary/README.md
|
| 77 |
+
```
|
| 78 |
+
|
| 79 |
+
未完成时用 `--partial` 诊断;partial 报告必须标记 `complete=false`,不能作为最终论文表。
|
docs/07_slurm.md
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 07. Slurm
|
| 2 |
+
|
| 3 |
+
Slurm 模板不硬编码 partition、GPU 型号、walltime 或内存。请根据服务器站点在提交命令上补充这些资源参数;模板通过注册表 TSV 的 `environment` 字段选择 Conda 环境。
|
| 4 |
+
|
| 5 |
+
## 生成 array
|
| 6 |
+
|
| 7 |
+
先在登录节点创建矩阵,再提交 132 个 array task:
|
| 8 |
+
|
| 9 |
+
```bash
|
| 10 |
+
python -m dlb.matrix --root "$DLB_ROOT" \
|
| 11 |
+
--output "$DLB_ROOT/results/matrix/generation.tsv"
|
| 12 |
+
sbatch --array=0-131%8 slurm/generate_array.sbatch
|
| 13 |
+
```
|
| 14 |
+
|
| 15 |
+
## 质量评测和 timing
|
| 16 |
+
|
| 17 |
+
```bash
|
| 18 |
+
sbatch slurm/evaluate.sbatch
|
| 19 |
+
sbatch --array=0-131%8 slurm/benchmark_array.sbatch
|
| 20 |
+
```
|
| 21 |
+
|
| 22 |
+
`evaluate.sbatch` 使用 `DLB_EVAL_ENV`(默认 `dlb-eval`);array 模板读取每一行的模型、数据集、steps 和 registry environment,不根据 task id 猜环境。
|
| 23 |
+
|
| 24 |
+
## 训练/蒸馏
|
| 25 |
+
|
| 26 |
+
通过环境变量指定 recipe:
|
| 27 |
+
|
| 28 |
+
```bash
|
| 29 |
+
DLB_RECIPE=candi DLB_DATASET=owt \
|
| 30 |
+
sbatch --gres=gpu:1 slurm/train.sbatch
|
| 31 |
+
DLB_RECIPE=di4c DLB_MODEL=duo_di4c DLB_DATASET=lm1b \
|
| 32 |
+
sbatch --gres=gpu:1 slurm/train.sbatch
|
| 33 |
+
```
|
| 34 |
+
|
| 35 |
+
如站点需要 partition、constraint、时间或内存,请加到 `sbatch` 命令,而不是修改模板中的固定值。大训练任务的 devices/nodes 通过 `DLB_TRAIN_DEVICES`、`DLB_TRAIN_NODES` 或 recipe 命令参数设置,必须保持锁定 global batch。
|
| 36 |
+
|
| 37 |
+
## 日志与重试
|
| 38 |
+
|
| 39 |
+
Slurm 输出在 `results/logs/slurm-*.out`/`.err`;单个 array task 失败可只重提对应 index。不要删除成功样本或 timing 后盲目重跑,先核对 `run_metadata.json`、`checkpoint_lock.json` 和失败日志。
|
docs/09_four_gpu_local.md
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 09. 4-GPU 本地并行运行
|
| 2 |
+
|
| 3 |
+
本路径适用于单机服务器直接用 Bash/Conda 跑实验,不依赖 Slurm。默认只使用
|
| 4 |
+
GPU `0,1,2,3`,并且最多同时运行 4 个 matrix task。
|
| 5 |
+
|
| 6 |
+
## 默认资源策略
|
| 7 |
+
|
| 8 |
+
- 生成、smoke 和 benchmark:每个 task 绑定 1 张 GPU。
|
| 9 |
+
- 评测:每个 task 绑定 1 张 GPU,GPT-2 Large PPL 在该 GPU 上计算。
|
| 10 |
+
- 绑定方式:对子进程设置 `CUDA_VISIBLE_DEVICES=<gpu_id>`。子进程内看到的设备通常是 `cuda:0`。
|
| 11 |
+
- 默认并发:`DLB_MAX_JOBS=4`。
|
| 12 |
+
|
| 13 |
+
可以覆盖 GPU 列表或并发数:
|
| 14 |
+
|
| 15 |
+
```bash
|
| 16 |
+
DLB_GPUS=0,1,2,3 DLB_MAX_JOBS=4 bash scripts/run_4gpu.sh
|
| 17 |
+
DLB_GPUS=0,1 DLB_MAX_JOBS=2 bash scripts/run_4gpu.sh
|
| 18 |
+
```
|
| 19 |
+
|
| 20 |
+
## 推荐顺序
|
| 21 |
+
|
| 22 |
+
先完成 README 中的源码、数据、checkpoint 和 Conda 环境准备。需要
|
| 23 |
+
`reference_reproduction` 的 checkpoint 时,先按 `docs/05_training.md` 训练或蒸馏
|
| 24 |
+
对应产物。
|
| 25 |
+
|
| 26 |
+
然后执行:
|
| 27 |
+
|
| 28 |
+
```bash
|
| 29 |
+
export DLB_GPUS=0,1,2,3
|
| 30 |
+
export DLB_MAX_JOBS=4
|
| 31 |
+
|
| 32 |
+
bash scripts/smoke_4gpu.sh
|
| 33 |
+
bash scripts/run_4gpu.sh
|
| 34 |
+
bash scripts/evaluate_4gpu.sh
|
| 35 |
+
bash scripts/benchmark_4gpu.sh
|
| 36 |
+
"$DLB_PYTHON" scripts/aggregate_results.py --root "$DLB_ROOT"
|
| 37 |
+
```
|
| 38 |
+
|
| 39 |
+
## 输出
|
| 40 |
+
|
| 41 |
+
- smoke 样本:`results/smoke/samples/...`
|
| 42 |
+
- 正式样本:`results/samples/<dataset>/<model>/steps_<steps>/samples.jsonl`
|
| 43 |
+
- 指标:`results/metrics/<dataset>/<model>/steps_<steps>/metrics.json`
|
| 44 |
+
- timing:`results/timing/<dataset>/<model>/steps_<steps>/timing.json`
|
| 45 |
+
- 汇总表:`results/summary/results_wide.csv` 和 `results/summary/results_long.csv`
|
| 46 |
+
|
| 47 |
+
每个 task 的 stdout/stderr 会写入:
|
| 48 |
+
|
| 49 |
+
```text
|
| 50 |
+
results/logs/4gpu-<stage>-<task_id>.out
|
| 51 |
+
results/logs/4gpu-<stage>-<task_id>.err
|
| 52 |
+
```
|
| 53 |
+
|
| 54 |
+
失败清单按阶段写入:
|
| 55 |
+
|
| 56 |
+
```text
|
| 57 |
+
results/logs/smoke_4gpu_failures.tsv
|
| 58 |
+
results/logs/generation_4gpu_failures.tsv
|
| 59 |
+
results/logs/evaluation_4gpu_failures.tsv
|
| 60 |
+
results/logs/benchmark_4gpu_failures.tsv
|
| 61 |
+
```
|
| 62 |
+
|
| 63 |
+
如果失败清单不存在,表示该阶段没有失败 task。
|
| 64 |
+
|
| 65 |
+
## Dry Run
|
| 66 |
+
|
| 67 |
+
在真正占用 GPU 前可以检查调度计划:
|
| 68 |
+
|
| 69 |
+
```bash
|
| 70 |
+
bash scripts/run_4gpu.sh --dry-run
|
| 71 |
+
bash scripts/evaluate_4gpu.sh --dry-run
|
| 72 |
+
bash scripts/benchmark_4gpu.sh --dry-run
|
| 73 |
+
```
|
| 74 |
+
|
| 75 |
+
dry-run 会输出每个 task 绑定的 GPU 和实际 argv,不启动模型。
|
docs/12_elf_condition_recovery.md
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ELF 正式条件实验中断:诊断和继续运行
|
| 2 |
+
|
| 3 |
+
**本页为故障诊断期间的历史步骤,现由 [文档 13:已确认的 EOS 修复与恢复](13_elf_eos_fix.md) 替代。** 用户提供的 audit 已证明唯一配对样本的作者条件等于 raw tokens 加 EOS。现在采用修复后的原始输入流程,不必再按本页切换作者全集;已有作者结果保留。
|
| 4 |
+
|
| 5 |
+
2026-09-12 最新日志显示:WMT14 连续完成了若干个 3000 样本格子,BLEU 为 13.4014、14.0021、14.1667;随后 XSum 在 `216/11332 samples` 后报 `ValueError: Empty condition`。这份恢复步骤优先于文档 11 中使用原始 `validation/test` 的条件实验命令。
|
| 6 |
+
|
| 7 |
+
**已确认的问题与尚未确认的原因**
|
| 8 |
+
|
| 9 |
+
- XSum 中断发生在下一批(零基索引 216–223)准备输入时:至少一行得到空的条件 token 列表。仅凭日志不能确定确切行号或是空字符串、空白字符还是分词后为空;新诊断会列出全部空条件 ID。
|
| 10 |
+
- `1069 > 512` 是 T5 tokenizer 的长度提示,不是这次异常。当前 ELF XSum 配置将 source 截到 1024,总 canvas 为 1088。不要把 source 上限改成 512 来消除提示,那会改变实验。新预检查使用 `verbose=False` 做完整分词,再沿用原有模型截断上限。
|
| 11 |
+
- WMT14 的 BLEU≈13–14 明显低于之前 official-validation sanity 的 25.79,需调查原始文本与作者条件 IDs 的差异。此次日志没有格子的步数标签;按此前循环顺序,最后完成的是 64 步,但应以 `request.json` 为准。不能把该降幅直接归为正常采样误差,也不能凭日志断定唯一原因是 EOS。
|
| 12 |
+
- 旧 `prepare` 对官方输入使用 `condition_input_ids`,对原始输入使用 `add_special_tokens=False` 重新分词。这与锁定版本的[官方 JSONL loader](https://github.com/lillian039/ELF/blob/b29d8833609e9ab7f67cd9da39435ac5cea04837/src/utils/data_utils.py#L121-L137)一致,但不能据此证明与发布的 Arrow tokens 等价。作者预处理 validation 才是[发布配置默认使用的数据](https://github.com/lillian039/ELF/blob/b29d8833609e9ab7f67cd9da39435ac5cea04837/README.md)。
|
| 13 |
+
|
| 14 |
+
此前从 sanity 直接切换原始 validation 全集的安排,缺少逐样本输入一致性核验。这次先补核验,并用已下载的作者 validation 全集建立 ELF 基线。原始 validation/test 暂停,不删空样本、不添加虚构 source、不猜测性插入 EOS。已有低分格子保留为待调查的原始输入协议结果;作者数据结果单独存放,不混成一条曲线。
|
| 15 |
+
|
| 16 |
+
**1.更新代码、恢复单张物理 GPU 2 环境**
|
| 17 |
+
|
| 18 |
+
新版本在加载 ELF 权重前扫描本次全部条件输入,空条件会报告数量和行号并提前退出,避免再生成几百条才失败。有效输入的 tokens、截断、采样器和指标没有改变。OWT 已成功完成的质量结果无需重跑。
|
| 19 |
+
|
| 20 |
+
```bash
|
| 21 |
+
cd ~/diffusion_baseline
|
| 22 |
+
git pull --ff-only origin main
|
| 23 |
+
conda activate dlb-elf
|
| 24 |
+
export DLB_ROOT="$PWD"
|
| 25 |
+
export ELF_PYTHON="$CONDA_PREFIX/bin/python"
|
| 26 |
+
export PYTHONDONTWRITEBYTECODE=1
|
| 27 |
+
export ELF_PHYSICAL_GPU=2
|
| 28 |
+
export ELF_GPU_UUID="$(nvidia-smi -i "$ELF_PHYSICAL_GPU" --query-gpu=uuid --format=csv,noheader)"
|
| 29 |
+
export CUDA_VISIBLE_DEVICES="$ELF_GPU_UUID"
|
| 30 |
+
export ELF_RESULTS="$DLB_ROOT/results/elf-h200-gpu2-official-v1"
|
| 31 |
+
export ELF_BATCH_SIZE=8 ELF_EVAL_BATCH_SIZE=8 ELF_SEED=42 ELF_COMPILE=0
|
| 32 |
+
unset ELF_STEPS ELF_COUNT ELF_SPLIT ELF_TIMING_PROMPT
|
| 33 |
+
mkdir -p "$ELF_RESULTS/logs" "$ELF_RESULTS/environment"
|
| 34 |
+
set -o pipefail
|
| 35 |
+
runlog() {
|
| 36 |
+
local label="$1"
|
| 37 |
+
shift
|
| 38 |
+
"$@" 2>&1 | tee "$ELF_RESULTS/logs/$label.log"
|
| 39 |
+
}
|
| 40 |
+
nvidia-smi -i "$ELF_GPU_UUID" > "$ELF_RESULTS/environment/gpu-before.txt"
|
| 41 |
+
```
|
| 42 |
+
|
| 43 |
+
失败的原始 XSum 目录包含 `samples.jsonl.tmp`,不是完整结果,也不能直接从第 217 条拼接续跑:脚本没有保存该位置的完整 RNG 状态。保留该目录,本次使用新的根目录和 `official-validation` split;不要移动或删除旧结果来伪装恢复。
|
| 44 |
+
|
| 45 |
+
**2.CPU 诊断,并核对之前各格子的任务与步数**
|
| 46 |
+
|
| 47 |
+
不下载数据、不加载模型权重、不占 GPU 推理。读取服务器现有 JSONL、manifest 和本地 T5 tokenizer:
|
| 48 |
+
|
| 49 |
+
```bash
|
| 50 |
+
runlog input_audit python scripts/audit_elf_inputs.py \
|
| 51 |
+
--output "$ELF_RESULTS/input-audit.json"
|
| 52 |
+
|
| 53 |
+
python scripts/summarize_elf.py \
|
| 54 |
+
--results "$DLB_ROOT/results/elf-h200-gpu2" \
|
| 55 |
+
--output "$DLB_ROOT/results/elf-h200-gpu2/summary-before-recovery.csv"
|
| 56 |
+
|
| 57 |
+
python - <<'PY'
|
| 58 |
+
import json, os
|
| 59 |
+
from pathlib import Path
|
| 60 |
+
root = Path(os.environ['DLB_ROOT']) / 'results/elf-h200-gpu2/quality'
|
| 61 |
+
for path in sorted(root.rglob('request.json')):
|
| 62 |
+
request = json.loads(path.read_text())
|
| 63 |
+
metrics = path.with_name('metrics.json')
|
| 64 |
+
m = json.loads(metrics.read_text()) if metrics.exists() else {}
|
| 65 |
+
print(request['task'], (request.get('input') or {}).get('split', 'unconditional'),
|
| 66 |
+
'steps=', request['steps'], 'valid=', m.get('valid'),
|
| 67 |
+
'BLEU=', m.get('metrics', {}).get('bleu'), str(path.parent))
|
| 68 |
+
PY
|
| 69 |
+
```
|
| 70 |
+
|
| 71 |
+
报告重点:`raw.empty_condition_ids`、`official.empty_condition_ids`、按 source/reference 文本配对后的 `exact_ids_after_source_cap`、`raw_plus_eos_equals_official`、`mismatch_examples`。匹配按完整文本对进行,不假设两个数据集同一行号就是同一样本;无法匹配或重复匹配的行也会计数。EOS 对比只是诊断,不会修改 token。将 `logs/input_audit.log` 提供给我,用它确定原始 validation/test 的后续预处理修复。
|
| 72 |
+
|
| 73 |
+
**3.先跑作者 validation 全集的 64 步标准点**
|
| 74 |
+
|
| 75 |
+
先检查诊断成功且作者输入全部非空;通过后运行 WMT14 3000 条、XSum 11332 条。之前 sanity 是独立目录的 1000 条,这里是全集,不会覆盖它。
|
| 76 |
+
|
| 77 |
+
```bash
|
| 78 |
+
(
|
| 79 |
+
set -e
|
| 80 |
+
python - <<'PY'
|
| 81 |
+
import json, os
|
| 82 |
+
from pathlib import Path
|
| 83 |
+
r = json.loads((Path(os.environ['ELF_RESULTS']) / 'input-audit.json').read_text())
|
| 84 |
+
for task in ('wmt14', 'xsum'):
|
| 85 |
+
assert r['tasks'][task]['official_conditions_nonempty'], (
|
| 86 |
+
task, r['tasks'][task]['official']['empty_condition_ids'])
|
| 87 |
+
print('Official validation conditions are nonempty; generation will recheck input hashes.')
|
| 88 |
+
PY
|
| 89 |
+
for task in wmt14 xsum; do
|
| 90 |
+
runlog "${task}_official_s64_generate" \
|
| 91 |
+
env ELF_COUNT=0 ELF_STEPS=64 ELF_SPLIT=official-validation \
|
| 92 |
+
bash scripts/run_elf_suite.sh generate "$task"
|
| 93 |
+
runlog "${task}_official_s64_evaluate" \
|
| 94 |
+
env ELF_STEPS=64 ELF_SPLIT=official-validation \
|
| 95 |
+
bash scripts/run_elf_suite.sh evaluate "$task"
|
| 96 |
+
done
|
| 97 |
+
)
|
| 98 |
+
```
|
| 99 |
+
|
| 100 |
+
先看 64 步的 BLEU/ROUGE 是否回到 sanity 的量级。全集与前 1000 条分数可以不同,不设置为了“通过”而调参的分数门槛;如果仍明显偏低,先提供这两个指标与 audit 日志,暂不跑第 4 步的全量扫描。若 preflight 仍报错,保留错误信息,不绕过它。这里的结果应标为 **official-validation 全集**,不能称 test 结果。
|
| 101 |
+
|
| 102 |
+
**4.标准点合理后,补 8/16/32 步质量**
|
| 103 |
+
|
| 104 |
+
```bash
|
| 105 |
+
(
|
| 106 |
+
set -e
|
| 107 |
+
for task in wmt14 xsum; do
|
| 108 |
+
for steps in 8 16 32; do
|
| 109 |
+
runlog "${task}_official_s${steps}_generate" \
|
| 110 |
+
env ELF_COUNT=0 ELF_STEPS="$steps" ELF_SPLIT=official-validation \
|
| 111 |
+
bash scripts/run_elf_suite.sh generate "$task"
|
| 112 |
+
runlog "${task}_official_s${steps}_evaluate" \
|
| 113 |
+
env ELF_STEPS="$steps" ELF_SPLIT=official-validation \
|
| 114 |
+
bash scripts/run_elf_suite.sh evaluate "$task"
|
| 115 |
+
done
|
| 116 |
+
done
|
| 117 |
+
)
|
| 118 |
+
```
|
| 119 |
+
|
| 120 |
+
**5.同一张 H200 独占时测 timing**
|
| 121 |
+
|
| 122 |
+
不要与 generation/evaluate 并行;确认 GPU 2 没有其他计算任务。条件 timing 与本次质量统一使用 `official-validation`,固定 batch=1,5 次预热、32 次测量,第 0 条输入。OWT 不受这次条件输入问题影响。
|
| 123 |
+
|
| 124 |
+
```bash
|
| 125 |
+
nvidia-smi -i "$ELF_GPU_UUID"
|
| 126 |
+
(
|
| 127 |
+
set -e
|
| 128 |
+
nvidia-smi -i "$ELF_GPU_UUID" > "$ELF_RESULTS/environment/gpu-before-timing.txt"
|
| 129 |
+
runlog owt_core_timing env ELF_STEPS="8 16 32 64" \
|
| 130 |
+
bash scripts/run_elf_suite.sh timing owt
|
| 131 |
+
for task in wmt14 xsum; do
|
| 132 |
+
runlog "${task}_official_core_timing" \
|
| 133 |
+
env ELF_STEPS="8 16 32 64" ELF_SPLIT=official-validation \
|
| 134 |
+
bash scripts/run_elf_suite.sh timing "$task"
|
| 135 |
+
done
|
| 136 |
+
nvidia-smi -i "$ELF_GPU_UUID" > "$ELF_RESULTS/environment/gpu-after-timing.txt"
|
| 137 |
+
)
|
| 138 |
+
python scripts/summarize_elf.py \
|
| 139 |
+
--results "$ELF_RESULTS" --output "$ELF_RESULTS/summary.csv"
|
| 140 |
+
```
|
| 141 |
+
|
| 142 |
+
OWT 质量仍在旧根目录,以上 OWT timing 在新根目录;汇总时按任务/步数/seed/配置匹配,保留两边 provenance。条件任务看 `results.encoder_plus_sampler.seconds_per_sample`(含 T5 编码)及 `results.sampler_cached_condition.seconds_per_sample`(缓存编码);OWT 看 `results.sampler.seconds_per_sample`。这些不含模型加载、CPU 分词、指标计算和 IO;第 0 条重复测量不代表全数据集平均服务延迟。
|
| 143 |
+
|
| 144 |
+
**6.后续实验**
|
| 145 |
+
|
| 146 |
+
OWT 前缀实验仍可按文档 11 的 F 准备和运行,不受 WMT14/XSum 输入问题影响。原始 validation/test 的正式恢复取决于本次 audit 的实证结果;不能将作者 validation 冒充 test,也不能删除空数据后继续声称原始全集。现在先取得可信的作者 validation 质量与 timing 基线,再完成原始集协议核验和 test。
|
docs/13_elf_eos_fix.md
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ELF 条件输入 EOS 修复与正式实验恢复
|
| 2 |
+
|
| 3 |
+
本页取代文档 12 中“原始 validation/test 暂停、先改跑作者全集”的临时安排。现有 JSONL、权重、环境无需重新准备;修复发生在 runner 的条件分词阶段,结果使用新目录 `results/elf-h200-gpu2-eos-v2`。
|
| 4 |
+
|
| 5 |
+
**诊断已经确认什么**
|
| 6 |
+
|
| 7 |
+
用户提交的服务器 audit 显示:WMT14 的 2992 条唯一文本对、XSum 的全部 11332 条,作者保存的条件 IDs 都严格等于 `raw_no_special_tokens + [EOS]`。EOS ID 为 1(`</s>`)。WMT14 剩余 8 条是重复文本对,旧诊断没有唯一配对;新诊断通过包含重复次数的完整记录比较覆盖它们,不会删除重复样本。
|
| 8 |
+
|
| 9 |
+
XSum 的 source ID **219、3834、4715、5677、8445** 是空白文本;旧路径返回 `[]`,作者路径是 `[1]`,所以原始实验在准备含第 219 行的 batch 时中断,而作者 sanity 能通过。EOS 属于输入编码格式,并非补写文章内容;这五条仍保留完整 reference 并参与评测。
|
| 10 |
+
|
| 11 |
+
WMT14 旧输入缺 EOS 的协议错误已确认,是质量下降的重要解释;是否足以恢复 BLEU,要以修正后实测为准,不能声称已经恢复 26 分。此前依据官方 JSONL loader 的无特殊 token 写法没有复现发布 Arrow 的编码,现根据实际发布数据修正。
|
| 12 |
+
|
| 13 |
+
**修复规则**
|
| 14 |
+
|
| 15 |
+
- WMT14/XSum 原始文本:完整分词后追加一个 EOS,再截到 source 上限 64/1024;不能先截断再强行补 EOS,也不能预留一个位置替换长文本的最后一个正文 token。
|
| 16 |
+
- 已有 `condition_input_ids`:原样使用,不再添加 EOS;已有 official-validation 生成结果不受影响。
|
| 17 |
+
- OWT unconditional 与 OWT 前缀续写:保留原协议。前缀续写不是已结束的 source 文本,不能套用这次 EOS 改动。
|
| 18 |
+
- `request.json`、`generation.json`、`timing.json` 记录 `condition_tokenization_policy=released_ids_or_raw_plus_eos_then_source_cap_v2`,汇总 CSV 也保留该字段。旧结果缺这个字段,不能仅按 task/steps 与新结果混用。
|
| 19 |
+
|
| 20 |
+
**1.同步代码并绑定物理 GPU 2**
|
| 21 |
+
|
| 22 |
+
最新服务器日志中的 `fatal: Cannot fast-forward to multiple branches.` 表明 pull 的整合阶段失败;随后继续使用旧 audit,虽然输出文件名叫 `input-audit-v2.json`,内部 schema 仍是 v1,触发了断言。由于实验块有 `set -e`,本次尚未启动 64 步生成。这不是 EOS 修复再次失效。仅凭日志不能确定为什么出现多个合并目标;先显式抓取和合并唯一的远端 main,不使用可能包含多个候选的 FETCH_HEAD,也不重置本地工作。
|
| 23 |
+
|
| 24 |
+
先单独执行同步块,看到 `ELF EOS code sync OK` 后再执行下面环境设置。Git pull 本身包含抓取和整合两个阶段,因此“显示抓取到 main”不代表工作区更新成功。[Git 文档](https://git-scm.com/docs/git-pull)
|
| 25 |
+
|
| 26 |
+
```bash
|
| 27 |
+
(
|
| 28 |
+
set -e
|
| 29 |
+
cd ~/diffusion_baseline
|
| 30 |
+
git status --short
|
| 31 |
+
test "$(git branch --show-current)" = main
|
| 32 |
+
git fetch --no-tags origin refs/heads/main:refs/remotes/origin/main
|
| 33 |
+
git merge --ff-only refs/remotes/origin/main
|
| 34 |
+
git merge-base --is-ancestor 1b253e6 HEAD
|
| 35 |
+
git diff --exit-code HEAD -- scripts/elf_common.py scripts/audit_elf_inputs.py scripts/run_elf.py
|
| 36 |
+
git log -1 --oneline
|
| 37 |
+
printf '%s\n' 'ELF EOS code sync OK'
|
| 38 |
+
)
|
| 39 |
+
```
|
| 40 |
+
|
| 41 |
+
若此块仍失败,停止并保留报错;不要继续 audit/generate。无须重新克隆、下载资源或删除结果。服务器有本地修改或分支分叉时,以上命令不会强行覆盖它们。
|
| 42 |
+
|
| 43 |
+
```bash
|
| 44 |
+
cd ~/diffusion_baseline
|
| 45 |
+
conda activate dlb-elf
|
| 46 |
+
export DLB_ROOT="$PWD"
|
| 47 |
+
export ELF_PYTHON="$CONDA_PREFIX/bin/python"
|
| 48 |
+
export PYTHONDONTWRITEBYTECODE=1
|
| 49 |
+
export ELF_PHYSICAL_GPU=2
|
| 50 |
+
export ELF_GPU_UUID="$(nvidia-smi -i "$ELF_PHYSICAL_GPU" --query-gpu=uuid --format=csv,noheader)"
|
| 51 |
+
export CUDA_VISIBLE_DEVICES="$ELF_GPU_UUID"
|
| 52 |
+
export ELF_RESULTS="$DLB_ROOT/results/elf-h200-gpu2-eos-v2"
|
| 53 |
+
export ELF_BATCH_SIZE=8 ELF_EVAL_BATCH_SIZE=8 ELF_SEED=42 ELF_COMPILE=0
|
| 54 |
+
unset ELF_STEPS ELF_COUNT ELF_SPLIT ELF_TIMING_PROMPT
|
| 55 |
+
mkdir -p "$ELF_RESULTS/logs" "$ELF_RESULTS/environment"
|
| 56 |
+
set -o pipefail
|
| 57 |
+
runlog() {
|
| 58 |
+
local label="$1"
|
| 59 |
+
shift
|
| 60 |
+
"$@" 2>&1 | tee "$ELF_RESULTS/logs/$label.log"
|
| 61 |
+
}
|
| 62 |
+
nvidia-smi -i "$ELF_GPU_UUID" > "$ELF_RESULTS/environment/gpu-before.txt"
|
| 63 |
+
```
|
| 64 |
+
|
| 65 |
+
**GPU 2 上的小进程是否影响实验**
|
| 66 |
+
|
| 67 |
+
2026-09-12 14:56 的快照显示总显存约 97 MiB / 143771 MiB、GPU 利用率 0%,有三个同伴的 Python CUDA 进程。当前显存压力很低,可以先运行质量生成与评测;共享运行主要可能拖慢完成时间,如果其他进程后来增加显存也可能导致 OOM。共享本身通常不会改变 BLEU/ROUGE/PPL 的计算口径,不用因此作废成功完成的质量结果。
|
| 68 |
+
|
| 69 |
+
显存大小不是算力占用指标;0% 也只对应采样窗口。正式 timing 需要与同伴约定 GPU 独占时段,其他任务可能间歇启动 kernel,影响延迟和抖动,目前无法从快照量化影响百分比。[NVIDIA-SMI 指标说明](https://docs.nvidia.com/deploy/nvidia-smi/index.html)
|
| 70 |
+
|
| 71 |
+
可在另一个终端观察 30 次每秒的进程利用率;`-` 代表指标不可用,不代表零占用。观察只能了解负载,不能保证随后的整个 timing 窗口独占:
|
| 72 |
+
|
| 73 |
+
```bash
|
| 74 |
+
nvidia-smi pmon -i 2 -s um -d 1 -c 30
|
| 75 |
+
```
|
| 76 |
+
|
| 77 |
+
**2.CPU 检查实际修复路径,再跑 64 步 validation 全集**
|
| 78 |
+
|
| 79 |
+
更新后的诊断会保留旧 `raw` 统计用于对照,所以其中仍显示 5 个空列表是预期的。真正判定修复的是新增的 `corrected_runner_comparison_including_duplicates`:`empty_condition_ids=[]`、`ready_for_raw_validation=true`。完整 token 比较和截断后比较会分别报告,并检查顺序是否一致。它调用 runner 共用的分词函数,不是仅对 EOS 做一个假设性比较。
|
| 80 |
+
|
| 81 |
+
以下整块先做 CPU 检查,通过后才加载 GPU 模型。两项分别生成全部 3000/11332 条;参考答案不参与条件编码。
|
| 82 |
+
|
| 83 |
+
本次旧 audit 在 schema 断言处就停止了,因此同步成功后可以直接重跑下块、沿用 `elf-h200-gpu2-eos-v2` 目录。诊断 JSON 会更新成 v2;无需清空整个结果目录。如果后来另外启动过生成,则仍需先检查相应格子是否已完成,避免重复运行。
|
| 84 |
+
|
| 85 |
+
```bash
|
| 86 |
+
(
|
| 87 |
+
set -e
|
| 88 |
+
runlog input_audit_v2 python scripts/audit_elf_inputs.py \
|
| 89 |
+
--output "$ELF_RESULTS/input-audit-v2.json"
|
| 90 |
+
python - <<'PY'
|
| 91 |
+
import json, os
|
| 92 |
+
from pathlib import Path
|
| 93 |
+
r = json.loads((Path(os.environ['ELF_RESULTS']) / 'input-audit-v2.json').read_text())
|
| 94 |
+
assert r['schema'] == 'elf-condition-audit-v2'
|
| 95 |
+
for task in ('wmt14', 'xsum'):
|
| 96 |
+
c = r['tasks'][task]['corrected_runner_comparison_including_duplicates']
|
| 97 |
+
print(task, c)
|
| 98 |
+
assert c['ready_for_raw_validation'], f'{task}: 修复后的条件与作者数据仍不一致,请保留报告'
|
| 99 |
+
print('条件输入核验通过,开始 64 步 validation 全集')
|
| 100 |
+
PY
|
| 101 |
+
for task in wmt14 xsum; do
|
| 102 |
+
runlog "${task}_val_s64_generate" \
|
| 103 |
+
env ELF_COUNT=0 ELF_STEPS=64 ELF_SPLIT=validation \
|
| 104 |
+
bash scripts/run_elf_suite.sh generate "$task"
|
| 105 |
+
runlog "${task}_val_s64_evaluate" \
|
| 106 |
+
env ELF_STEPS=64 ELF_SPLIT=validation \
|
| 107 |
+
bash scripts/run_elf_suite.sh evaluate "$task"
|
| 108 |
+
done
|
| 109 |
+
)
|
| 110 |
+
```
|
| 111 |
+
|
| 112 |
+
先看 WMT14 BLEU 和 XSum ROUGE 是否回到先前 sanity 的合理量级;全集与前 1000 条允许有抽样差异。如果仍明显异常,先提供这两项指标和 audit v2,不继续扩展扫描。无需再次跑 1000 条 sanity 或重复作者全集;如果作者全集已启动,先让它完成或正常停止,勿在同一 GPU 同时启动本节。
|
| 113 |
+
|
| 114 |
+
**3.64 步质量确认后,重跑原始 validation 的 8/16/32 步**
|
| 115 |
+
|
| 116 |
+
旧 WMT14 的 13–14 分结果使用了错误的 source 编码,不能只重算指标,必须重新生成;旧 XSum 的部分 `.tmp` 不能拼接进新结果。全部旧文件保留,不删除、不覆盖。
|
| 117 |
+
|
| 118 |
+
```bash
|
| 119 |
+
(
|
| 120 |
+
set -e
|
| 121 |
+
for task in wmt14 xsum; do
|
| 122 |
+
for steps in 8 16 32; do
|
| 123 |
+
runlog "${task}_val_s${steps}_generate" \
|
| 124 |
+
env ELF_COUNT=0 ELF_STEPS="$steps" ELF_SPLIT=validation \
|
| 125 |
+
bash scripts/run_elf_suite.sh generate "$task"
|
| 126 |
+
runlog "${task}_val_s${steps}_evaluate" \
|
| 127 |
+
env ELF_STEPS="$steps" ELF_SPLIT=validation \
|
| 128 |
+
bash scripts/run_elf_suite.sh evaluate "$task"
|
| 129 |
+
done
|
| 130 |
+
done
|
| 131 |
+
)
|
| 132 |
+
```
|
| 133 |
+
|
| 134 |
+
**4.单卡独占时测修复后的条件 timing**
|
| 135 |
+
|
| 136 |
+
固定 batch=1、5 次 warmup、32 次 repeats,默认第 0 条输入;不与生成/指标评测并行。旧原始输入的 timing 也使用了不同条件长度,不能替代修复后的 timing。
|
| 137 |
+
|
| 138 |
+
```bash
|
| 139 |
+
nvidia-smi -i "$ELF_GPU_UUID"
|
| 140 |
+
(
|
| 141 |
+
set -e
|
| 142 |
+
nvidia-smi -i "$ELF_GPU_UUID" > "$ELF_RESULTS/environment/gpu-before-timing.txt"
|
| 143 |
+
for task in wmt14 xsum; do
|
| 144 |
+
runlog "${task}_val_timing" \
|
| 145 |
+
env ELF_STEPS="8 16 32 64" ELF_SPLIT=validation \
|
| 146 |
+
bash scripts/run_elf_suite.sh timing "$task"
|
| 147 |
+
done
|
| 148 |
+
nvidia-smi -i "$ELF_GPU_UUID" > "$ELF_RESULTS/environment/gpu-after-timing.txt"
|
| 149 |
+
)
|
| 150 |
+
```
|
| 151 |
+
|
| 152 |
+
同时保留 `sampler_cached_condition` 和 `encoder_plus_sampler` 的 `seconds_per_sample`,后者包含 T5 条件编码;两者均不含加载、CPU tokenizer、IO 和指标计算。这是单输入重复测量,不是全集平均服务耗时。OWT 的质量/已有 timing 不受本次修复影响;若 OWT timing 尚未跑,可在独占时用本目录单独运行 `env ELF_STEPS="8 16 32 64" bash scripts/run_elf_suite.sh timing owt`,质量继续引用旧目录。
|
| 153 |
+
|
| 154 |
+
**5.恢复 test:先冻结标准 64 步,使用同一 EOS 规则**
|
| 155 |
+
|
| 156 |
+
validation 检查确认后可执行完整 WMT14 test 3003 条、XSum test 11334 条。此次 EOS 规则由作者 validation 实证确定,再应用于 test;没有作者 test token 文件用于逐行对照,因此报告“官方权重在公开 test 上按验证过的 source EOS 规则重评”,不宣称已确认论文全部 test 预处理相同。空 source 仍保留,完整 reference 不截断。
|
| 157 |
+
|
| 158 |
+
```bash
|
| 159 |
+
(
|
| 160 |
+
set -e
|
| 161 |
+
for task in wmt14 xsum; do
|
| 162 |
+
runlog "${task}_test_s64_generate" \
|
| 163 |
+
env ELF_COUNT=0 ELF_STEPS=64 ELF_SPLIT=test \
|
| 164 |
+
bash scripts/run_elf_suite.sh generate "$task"
|
| 165 |
+
runlog "${task}_test_s64_evaluate" \
|
| 166 |
+
env ELF_STEPS=64 ELF_SPLIT=test \
|
| 167 |
+
bash scripts/run_elf_suite.sh evaluate "$task"
|
| 168 |
+
# 此时同样要求 GPU 独占。
|
| 169 |
+
runlog "${task}_test_s64_timing" \
|
| 170 |
+
env ELF_STEPS=64 ELF_SPLIT=test \
|
| 171 |
+
bash scripts/run_elf_suite.sh timing "$task"
|
| 172 |
+
done
|
| 173 |
+
)
|
| 174 |
+
python scripts/summarize_elf.py \
|
| 175 |
+
--results "$ELF_RESULTS" --output "$ELF_RESULTS/summary.csv"
|
| 176 |
+
```
|
| 177 |
+
|
| 178 |
+
更少步数的 test 配置先根据 validation 冻结,再补测。OWT 前缀和扩展步数仍按文档 11 的 F/H2 实施;原始条件任务的相关命令使用本页的新结果根目录。任何已经成功的格子都不要原样重复 generate;若只缺指标则只 evaluate,若只缺 timing 则只 timing。
|
docs/19_elf_short_response_rescore.md
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ELF OWT条件4步、8步:短输出补评分
|
| 2 |
+
|
| 3 |
+
用户提供的结果:4步已生成2048条,其中2条不足64个GPT-2 tokens;8步已生成2048条,其中1条不足64个GPT-2 tokens。两格都有timing。这不是生成任务缺失或GPU报错,而是严格C64评分拒绝了短输出。仅凭CSV还不能判断是提前EOS还是T5/GPT-2重分词长度差异。
|
| 4 |
+
|
| 5 |
+
保持相同seed和生成配置,重复生成不保证解决短输出;改变seed、只替换短样本或反复抽到足长样本会引入筛选。因此这次复用原始2048条,**只重新运行评分,不训练、不重新生成、不重测timing**。
|
| 6 |
+
|
| 7 |
+
## 两种评分结果分开保存
|
| 8 |
+
|
| 9 |
+
- `strict`(原默认):每条response必须至少64 tokens。旧4步、8步仍为 `valid=false`,本次不覆盖。
|
| 10 |
+
- `observed`(新增可选):所有2048条都参与,长输出取前64 tokens,短输出取实际非空长度,不补padding、不丢样本。PPL仍使用GPT-2 Large并排除prompt loss,按总NLL/总实际有效response token数汇总;entropy是每条实际评分片段熵的均值。若有空response,仍不能产生完整主指标。
|
| 11 |
+
|
| 12 |
+
后一种结果必须标注为“response最多64个GPT-2 tokens,短输出按实际长度评分”。它并不是把原来严格64的失败变成成功。其他5个步数没有短输出,在两种长度政策下评分片段相同;如果合并展示,整张表统一标注最多64政策,并保留每格短输出计数及原结果来源。可用 `--short-response-policy observed` 不指定 `--steps` 为全部7格生成同一协议的新表。
|
| 13 |
+
|
| 14 |
+
## 服务器:只补4步和8步
|
| 15 |
+
|
| 16 |
+
一张空闲GPU足够,以下使用GPU0。与ELF1024步生成无关,只加载评分器。
|
| 17 |
+
|
| 18 |
+
```bash
|
| 19 |
+
cd ~/diffusion_baseline
|
| 20 |
+
git pull --ff-only
|
| 21 |
+
conda activate dlb-elf
|
| 22 |
+
export PYTHONDONTWRITEBYTECODE=1
|
| 23 |
+
mkdir -p results/elf-owt-baseline-logs
|
| 24 |
+
set -o pipefail
|
| 25 |
+
|
| 26 |
+
python scripts/evaluate_elf_owt_baseline.py \
|
| 27 |
+
--task owt-prefix --steps 4 8 \
|
| 28 |
+
--short-response-policy observed --plan
|
| 29 |
+
|
| 30 |
+
CUDA_VISIBLE_DEVICES=0 python -u scripts/evaluate_elf_owt_baseline.py \
|
| 31 |
+
--task owt-prefix --steps 4 8 \
|
| 32 |
+
--short-response-policy observed --batch-size 8 \
|
| 33 |
+
2>&1 | tee results/elf-owt-baseline-logs/owt-prefix-s4-s8-observed.log
|
| 34 |
+
```
|
| 35 |
+
|
| 36 |
+
`--plan`应只列出两格,不进行GPU推理。新结果默认写入独立的 `results/elf-owt-baseline-eval-owt-prefix-observed`,不需要设置ELF_RESULTS。中断后重复同一条评分命令即可,已核验的新指标会跳过。
|
| 37 |
+
|
| 38 |
+
```bash
|
| 39 |
+
cat results/elf-owt-baseline-eval-owt-prefix-observed/summary-steps-4-8.csv
|
| 40 |
+
```
|
| 41 |
+
|
| 42 |
+
预期:两格sample_count均2048;如所有短输出均非空且有有效评分目标,valid=True;short_count仍为2和1。short_count不会被“修成0”。若依然失败,保留日志,不更换seed或删样本。
|
| 43 |
+
|
| 44 |
+
## 可选:查看原短输出的真实长度与文本
|
| 45 |
+
|
| 46 |
+
下面只用CPU tokenizer,不加载GPU模型。读取旧指标定位样本,并校验保存的样本哈希。
|
| 47 |
+
|
| 48 |
+
```bash
|
| 49 |
+
python - <<'PY'
|
| 50 |
+
import json, sys
|
| 51 |
+
from pathlib import Path
|
| 52 |
+
sys.path.insert(0, 'scripts')
|
| 53 |
+
from elf_common import asset, sha256
|
| 54 |
+
from transformers import AutoTokenizer
|
| 55 |
+
root = Path.cwd()
|
| 56 |
+
tok = AutoTokenizer.from_pretrained(str(asset(root, 'gpt2')), local_files_only=True)
|
| 57 |
+
for steps in (4, 8):
|
| 58 |
+
mp = root / f'results/elf-owt-baseline-eval-owt-prefix/steps_{steps}/metrics.json'
|
| 59 |
+
meta = json.loads(mp.read_text())
|
| 60 |
+
samples = Path(meta['source_run']) / 'samples.jsonl'
|
| 61 |
+
assert sha256(samples) == meta['samples_sha256']
|
| 62 |
+
selected = set(meta['short_sample_ids'])
|
| 63 |
+
for line in samples.open():
|
| 64 |
+
row = json.loads(line)
|
| 65 |
+
if row['id'] in selected:
|
| 66 |
+
print(json.dumps({
|
| 67 |
+
'steps': steps, 'id': row['id'],
|
| 68 |
+
'prompt_id': row['prompt_id'], 'completion_id': row['completion_id'],
|
| 69 |
+
'gpt2_tokens': len(tok(row['generated'], add_special_tokens=False)['input_ids']),
|
| 70 |
+
'text': row['generated'],
|
| 71 |
+
}, ensure_ascii=False))
|
| 72 |
+
PY
|
| 73 |
+
```
|
| 74 |
+
|
| 75 |
+
由于旧生成文件已经裁剪EOS之后的内容,本次不能从现有文件恢复完整原生输出尾部;诊断只能说明已保存文本的情况。无需为补这两个评分数字再生成整批样本。
|
docs/EXPERIMENT_MATRIX.md
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 实验矩阵
|
| 2 |
+
|
| 3 |
+
## 任务展开
|
| 4 |
+
|
| 5 |
+
| 类别 | 逻辑方法 | 步数 | 数据集 cell | 任务数 |
|
| 6 |
+
|---|---|---|---:|---:|
|
| 7 |
+
| Many-step | FLM | 1,2,4,8,16,32,1024 | LM1B、OWT | 14 |
|
| 8 |
+
| Many-step | LangFlow | 1,2,4,8,16,32,1024 | LM1B、OWT | 14 |
|
| 9 |
+
| Many-step | Duo | 1,2,4,8,16,32,1024 | LM1B、OWT | 14 |
|
| 10 |
+
| Many-step | MDLM | 1,2,4,8,16,32,1024 | LM1B、OWT | 14 |
|
| 11 |
+
| Many-step | CANDI | 1,2,4,8,16,32,1024 | LM1B、OWT | 14 |
|
| 12 |
+
| Many-step | RDLM | 1000,1024 | LM1B | 2 |
|
| 13 |
+
| Few-step | FMLM | 1,2,4,8,16,32 | LM1B、OWT | 12 |
|
| 14 |
+
| Few-step | Duo+DCD | 1,2,4,8,16,32 | LM1B、OWT | 12 |
|
| 15 |
+
| Few-step | Duo+Di4C | 1,2,4,8,16,32 | LM1B、OWT | 12 |
|
| 16 |
+
| Few-step | MDLM+SDTT | 1,2,4,8,16,32 | LM1B、OWT | 12 |
|
| 17 |
+
| Few-step | MDLM+Di4C | 1,2,4,8,16,32 | LM1B、OWT | 12 |
|
| 18 |
+
| **合计** | | | | **132** |
|
| 19 |
+
|
| 20 |
+
RDLM/OWT 是注册表声明的 unsupported cell,只保留一条 reason,不展开为 steps。
|
| 21 |
+
RDLM/LM1B 也不展开 `1,2,4,8,16,32`:RDLM 原始 baseline 是官方 SDE sampler,默认约 1000 个数值积分步;少步网格不是论文方法,也不是训练过的蒸馏版本。服务器诊断显示这些低步 RDLM 行存在 retokenization、非规范上游 ID、低 entropy 与 PPL 非单调信号,因此主表只保留官方默认 `1000` 和与本项目旧网格对齐的 `1024`。
|
| 22 |
+
|
| 23 |
+
## 结果契约
|
| 24 |
+
|
| 25 |
+
每行任务对应:
|
| 26 |
+
|
| 27 |
+
```text
|
| 28 |
+
results/samples/<dataset>/<model>/steps_<N>/samples.jsonl
|
| 29 |
+
results/samples/<dataset>/<model>/steps_<N>/run_metadata.json
|
| 30 |
+
results/metrics/<dataset>/<model>/steps_<N>/metrics.json
|
| 31 |
+
results/timing/<dataset>/<model>/steps_<N>/timing.json
|
| 32 |
+
```
|
| 33 |
+
|
| 34 |
+
正式任务必须 `sample_count=1024`、`seed=42`,并通过 source/config/checkpoint/adapter/environment provenance 审计。质量指标使用 GPT-2 Large Generative PPL、平均 per-sample unigram entropy、Self-BLEU;延迟使用独立 timing 的 `seconds_per_sample`。
|
| 35 |
+
|
| 36 |
+
## 可执行矩阵文件
|
| 37 |
+
|
| 38 |
+
```bash
|
| 39 |
+
python -m dlb.matrix --root "$DLB_ROOT" \
|
| 40 |
+
--output "$DLB_ROOT/results/matrix/generation.tsv"
|
| 41 |
+
```
|
| 42 |
+
|
| 43 |
+
TSV 第一行是 `# schema=dlb-generation-matrix-v1`,第二行是固定字段名。解析使用 CSV/TSV reader,不使用 `eval`。不要手动重排或编辑任务行;如果 registry 改动,重新生成并审阅 unsupported inventory。
|
docs/superpowers/plans/2026-08-25-conditional-generation.md
ADDED
|
@@ -0,0 +1,1116 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Zero-Shot Conditional Generation Implementation Plan
|
| 2 |
+
|
| 3 |
+
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
| 4 |
+
|
| 5 |
+
**Goal:** Add a zero-training `c64_zs_v1` benchmark that evaluates every existing unconditional checkpoint with an immutable 64-token real prefix and a generated continuation.
|
| 6 |
+
|
| 7 |
+
**Architecture:** Extend the shared request and adapter boundaries with an explicit conditional mode while keeping unconditional identities and artifacts backward-compatible. Conditional prompts, samples, matrix, metrics, timing, aggregation, and scripts use separate versioned contracts and `results/conditional/` paths; native sampler projection APIs are used when available, with scoped adapter-level runtime clamps for other families.
|
| 8 |
+
|
| 9 |
+
**Tech Stack:** Python 3.11, Pydantic 2, PyYAML, Hugging Face Datasets/Transformers, PyTorch, MAUVE 0.3.0, pytest 8, Bash, existing pinned model-specific Conda environments.
|
| 10 |
+
|
| 11 |
+
**Spec:** `docs/superpowers/specs/2026-08-25-conditional-generation-design.md`
|
| 12 |
+
|
| 13 |
+
## Global Constraints
|
| 14 |
+
|
| 15 |
+
- Protocol identifier is exactly `c64_zs_v1`; configuration schema version is `1`.
|
| 16 |
+
- Reuse existing checkpoints only: no training, fine-tuning, checkpoint rewriting, or upstream repository edits.
|
| 17 |
+
- Use selection seed `42`, sampling seed `42`, exactly `1,024` prompts, prefix length `64`, and aligned evaluation-continuation length `64`.
|
| 18 |
+
- Produce completion `0` for prompt IDs `0..1023` and completions `1..4` for prompt IDs `0..255`, exactly `2,048` records per supported step task.
|
| 19 |
+
- LM1B uses the existing BERT 128-token processed validation split; OWT uses the existing GPT-2 1,024-token processed validation split.
|
| 20 |
+
- OWT generation retains the full 960-token suffix; aligned quality metrics use only its first 64 suffix tokens.
|
| 21 |
+
- The first 64 output tokens must equal the prompt prefix after every relevant sampler transition and in every published record; any mismatch invalidates the artifact.
|
| 22 |
+
- The conditional matrix contains the same 132 supported step tasks as the unconditional matrix, with RDLM/OWT retained in a separate unsupported inventory.
|
| 23 |
+
- Conditional outputs live below `results/conditional/`; unconditional result paths and successful-publication identity semantics remain unchanged.
|
| 24 |
+
- All model/tokenizer/checkpoint/prompt/sample artifacts are local-only and SHA-256 bound; no runtime network fallback is allowed.
|
| 25 |
+
- Artifact writes are atomic and fail closed; batch-size incompatibility, partial schedules, duplicate IDs, digest drift, and token-bound violations are errors.
|
| 26 |
+
- Preserve unrelated working-tree changes, including user-owned untracked diagnostics files.
|
| 27 |
+
|
| 28 |
+
## File and responsibility map
|
| 29 |
+
|
| 30 |
+
**Protocol and prompt artifacts**
|
| 31 |
+
|
| 32 |
+
- Create `configs/conditional.yaml`: canonical experiment constants.
|
| 33 |
+
- Create `src/dlb/conditional_prompts.py`: typed protocol/prompt contracts, deterministic selection, prompt build/verify logic.
|
| 34 |
+
- Create `scripts/build_conditional_prompts.py` and `scripts/verify_conditional_prompts.py`: thin CLIs.
|
| 35 |
+
|
| 36 |
+
**Sample and execution contracts**
|
| 37 |
+
|
| 38 |
+
- Modify `src/dlb/schema.py`: add `ConditionalSampleRecord` without changing `SampleRecord`.
|
| 39 |
+
- Modify `src/dlb/io.py`: conditional JSONL reader/validator/atomic writer.
|
| 40 |
+
- Modify `src/dlb/runner.py`: conditional request identity, validation, publication branch, and CLI flags.
|
| 41 |
+
- Create `src/dlb/conditional_matrix.py`: separate versioned 132-task matrix and unsupported inventory.
|
| 42 |
+
|
| 43 |
+
**Conditioning runtime**
|
| 44 |
+
|
| 45 |
+
- Create `src/dlb/adapters/conditional_runtime.py`: prompt batching, token/one-hot/embedding clamps, scoped monkey-patch restoration, native projection builders.
|
| 46 |
+
- Modify `src/dlb/adapters/base.py`, `src/dlb/adapters/capture.py`, and all files in `src/dlb/adapters/{flm,mdlm,duo,candi,langflow,rdlm,sdtt,di4c}.py`: serialize and consume conditional arguments.
|
| 47 |
+
- Modify `adapters/_distilled_runtime.py`, `adapters/sample_sdtt.py`, `adapters/sample_di4c.py`, and `adapters/sample_langflow.py`: server-backed prompt loading, projection, capture, and timing.
|
| 48 |
+
|
| 49 |
+
**Evaluation and reporting**
|
| 50 |
+
|
| 51 |
+
- Create `evaluation/conditional_perplexity.py`: suffix-only causal loss masking.
|
| 52 |
+
- Create `evaluation/mauve_score.py`: pinned local-only MAUVE wrapper.
|
| 53 |
+
- Create `evaluation/conditional_evaluate.py`: exact schedule validation and all conditional metrics.
|
| 54 |
+
- Modify `evaluation/self_bleu.py` and `evaluation/unigram_entropy.py`: token-row/group APIs reusable by both protocols.
|
| 55 |
+
- Create `src/dlb/conditional_benchmarking.py`: conditional latency command and provenance publication.
|
| 56 |
+
- Create `src/dlb/conditional_aggregate.py`: strict conditional result validation and summary publication.
|
| 57 |
+
|
| 58 |
+
**Launch and verification**
|
| 59 |
+
|
| 60 |
+
- Create conditional one/all/four-GPU generation, evaluation, timing, smoke, and aggregate scripts under `scripts/`.
|
| 61 |
+
- Modify `src/dlb/gpu_matrix.py`: accept an explicit conditional protocol and route to conditional scripts without altering default routing.
|
| 62 |
+
- Modify `README.md` and the relevant experiment documentation with server commands and zero-shot/OOD interpretation.
|
| 63 |
+
|
| 64 |
+
---
|
| 65 |
+
|
| 66 |
+
### Task 1: Canonical protocol and deterministic prompt artifacts
|
| 67 |
+
|
| 68 |
+
**Files:**
|
| 69 |
+
|
| 70 |
+
- Create: `configs/conditional.yaml`
|
| 71 |
+
- Create: `src/dlb/conditional_prompts.py`
|
| 72 |
+
- Create: `scripts/build_conditional_prompts.py`
|
| 73 |
+
- Create: `scripts/verify_conditional_prompts.py`
|
| 74 |
+
- Test: `tests/test_conditional_prompts.py`
|
| 75 |
+
|
| 76 |
+
**Interfaces:**
|
| 77 |
+
|
| 78 |
+
- Consumes: `artifacts/data.yaml`, `data/manifests/{lm1b,owt}.json`, and processed validation rows containing `input_ids`.
|
| 79 |
+
- Produces: `ConditionalProtocol`, `PromptRecord`, `PromptManifest`, `load_protocol(path)`, `select_source_indices(row_count, count, seed)`, `build_prompts(root, dataset_id, protocol)`, and `verify_prompts(root, dataset_id, protocol) -> PromptManifest`.
|
| 80 |
+
|
| 81 |
+
- [ ] **Step 1: Write failing protocol and selection tests**
|
| 82 |
+
|
| 83 |
+
```python
|
| 84 |
+
def test_protocol_has_exact_c64_production_contract(tmp_path):
|
| 85 |
+
path = tmp_path / "conditional.yaml"
|
| 86 |
+
path.write_text(PRODUCTION_CONFIG, encoding="utf-8")
|
| 87 |
+
protocol = load_protocol(path)
|
| 88 |
+
assert protocol.protocol == "c64_zs_v1"
|
| 89 |
+
assert (protocol.prompt_count, protocol.prefix_length) == (1024, 64)
|
| 90 |
+
assert protocol.datasets["owt"].model_length == 1024
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def test_selection_is_unique_stable_and_seed_bound():
|
| 94 |
+
first = select_source_indices(4096, 1024, 42)
|
| 95 |
+
assert first == select_source_indices(4096, 1024, 42)
|
| 96 |
+
assert first != select_source_indices(4096, 1024, 43)
|
| 97 |
+
assert len(first) == len(set(first)) == 1024
|
| 98 |
+
```
|
| 99 |
+
|
| 100 |
+
- [ ] **Step 2: Run tests and confirm missing-module failure**
|
| 101 |
+
|
| 102 |
+
Run: `pytest tests/test_conditional_prompts.py -q`
|
| 103 |
+
|
| 104 |
+
Expected: collection fails because `dlb.conditional_prompts` does not exist.
|
| 105 |
+
|
| 106 |
+
- [ ] **Step 3: Implement the typed protocol, SHA-256 ordering, and prompt records**
|
| 107 |
+
|
| 108 |
+
```python
|
| 109 |
+
class ConditionalProtocol(StrictModel):
|
| 110 |
+
schema_version: Literal[1]
|
| 111 |
+
protocol: Literal["c64_zs_v1"]
|
| 112 |
+
selection_seed: StrictInt
|
| 113 |
+
sampling_seed: StrictInt
|
| 114 |
+
prompt_count: Literal[1024]
|
| 115 |
+
prefix_length: Literal[64]
|
| 116 |
+
evaluation_continuation_length: Literal[64]
|
| 117 |
+
diversity_prompt_count: Literal[256]
|
| 118 |
+
completions_per_diversity_prompt: Literal[5]
|
| 119 |
+
datasets: dict[str, ConditionalDataset]
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def select_source_indices(row_count: int, count: int, seed: int) -> list[int]:
|
| 123 |
+
if row_count < count or count <= 0:
|
| 124 |
+
raise ValueError("processed validation split has too few rows")
|
| 125 |
+
return sorted(
|
| 126 |
+
range(row_count),
|
| 127 |
+
key=lambda index: hashlib.sha256(f"{seed}:{index}".encode()).digest(),
|
| 128 |
+
)[:count]
|
| 129 |
+
```
|
| 130 |
+
|
| 131 |
+
`build_prompts` must load `data/processed/<dataset-contract>/validation` with `datasets.load_from_disk`, preserve selected order as `prompt_id`, slice exactly `[0:64]` and `[64:128]`, validate against the dataset vocabulary size, write deterministic compact JSONL through the safe atomic writer, and publish a manifest that binds source manifest SHA, tokenizer ID/revision, selection algorithm, seed, counts, lengths, and prompt-file SHA.
|
| 132 |
+
|
| 133 |
+
- [ ] **Step 4: Add tamper, token-bound, and CLI tests, then make them pass**
|
| 134 |
+
|
| 135 |
+
```python
|
| 136 |
+
def test_verify_rejects_prompt_file_tampering(built_prompt_tree):
|
| 137 |
+
prompts = built_prompt_tree / "data/conditional/lm1b-c64/prompts.jsonl"
|
| 138 |
+
prompts.write_bytes(prompts.read_bytes() + b"\n")
|
| 139 |
+
with pytest.raises(ValueError, match="prompt file SHA-256"):
|
| 140 |
+
verify_prompts(built_prompt_tree, "lm1b", production_protocol())
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def test_prompt_cli_uses_processed_validation_without_repacking(monkeypatch, tmp_path):
|
| 144 |
+
observed = install_fake_load_from_disk(monkeypatch, rows=2048)
|
| 145 |
+
assert build_main(["--root", str(tmp_path), "--dataset", "lm1b"]) == 0
|
| 146 |
+
assert observed == [tmp_path / "data/processed/lm1b-bert-128/validation"]
|
| 147 |
+
```
|
| 148 |
+
|
| 149 |
+
Run: `pytest tests/test_conditional_prompts.py -q`
|
| 150 |
+
|
| 151 |
+
Expected: all tests pass.
|
| 152 |
+
|
| 153 |
+
- [ ] **Step 5: Commit protocol and prompt artifacts**
|
| 154 |
+
|
| 155 |
+
```bash
|
| 156 |
+
git add configs/conditional.yaml src/dlb/conditional_prompts.py \
|
| 157 |
+
scripts/build_conditional_prompts.py scripts/verify_conditional_prompts.py \
|
| 158 |
+
tests/test_conditional_prompts.py
|
| 159 |
+
git commit -m "feat: add deterministic conditional prompt manifests"
|
| 160 |
+
```
|
| 161 |
+
|
| 162 |
+
### Task 2: Conditional sample schema and atomic I/O
|
| 163 |
+
|
| 164 |
+
**Files:**
|
| 165 |
+
|
| 166 |
+
- Modify: `src/dlb/schema.py`
|
| 167 |
+
- Modify: `src/dlb/io.py`
|
| 168 |
+
- Create: `tests/test_conditional_schema.py`
|
| 169 |
+
- Create: `tests/test_conditional_io.py`
|
| 170 |
+
|
| 171 |
+
**Interfaces:**
|
| 172 |
+
|
| 173 |
+
- Consumes: `PromptRecord` and protocol lengths from Task 1.
|
| 174 |
+
- Produces: `ConditionalSampleRecord`, `expected_conditional_schedule(prompt_count=1024, diversity_prompt_count=256, completions=5) -> Sequence[tuple[int, int]]`, `read_conditional_samples(path)`, `validate_conditional_samples(path, expected=2048, schedule=production_schedule, sequence_length=model_length, vocab_size=dataset_vocab_size)`, and `write_conditional_samples_atomic(path, records, expected=2048, schedule=production_schedule, sequence_length=model_length, vocab_size=dataset_vocab_size)`.
|
| 175 |
+
|
| 176 |
+
- [ ] **Step 1: Write failing schema-invariant tests**
|
| 177 |
+
|
| 178 |
+
```python
|
| 179 |
+
def test_conditional_record_binds_prefix_full_and_suffix_slices():
|
| 180 |
+
record = ConditionalSampleRecord.model_validate(valid_conditional_record())
|
| 181 |
+
assert record.full_token_ids[:64] == record.prefix_token_ids
|
| 182 |
+
assert record.full_token_ids[64:] == record.continuation_token_ids
|
| 183 |
+
assert record.prefix_exact_match is True
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
@pytest.mark.parametrize("mutation,message", [
|
| 187 |
+
(lambda value: value.update(prefix_exact_match=False), "prefix_exact_match"),
|
| 188 |
+
(lambda value: value["full_token_ids"].__setitem__(0, 999), "prefix"),
|
| 189 |
+
(lambda value: value.update(reference_token_ids=[1] * 63), "64"),
|
| 190 |
+
])
|
| 191 |
+
def test_conditional_record_rejects_invalid_publication(mutation, message):
|
| 192 |
+
value = valid_conditional_record()
|
| 193 |
+
mutation(value)
|
| 194 |
+
with pytest.raises(ValidationError, match=message):
|
| 195 |
+
ConditionalSampleRecord.model_validate(value)
|
| 196 |
+
```
|
| 197 |
+
|
| 198 |
+
- [ ] **Step 2: Run the schema tests and verify they fail**
|
| 199 |
+
|
| 200 |
+
Run: `pytest tests/test_conditional_schema.py -q`
|
| 201 |
+
|
| 202 |
+
Expected: import fails because `ConditionalSampleRecord` is absent.
|
| 203 |
+
|
| 204 |
+
- [ ] **Step 3: Implement the separate record and canonical schedule**
|
| 205 |
+
|
| 206 |
+
```python
|
| 207 |
+
NonNegativeToken = Annotated[StrictInt, Field(ge=0)]
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
class ConditionalSampleRecord(StrictModel):
|
| 211 |
+
sample_id: StrictInt = Field(ge=0)
|
| 212 |
+
prompt_id: StrictInt = Field(ge=0, le=1023)
|
| 213 |
+
completion_id: StrictInt = Field(ge=0, le=4)
|
| 214 |
+
source_index: StrictInt = Field(ge=0)
|
| 215 |
+
prefix_token_ids: list[NonNegativeToken]
|
| 216 |
+
continuation_token_ids: list[NonNegativeToken] = Field(min_length=1)
|
| 217 |
+
reference_token_ids: list[NonNegativeToken]
|
| 218 |
+
full_token_ids: list[NonNegativeToken] = Field(min_length=65)
|
| 219 |
+
prefix_text: str
|
| 220 |
+
continuation_text: str
|
| 221 |
+
reference_text: str
|
| 222 |
+
full_text: str
|
| 223 |
+
seed: StrictInt
|
| 224 |
+
generation_seconds: Annotated[FiniteFloat, Field(ge=0)]
|
| 225 |
+
prefix_exact_match: Literal[True]
|
| 226 |
+
|
| 227 |
+
@model_validator(mode="after")
|
| 228 |
+
def validate_slices(self):
|
| 229 |
+
if len(self.prefix_token_ids) != 64 or len(self.reference_token_ids) != 64:
|
| 230 |
+
raise ValueError("prefix and reference must each contain 64 tokens")
|
| 231 |
+
if self.full_token_ids[:64] != self.prefix_token_ids:
|
| 232 |
+
raise ValueError("full token prefix differs from fixed prefix")
|
| 233 |
+
if self.full_token_ids[64:] != self.continuation_token_ids:
|
| 234 |
+
raise ValueError("continuation differs from full token suffix")
|
| 235 |
+
return self
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
def expected_conditional_schedule(
|
| 239 |
+
prompt_count: int = 1024,
|
| 240 |
+
diversity_prompt_count: int = 256,
|
| 241 |
+
completions: int = 5,
|
| 242 |
+
) -> list[tuple[int, int]]:
|
| 243 |
+
return [(prompt, 0) for prompt in range(prompt_count)] + [
|
| 244 |
+
(prompt, completion)
|
| 245 |
+
for completion in range(1, completions)
|
| 246 |
+
for prompt in range(diversity_prompt_count)
|
| 247 |
+
]
|
| 248 |
+
```
|
| 249 |
+
|
| 250 |
+
- [ ] **Step 4: Implement streaming conditional I/O and verify exact schedule**
|
| 251 |
+
|
| 252 |
+
The conditional validator must parse JSON with duplicate-key rejection, require `sample_id == line index`, require `(prompt_id, completion_id)` to equal the same index in `expected_conditional_schedule()`, require the dataset model length (128 for LM1B or 1,024 for OWT), validate token bounds against the supplied `vocab_size`, and never call or weaken `validate_samples`.
|
| 253 |
+
|
| 254 |
+
```python
|
| 255 |
+
def test_writer_rejects_duplicate_schedule_entry_without_replacing_final(tmp_path):
|
| 256 |
+
final = tmp_path / "samples.jsonl"
|
| 257 |
+
final.write_text("old\n", encoding="utf-8")
|
| 258 |
+
records = production_records()
|
| 259 |
+
records[1024]["prompt_id"] = 1
|
| 260 |
+
with pytest.raises(SampleValidationError, match="expected prompt/completion"):
|
| 261 |
+
write_conditional_samples_atomic(final, records, expected=2048)
|
| 262 |
+
assert final.read_text(encoding="utf-8") == "old\n"
|
| 263 |
+
```
|
| 264 |
+
|
| 265 |
+
Run: `pytest tests/test_conditional_schema.py tests/test_conditional_io.py tests/test_schema.py tests/test_io.py -q`
|
| 266 |
+
|
| 267 |
+
Expected: all conditional and unconditional schema/I/O tests pass.
|
| 268 |
+
|
| 269 |
+
- [ ] **Step 5: Commit the artifact contract**
|
| 270 |
+
|
| 271 |
+
```bash
|
| 272 |
+
git add src/dlb/schema.py src/dlb/io.py \
|
| 273 |
+
tests/test_conditional_schema.py tests/test_conditional_io.py
|
| 274 |
+
git commit -m "feat: add conditional sample artifact contract"
|
| 275 |
+
```
|
| 276 |
+
|
| 277 |
+
### Task 3: Conditional matrix and shared runner mode
|
| 278 |
+
|
| 279 |
+
**Files:**
|
| 280 |
+
|
| 281 |
+
- Create: `src/dlb/conditional_matrix.py`
|
| 282 |
+
- Modify: `src/dlb/runner.py`
|
| 283 |
+
- Create: `tests/test_conditional_matrix.py`
|
| 284 |
+
- Create: `tests/test_conditional_runner.py`
|
| 285 |
+
|
| 286 |
+
**Interfaces:**
|
| 287 |
+
|
| 288 |
+
- Consumes: `ConditionalProtocol`, verified prompt manifests, conditional I/O, existing registry/checkpoint provenance, and existing `SampleAdapter`.
|
| 289 |
+
- Produces: `ConditionalMatrixTask`, `build_conditional_matrix(registry, root=None, protocol=None) -> list[ConditionalMatrixTask]`, schema `dlb-conditional-generation-matrix-v1`, and conditional `RunRequest` fields passed unchanged to adapters.
|
| 290 |
+
|
| 291 |
+
- [ ] **Step 1: Write failing matrix parity and path-isolation tests**
|
| 292 |
+
|
| 293 |
+
```python
|
| 294 |
+
def test_conditional_matrix_matches_supported_unconditional_tasks(registry, tmp_path):
|
| 295 |
+
ordinary = build_matrix(registry, root=tmp_path)
|
| 296 |
+
conditional = build_conditional_matrix(registry, root=tmp_path)
|
| 297 |
+
assert len(conditional) == len(ordinary) == 132
|
| 298 |
+
assert [(t.model, t.dataset, t.steps) for t in conditional] == [
|
| 299 |
+
(t.model, t.dataset, t.steps) for t in ordinary
|
| 300 |
+
]
|
| 301 |
+
assert all("/results/conditional/" in task.sample_dir for task in conditional)
|
| 302 |
+
assert all(task.sample_count == 2048 for task in conditional)
|
| 303 |
+
|
| 304 |
+
|
| 305 |
+
def test_conditional_unsupported_inventory_is_rdlm_owt(registry):
|
| 306 |
+
assert conditional_unsupported_inventory(registry) == [{
|
| 307 |
+
"status": "unsupported", "model": "rdlm", "dataset": "owt",
|
| 308 |
+
"category": "few", "reason": registry.models["rdlm"].datasets["owt"].reason,
|
| 309 |
+
}]
|
| 310 |
+
```
|
| 311 |
+
|
| 312 |
+
- [ ] **Step 2: Write failing request identity and publication tests**
|
| 313 |
+
|
| 314 |
+
```python
|
| 315 |
+
def test_unconditional_identity_has_no_conditional_keys(base_request):
|
| 316 |
+
identity = _identity(base_request, ["python", "sample.py"])
|
| 317 |
+
assert "generation_mode" not in identity
|
| 318 |
+
assert "conditioning_manifest_sha256" not in identity
|
| 319 |
+
|
| 320 |
+
|
| 321 |
+
def test_conditional_identity_binds_manifest_and_schedule(conditional_request):
|
| 322 |
+
identity = _identity(conditional_request, ["python", "sample.py"])
|
| 323 |
+
assert identity["generation_mode"] == "conditional_prefix"
|
| 324 |
+
assert identity["conditioning_manifest_sha256"] == "a" * 64
|
| 325 |
+
assert identity["prefix_length"] == 64
|
| 326 |
+
assert identity["completion_schedule"] == "c0:p0-1023;c1-4:p0-255"
|
| 327 |
+
```
|
| 328 |
+
|
| 329 |
+
- [ ] **Step 3: Implement the separate matrix**
|
| 330 |
+
|
| 331 |
+
```python
|
| 332 |
+
CONDITIONAL_MATRIX_SCHEMA = "dlb-conditional-generation-matrix-v1"
|
| 333 |
+
|
| 334 |
+
@dataclass(frozen=True)
|
| 335 |
+
class ConditionalMatrixTask:
|
| 336 |
+
task_id: str
|
| 337 |
+
category: str
|
| 338 |
+
model: str
|
| 339 |
+
dataset: str
|
| 340 |
+
steps: int
|
| 341 |
+
sample_count: Literal[2048]
|
| 342 |
+
seed: int
|
| 343 |
+
environment: str
|
| 344 |
+
adapter: str
|
| 345 |
+
source: str
|
| 346 |
+
provenance: str
|
| 347 |
+
protocol: Literal["c64_zs_v1"]
|
| 348 |
+
conditioning_manifest: str
|
| 349 |
+
conditioning_manifest_sha256: str
|
| 350 |
+
sample_dir: str
|
| 351 |
+
metrics_path: str
|
| 352 |
+
timing_path: str
|
| 353 |
+
```
|
| 354 |
+
|
| 355 |
+
Derive model/dataset/step ordering from `build_matrix`, replace only protocol-specific columns and paths, validate the two prompt manifests first, and reuse the existing unsupported inventory writer with a conditional schema header.
|
| 356 |
+
|
| 357 |
+
- [ ] **Step 4: Extend `RunRequest` and branch publication without changing unconditional behavior**
|
| 358 |
+
|
| 359 |
+
```python
|
| 360 |
+
@dataclass(frozen=True)
|
| 361 |
+
class RunRequest:
|
| 362 |
+
# existing fields remain in their current order
|
| 363 |
+
generation_mode: Literal["unconditional", "conditional_prefix"] = "unconditional"
|
| 364 |
+
conditioning_manifest: str | None = None
|
| 365 |
+
conditioning_manifest_sha256: str | None = None
|
| 366 |
+
conditioning_config_sha256: str | None = None
|
| 367 |
+
prefix_length: int | None = None
|
| 368 |
+
evaluation_continuation_length: int | None = None
|
| 369 |
+
prompt_count: int | None = None
|
| 370 |
+
diversity_prompt_count: int | None = None
|
| 371 |
+
completions_per_diversity_prompt: int | None = None
|
| 372 |
+
completion_schedule: str | None = None
|
| 373 |
+
```
|
| 374 |
+
|
| 375 |
+
`_resolve_request` must call `verify_prompts` for conditional requests and reject conditional fields on unconditional requests. `_identity` must add conditional keys only when `generation_mode == "conditional_prefix"`. `run_experiment` selects `validate_samples`/`write_samples_atomic` for unconditional mode and `validate_conditional_samples`/`write_conditional_samples_atomic` for conditional mode. Add matching CLI flags including `--generation-mode`, `--conditioning-manifest`, and `--conditioning-config`.
|
| 376 |
+
|
| 377 |
+
Run: `pytest tests/test_conditional_matrix.py tests/test_conditional_runner.py tests/test_matrix.py tests/test_runner.py -q`
|
| 378 |
+
|
| 379 |
+
Expected: all tests pass and the old runner fixtures produce byte-for-byte equivalent unconditional identity dictionaries.
|
| 380 |
+
|
| 381 |
+
- [ ] **Step 5: Commit runner and matrix support**
|
| 382 |
+
|
| 383 |
+
```bash
|
| 384 |
+
git add src/dlb/conditional_matrix.py src/dlb/runner.py \
|
| 385 |
+
tests/test_conditional_matrix.py tests/test_conditional_runner.py
|
| 386 |
+
git commit -m "feat: add conditional matrix and runner mode"
|
| 387 |
+
```
|
| 388 |
+
|
| 389 |
+
### Task 4: Shared hard-prefix runtime primitives
|
| 390 |
+
|
| 391 |
+
**Files:**
|
| 392 |
+
|
| 393 |
+
- Create: `src/dlb/adapters/conditional_runtime.py`
|
| 394 |
+
- Create: `tests/test_conditional_runtime.py`
|
| 395 |
+
|
| 396 |
+
**Interfaces:**
|
| 397 |
+
|
| 398 |
+
- Consumes: verified `PromptRecord` rows and tensors produced by model-specific samplers.
|
| 399 |
+
- Produces: `ConditioningBatch`, `load_conditioning_batch(manifest_path, expected_manifest_sha256, completion_id, prompt_start, batch_size, device, vocab_size) -> ConditioningBatch`, `clamp_token_prefix(state, prefix_ids)`, `clamp_vocab_prefix(state, prefix_ids)`, `clamp_embedding_prefix(state, clean_embeddings)`, `token_project_fn(prefix_ids)`, `vocab_project_fn(prefix_ids)`, `embedding_project_fn(clean_embeddings)`, and `patched_attribute(owner, name, replacement)`.
|
| 400 |
+
|
| 401 |
+
- [ ] **Step 1: Write failing clamp and restoration tests**
|
| 402 |
+
|
| 403 |
+
```python
|
| 404 |
+
def test_token_projector_clamps_every_call():
|
| 405 |
+
prefix = torch.tensor([[4, 5], [6, 7]])
|
| 406 |
+
project = token_project_fn(prefix)
|
| 407 |
+
state = torch.tensor([[99, 99, 8], [99, 99, 9]])
|
| 408 |
+
assert project(state).tolist() == [[4, 5, 8], [6, 7, 9]]
|
| 409 |
+
state[:, :2] = 88
|
| 410 |
+
assert project(state).tolist() == [[4, 5, 8], [6, 7, 9]]
|
| 411 |
+
|
| 412 |
+
|
| 413 |
+
def test_scoped_patch_restores_even_after_sampler_error():
|
| 414 |
+
owner = FakeOwner()
|
| 415 |
+
original = owner.update
|
| 416 |
+
with pytest.raises(RuntimeError):
|
| 417 |
+
with patched_attribute(owner, "update", lambda *_: (_ for _ in ()).throw(RuntimeError())):
|
| 418 |
+
owner.update(None)
|
| 419 |
+
assert owner.update.__func__ is original.__func__
|
| 420 |
+
```
|
| 421 |
+
|
| 422 |
+
- [ ] **Step 2: Run tests and verify the runtime module is missing**
|
| 423 |
+
|
| 424 |
+
Run: `pytest tests/test_conditional_runtime.py -q`
|
| 425 |
+
|
| 426 |
+
Expected: import failure for `dlb.adapters.conditional_runtime`.
|
| 427 |
+
|
| 428 |
+
- [ ] **Step 3: Implement shape-safe token, vocabulary, and embedding projection**
|
| 429 |
+
|
| 430 |
+
```python
|
| 431 |
+
@dataclass(frozen=True)
|
| 432 |
+
class ConditioningBatch:
|
| 433 |
+
prompt_ids: Sequence[int]
|
| 434 |
+
source_indices: Sequence[int]
|
| 435 |
+
prefix_token_ids: object
|
| 436 |
+
reference_token_ids: object
|
| 437 |
+
|
| 438 |
+
|
| 439 |
+
def clamp_token_prefix(state, prefix_ids):
|
| 440 |
+
if state.ndim != 2 or prefix_ids.ndim != 2:
|
| 441 |
+
raise ValueError("token state and prefix must be rank two")
|
| 442 |
+
if state.shape[0] != prefix_ids.shape[0] or state.shape[1] < prefix_ids.shape[1]:
|
| 443 |
+
raise ValueError("token state is incompatible with prefix batch")
|
| 444 |
+
result = state.clone()
|
| 445 |
+
result[:, : prefix_ids.shape[1]] = prefix_ids.to(device=state.device, dtype=state.dtype)
|
| 446 |
+
return result
|
| 447 |
+
|
| 448 |
+
|
| 449 |
+
def clamp_vocab_prefix(state, prefix_ids):
|
| 450 |
+
clean = torch.nn.functional.one_hot(prefix_ids, num_classes=state.shape[-1]).to(state)
|
| 451 |
+
result = state.clone()
|
| 452 |
+
result[:, : prefix_ids.shape[1], :] = clean
|
| 453 |
+
return result
|
| 454 |
+
|
| 455 |
+
|
| 456 |
+
def clamp_embedding_prefix(state, clean_embeddings):
|
| 457 |
+
result = state.clone()
|
| 458 |
+
result[:, : clean_embeddings.shape[1], :] = clean_embeddings.to(state)
|
| 459 |
+
return result
|
| 460 |
+
```
|
| 461 |
+
|
| 462 |
+
All functions must reject device/batch/sequence/vocabulary mismatches, preserve suffix bytes, and avoid in-place mutation of caller-owned tensors. Projector closures expose `conditioning_implementation` metadata and are safe for repeated calls.
|
| 463 |
+
|
| 464 |
+
- [ ] **Step 4: Add exact prompt-batch schedule tests and pass them**
|
| 465 |
+
|
| 466 |
+
`load_conditioning_batch` accepts a prompt JSONL path, `completion_id`, `start_prompt`, `batch_size`, device, and tokenizer vocabulary. It rejects completion 0 beyond prompt 1023, completions 1–4 beyond prompt 255, and any batch that crosses its schedule boundary.
|
| 467 |
+
|
| 468 |
+
Run: `pytest tests/test_conditional_runtime.py -q`
|
| 469 |
+
|
| 470 |
+
Expected: token, vocabulary, embedding, schedule, and restoration cases pass.
|
| 471 |
+
|
| 472 |
+
- [ ] **Step 5: Commit runtime primitives**
|
| 473 |
+
|
| 474 |
+
```bash
|
| 475 |
+
git add src/dlb/adapters/conditional_runtime.py tests/test_conditional_runtime.py
|
| 476 |
+
git commit -m "feat: add hard-prefix conditioning runtime"
|
| 477 |
+
```
|
| 478 |
+
|
| 479 |
+
### Task 5: Teacher-family adapters for MDLM, Duo, FLM/FMLM, and CANDI
|
| 480 |
+
|
| 481 |
+
**Files:**
|
| 482 |
+
|
| 483 |
+
- Modify: `src/dlb/adapters/base.py`
|
| 484 |
+
- Modify: `src/dlb/adapters/capture.py`
|
| 485 |
+
- Modify: `src/dlb/adapters/mdlm.py`
|
| 486 |
+
- Modify: `src/dlb/adapters/duo.py`
|
| 487 |
+
- Modify: `src/dlb/adapters/flm.py`
|
| 488 |
+
- Modify: `src/dlb/adapters/candi.py`
|
| 489 |
+
- Create: `tests/test_conditional_teacher_adapters.py`
|
| 490 |
+
- Modify: `tests/test_capture.py`
|
| 491 |
+
|
| 492 |
+
**Interfaces:**
|
| 493 |
+
|
| 494 |
+
- Consumes: conditional request fields and runtime projectors from Tasks 3–4.
|
| 495 |
+
- Produces: uniform capture flags, `install_teacher_conditioning(owner, family, batch, tokenizer)`, and conditional capture records containing prompt/source/completion identity plus full generated token IDs.
|
| 496 |
+
|
| 497 |
+
- [ ] **Step 1: Write failing command serialization tests for every teacher family**
|
| 498 |
+
|
| 499 |
+
```python
|
| 500 |
+
@pytest.mark.parametrize("model", ["flm", "fmlm", "duo", "duo-dcd", "mdlm", "candi"])
|
| 501 |
+
def test_teacher_command_serializes_complete_conditioning_contract(model, conditional_request, adapter_for):
|
| 502 |
+
command = adapter_for(model).build_command(conditional_request(model), RUN_DIR)
|
| 503 |
+
assert option(command, "--generation-mode") == "conditional_prefix"
|
| 504 |
+
assert option(command, "--conditioning-manifest") == conditional_request(model).conditioning_manifest
|
| 505 |
+
assert option(command, "--prefix-length") == "64"
|
| 506 |
+
assert option(command, "--completion-schedule") == "c0:p0-1023;c1-4:p0-255"
|
| 507 |
+
```
|
| 508 |
+
|
| 509 |
+
- [ ] **Step 2: Write fake-sampler tests that observe the prefix at every update**
|
| 510 |
+
|
| 511 |
+
```python
|
| 512 |
+
def test_mdlm_prior_and_every_ancestral_state_keep_real_prefix(fake_mdlm, batch):
|
| 513 |
+
with install_teacher_conditioning(fake_mdlm, "mdlm", batch, fake_tokenizer):
|
| 514 |
+
output = fake_mdlm.sample()
|
| 515 |
+
assert fake_mdlm.observed_prefixes == [batch.prefix.tolist()] * fake_mdlm.step_count
|
| 516 |
+
assert output[:, :64].tolist() == batch.prefix.tolist()
|
| 517 |
+
|
| 518 |
+
|
| 519 |
+
def test_duo_clamps_before_and_after_every_ancestral_update(fake_duo, batch):
|
| 520 |
+
with install_teacher_conditioning(fake_duo, "duo", batch, fake_tokenizer):
|
| 521 |
+
output = fake_duo.sample()
|
| 522 |
+
assert all(prefix == batch.prefix.tolist() for prefix in fake_duo.before_update)
|
| 523 |
+
assert all(prefix == batch.prefix.tolist() for prefix in fake_duo.after_update)
|
| 524 |
+
|
| 525 |
+
|
| 526 |
+
def test_flm_forward_always_receives_clean_prefix_one_hot(fake_flm, batch):
|
| 527 |
+
with install_teacher_conditioning(fake_flm, "flm", batch, fake_tokenizer):
|
| 528 |
+
fake_flm.sample()
|
| 529 |
+
assert all(torch.equal(value, batch.clean_one_hot) for value in fake_flm.forward_prefixes)
|
| 530 |
+
```
|
| 531 |
+
|
| 532 |
+
- [ ] **Step 3: Extend the capture invocation and adapter output conversion**
|
| 533 |
+
|
| 534 |
+
Add these exact capture-only arguments: `--generation-mode`, `--conditioning-manifest`, `--conditioning-manifest-sha256`, `--prefix-length`, `--prompt-count`, `--diversity-prompt-count`, `--completions-per-diversity-prompt`, and `--completion-schedule`. In conditional mode, `_capture_teacher` iterates the canonical schedule without ceiling batches and emits one raw capture item per scheduled prompt/completion pair with `prompt_id`, `completion_id`, `source_index`, `prefix_token_ids`, `reference_token_ids`, `full_token_ids`, and batch elapsed time divided by the exact batch size.
|
| 535 |
+
|
| 536 |
+
`BaseTeacherAdapter.convert_outputs` must branch to a `_convert_conditional_capture_outputs` method that decodes prefix/reference/full/continuation separately with the locked canonical tokenizer and constructs `ConditionalSampleRecord`; its unconditional branch remains unchanged.
|
| 537 |
+
|
| 538 |
+
- [ ] **Step 4: Implement family-specific hard conditioning**
|
| 539 |
+
|
| 540 |
+
```python
|
| 541 |
+
if family == "mdlm":
|
| 542 |
+
patch_prior_to_insert_clean_prefix(owner, batch.prefix_token_ids)
|
| 543 |
+
patch_ancestral_update_with_token_pre_post_clamp(owner, batch.prefix_token_ids)
|
| 544 |
+
elif family == "duo":
|
| 545 |
+
patch_ancestral_update_with_token_pre_post_clamp(owner, batch.prefix_token_ids)
|
| 546 |
+
elif family in {"flm", "fmlm"}:
|
| 547 |
+
patch_forward_with_vocab_prefix(owner, batch.prefix_token_ids)
|
| 548 |
+
elif family == "candi":
|
| 549 |
+
kwargs["prompt_tokens"] = batch.prefix_token_ids
|
| 550 |
+
kwargs["prompt_mask"] = torch.ones_like(batch.prefix_token_ids, dtype=torch.bool)
|
| 551 |
+
```
|
| 552 |
+
|
| 553 |
+
All paths perform a final token clamp plus exact assertion. Metadata is `native_projection` for CANDI and `zero_shot_runtime_projection` for MDLM/Duo/FLM/FMLM. FMLM metadata also contains `paper_conditional_training_reproduced: false`.
|
| 554 |
+
|
| 555 |
+
Run: `pytest tests/test_conditional_teacher_adapters.py tests/test_capture.py tests/test_teacher_adapters.py -q`
|
| 556 |
+
|
| 557 |
+
Expected: conditional tests pass and all existing teacher/capture tests remain green.
|
| 558 |
+
|
| 559 |
+
- [ ] **Step 5: Commit teacher-family integration**
|
| 560 |
+
|
| 561 |
+
```bash
|
| 562 |
+
git add src/dlb/adapters/base.py src/dlb/adapters/capture.py \
|
| 563 |
+
src/dlb/adapters/mdlm.py src/dlb/adapters/duo.py \
|
| 564 |
+
src/dlb/adapters/flm.py src/dlb/adapters/candi.py \
|
| 565 |
+
tests/test_conditional_teacher_adapters.py tests/test_capture.py
|
| 566 |
+
git commit -m "feat: condition teacher-family samplers on fixed prefixes"
|
| 567 |
+
```
|
| 568 |
+
|
| 569 |
+
### Task 6: Native projected SDTT and Di4C samplers
|
| 570 |
+
|
| 571 |
+
**Files:**
|
| 572 |
+
|
| 573 |
+
- Modify: `src/dlb/adapters/sdtt.py`
|
| 574 |
+
- Modify: `src/dlb/adapters/di4c.py`
|
| 575 |
+
- Modify: `adapters/_distilled_runtime.py`
|
| 576 |
+
- Modify: `adapters/sample_sdtt.py`
|
| 577 |
+
- Modify: `adapters/sample_di4c.py`
|
| 578 |
+
- Create: `tests/test_conditional_distilled_adapters.py`
|
| 579 |
+
|
| 580 |
+
**Interfaces:**
|
| 581 |
+
|
| 582 |
+
- Consumes: conditional request/capture contract and `token_project_fn`/`vocab_project_fn`.
|
| 583 |
+
- Produces: native `project_fn` wiring for SDTT and Di4C, exact conditional capture JSON, and benchmark callbacks that include projection time.
|
| 584 |
+
|
| 585 |
+
- [ ] **Step 1: Write failing SDTT/Di4C CLI and native-projection tests**
|
| 586 |
+
|
| 587 |
+
```python
|
| 588 |
+
@pytest.mark.parametrize("model", ["mdlm-sdtt", "duo-di4c", "mdlm-di4c"])
|
| 589 |
+
def test_distilled_command_passes_manifest_sha_and_schedule(model, request, adapter_for):
|
| 590 |
+
command = adapter_for(model).build_command(request(model), RUN_DIR)
|
| 591 |
+
assert option(command, "--conditioning-manifest-sha256") == "a" * 64
|
| 592 |
+
assert option(command, "--completion-schedule") == "c0:p0-1023;c1-4:p0-255"
|
| 593 |
+
|
| 594 |
+
|
| 595 |
+
def test_sdtt_native_project_fn_is_called_at_every_step(fake_sdtt_runtime, prefix):
|
| 596 |
+
result = fake_sdtt_runtime.sample_with_prefix(prefix, steps=7)
|
| 597 |
+
assert fake_sdtt_runtime.project_calls == 7
|
| 598 |
+
assert torch.equal(result[:, :64], prefix)
|
| 599 |
+
```
|
| 600 |
+
|
| 601 |
+
- [ ] **Step 2: Run focused tests and confirm missing flags/projection failures**
|
| 602 |
+
|
| 603 |
+
Run: `pytest tests/test_conditional_distilled_adapters.py -q`
|
| 604 |
+
|
| 605 |
+
Expected: assertions fail because current wrappers have no conditional arguments.
|
| 606 |
+
|
| 607 |
+
- [ ] **Step 3: Add authenticated prompt loading to the server runtime**
|
| 608 |
+
|
| 609 |
+
`adapters/_distilled_runtime.py` must verify prompt-file and manifest SHA-256 before model materialization, validate tokenizer ID/revision and vocabulary bounds, materialize only the exact prompt batch, and expose:
|
| 610 |
+
|
| 611 |
+
```python
|
| 612 |
+
def load_conditional_batch(
|
| 613 |
+
manifest_path: Path,
|
| 614 |
+
expected_manifest_sha256: str,
|
| 615 |
+
*,
|
| 616 |
+
completion_id: int,
|
| 617 |
+
prompt_start: int,
|
| 618 |
+
prompt_count: int,
|
| 619 |
+
device: str,
|
| 620 |
+
vocab_size: int,
|
| 621 |
+
) -> ConditioningBatch:
|
| 622 |
+
return load_conditioning_batch(
|
| 623 |
+
manifest_path,
|
| 624 |
+
expected_manifest_sha256,
|
| 625 |
+
completion_id=completion_id,
|
| 626 |
+
prompt_start=prompt_start,
|
| 627 |
+
batch_size=prompt_count,
|
| 628 |
+
device=device,
|
| 629 |
+
vocab_size=vocab_size,
|
| 630 |
+
)
|
| 631 |
+
```
|
| 632 |
+
|
| 633 |
+
- [ ] **Step 4: Pass native `project_fn` throughout both sampler entrypoints**
|
| 634 |
+
|
| 635 |
+
For masked states, the native projection writes prefix token IDs into the discrete state. For categorical/one-hot states, it writes the clean prefix one-hot endpoint. Pass the closure to the pinned upstream sampler's existing `project_fn` parameter for every reverse step, assert the result prefix, and write conditional capture fields in canonical schedule order. `benchmark_model` must time the same closure inside the measured generation callback.
|
| 636 |
+
|
| 637 |
+
Run: `pytest tests/test_conditional_distilled_adapters.py tests/test_distilled_adapters.py tests/test_timing.py -q`
|
| 638 |
+
|
| 639 |
+
Expected: new and existing distilled adapter tests pass.
|
| 640 |
+
|
| 641 |
+
- [ ] **Step 5: Commit native distilled projections**
|
| 642 |
+
|
| 643 |
+
```bash
|
| 644 |
+
git add src/dlb/adapters/sdtt.py src/dlb/adapters/di4c.py \
|
| 645 |
+
adapters/_distilled_runtime.py adapters/sample_sdtt.py adapters/sample_di4c.py \
|
| 646 |
+
tests/test_conditional_distilled_adapters.py
|
| 647 |
+
git commit -m "feat: add native prefix projection to distilled samplers"
|
| 648 |
+
```
|
| 649 |
+
|
| 650 |
+
### Task 7: LangFlow embedding projection and RDLM manifold projection
|
| 651 |
+
|
| 652 |
+
**Files:**
|
| 653 |
+
|
| 654 |
+
- Modify: `src/dlb/adapters/langflow.py`
|
| 655 |
+
- Modify: `src/dlb/adapters/rdlm.py`
|
| 656 |
+
- Modify: `src/dlb/adapters/capture.py`
|
| 657 |
+
- Modify: `adapters/sample_langflow.py`
|
| 658 |
+
- Create: `tests/test_conditional_continuous_adapters.py`
|
| 659 |
+
|
| 660 |
+
**Interfaces:**
|
| 661 |
+
|
| 662 |
+
- Consumes: `embedding_project_fn`, `vocab_project_fn`, conditional request fields, and canonical prompt batches.
|
| 663 |
+
- Produces: `install_langflow_conditioning(model, batch, tokenizer)` and `run_rdlm_conditional(model, batch, tokenizer, sampling_config)`, both with final token exact-match assertions.
|
| 664 |
+
|
| 665 |
+
- [ ] **Step 1: Write failing continuous-state observation tests**
|
| 666 |
+
|
| 667 |
+
```python
|
| 668 |
+
def test_langflow_clamps_noisy_and_self_conditioning_before_every_forward(fake_langflow, batch):
|
| 669 |
+
with install_langflow_conditioning(fake_langflow, batch):
|
| 670 |
+
output = fake_langflow.generate()
|
| 671 |
+
assert len(fake_langflow.noisy_prefixes) == fake_langflow.step_count
|
| 672 |
+
assert all(torch.equal(x, batch.clean_embeddings) for x in fake_langflow.noisy_prefixes)
|
| 673 |
+
assert all(torch.equal(x, batch.clean_embeddings) for x in fake_langflow.self_cond_prefixes)
|
| 674 |
+
assert torch.equal(output[:, :64], batch.prefix_token_ids)
|
| 675 |
+
|
| 676 |
+
|
| 677 |
+
def test_rdlm_passes_clean_manifold_projection_to_sampling_factory(fake_rdlm, batch):
|
| 678 |
+
output = run_rdlm_conditional(fake_rdlm, batch)
|
| 679 |
+
assert fake_rdlm.proj_fn is not None
|
| 680 |
+
assert fake_rdlm.proj_calls == fake_rdlm.step_count
|
| 681 |
+
assert torch.equal(output[:, :64], batch.prefix_token_ids)
|
| 682 |
+
```
|
| 683 |
+
|
| 684 |
+
- [ ] **Step 2: Run focused tests and verify current adapters fail**
|
| 685 |
+
|
| 686 |
+
Run: `pytest tests/test_conditional_continuous_adapters.py -q`
|
| 687 |
+
|
| 688 |
+
Expected: failures show no LangFlow conditioning hook and no RDLM projection.
|
| 689 |
+
|
| 690 |
+
- [ ] **Step 3: Implement LangFlow clean-embedding projection**
|
| 691 |
+
|
| 692 |
+
Before each LangFlow forward evaluation, derive clean prefix embeddings from the loaded model embedding table and replace positions `:64` in both `noisy_embeds` and non-null `x_self_cond`. The suffix tensor must be byte-identical before/after the hook. Restore the original forward method after sampling and record `conditioning_implementation: zero_shot_runtime_projection` and `paper_conditional_training_reproduced: false`.
|
| 693 |
+
|
| 694 |
+
- [ ] **Step 4: Implement RDLM native manifold projection**
|
| 695 |
+
|
| 696 |
+
Construct the clean one-hot prefix endpoint in RDLM's vocabulary/manifold coordinates and pass `proj_fn=project` to the pinned `get_sampling_fn` call. The closure clamps every call, preserves suffix coordinates, and the decoded output receives a final token-level assertion. Record `conditioning_implementation: native_projection`.
|
| 697 |
+
|
| 698 |
+
Run: `pytest tests/test_conditional_continuous_adapters.py tests/test_continuous_adapters.py tests/test_capture.py -q`
|
| 699 |
+
|
| 700 |
+
Expected: all continuous, capture, LangFlow, and RDLM tests pass.
|
| 701 |
+
|
| 702 |
+
- [ ] **Step 5: Commit continuous/manifold integration**
|
| 703 |
+
|
| 704 |
+
```bash
|
| 705 |
+
git add src/dlb/adapters/langflow.py src/dlb/adapters/rdlm.py \
|
| 706 |
+
src/dlb/adapters/capture.py adapters/sample_langflow.py \
|
| 707 |
+
tests/test_conditional_continuous_adapters.py
|
| 708 |
+
git commit -m "feat: condition flow and manifold samplers on prefixes"
|
| 709 |
+
```
|
| 710 |
+
|
| 711 |
+
### Task 8: Conditional quality and diversity evaluation
|
| 712 |
+
|
| 713 |
+
**Files:**
|
| 714 |
+
|
| 715 |
+
- Create: `evaluation/conditional_perplexity.py`
|
| 716 |
+
- Create: `evaluation/mauve_score.py`
|
| 717 |
+
- Create: `evaluation/conditional_evaluate.py`
|
| 718 |
+
- Modify: `evaluation/self_bleu.py`
|
| 719 |
+
- Modify: `evaluation/unigram_entropy.py`
|
| 720 |
+
- Create: `tests/test_conditional_perplexity.py`
|
| 721 |
+
- Create: `tests/test_mauve_score.py`
|
| 722 |
+
- Create: `tests/test_conditional_evaluate.py`
|
| 723 |
+
|
| 724 |
+
**Interfaces:**
|
| 725 |
+
|
| 726 |
+
- Consumes: 2,048 validated `ConditionalSampleRecord` objects and pinned local GPT-2-large assets.
|
| 727 |
+
- Produces: `compute_conditional_ppl(prefixes, continuations, model, tokenizer, batch_size=8)`, `compute_mauve_score(generated, references, assets, device_id)`, `compute_grouped_self_bleu(token_rows_by_prompt)`, `conditional_entropy(records, slice_length=64)`, and schema `dlb-conditional-metrics-v1`.
|
| 728 |
+
|
| 729 |
+
- [ ] **Step 1: Write failing suffix-only causal-loss tests**
|
| 730 |
+
|
| 731 |
+
```python
|
| 732 |
+
def test_conditional_ppl_masks_prefix_targets_and_counts_suffix_only(fake_scorer, tokenizer):
|
| 733 |
+
result = compute_conditional_ppl(
|
| 734 |
+
prefixes=["alpha beta"], continuations=["gamma delta"],
|
| 735 |
+
model=fake_scorer, tokenizer=tokenizer, batch_size=1,
|
| 736 |
+
)
|
| 737 |
+
assert fake_scorer.target_masks == [[0, 0, 1, 1]]
|
| 738 |
+
assert result.valid_token_count == 2
|
| 739 |
+
|
| 740 |
+
|
| 741 |
+
def test_conditional_ppl_tokenizes_prefix_and_suffix_separately(spy_tokenizer, fake_scorer):
|
| 742 |
+
compute_conditional_ppl(["prefix"], [" suffix"], fake_scorer, spy_tokenizer)
|
| 743 |
+
assert spy_tokenizer.calls == [(["prefix"], False), ([" suffix"], False)]
|
| 744 |
+
```
|
| 745 |
+
|
| 746 |
+
- [ ] **Step 2: Implement target-mask scoring**
|
| 747 |
+
|
| 748 |
+
```python
|
| 749 |
+
def compute_conditional_ppl(prefixes, continuations, model, tokenizer, *, batch_size=8):
|
| 750 |
+
prefix_rows = tokenize_without_special_tokens(tokenizer, prefixes)
|
| 751 |
+
suffix_rows = tokenize_without_special_tokens(tokenizer, continuations)
|
| 752 |
+
input_rows = [prefix + suffix for prefix, suffix in zip(prefix_rows, suffix_rows, strict=True)]
|
| 753 |
+
target_masks = [[0] * len(prefix) + [1] * len(suffix) for prefix, suffix in zip(prefix_rows, suffix_rows, strict=True)]
|
| 754 |
+
return score_causal_targets(model, input_rows, target_masks, batch_size=batch_size)
|
| 755 |
+
```
|
| 756 |
+
|
| 757 |
+
The scorer shifts logits and target masks together, rejects empty/truncated suffix targets, and limits the combined sequence to 1,024 evaluator tokens. Run the same computation for ground-truth references.
|
| 758 |
+
|
| 759 |
+
- [ ] **Step 3: Write and pass pinned MAUVE tests**
|
| 760 |
+
|
| 761 |
+
```python
|
| 762 |
+
def test_mauve_uses_local_snapshot_and_aligned_completion_zero(monkeypatch, assets):
|
| 763 |
+
call = install_fake_mauve(monkeypatch)
|
| 764 |
+
result = compute_mauve_score(GENERATED_1024, REFERENCES_1024, assets, device_id=0)
|
| 765 |
+
assert call["featurize_model_name"] == str(assets.model_path)
|
| 766 |
+
assert call["p_text"] == REFERENCES_1024
|
| 767 |
+
assert call["q_text"] == GENERATED_1024
|
| 768 |
+
assert result.sample_count == 1024
|
| 769 |
+
```
|
| 770 |
+
|
| 771 |
+
The wrapper imports `mauve` lazily, requires `mauve-text==0.3.0`, sets Hugging Face offline variables, uses the pinned local GPT-2-large snapshot, and records seed, bucket count, model revision, and sample count.
|
| 772 |
+
|
| 773 |
+
- [ ] **Step 4: Implement grouped diversity, entropy, exact-prefix gating, and CLI artifact**
|
| 774 |
+
|
| 775 |
+
`conditional_evaluate` retokenizes each continuation text independently with the pinned local GPT-2 evaluator tokenizer, records that evaluator revision in the metric artifact, and passes exactly five 64-token-truncated GPT-2 rows for each prompt `0..255` to `compute_grouped_self_bleu`; each candidate uses its other four same-prompt rows as references with BLEU-4 equal weights and method-1 smoothing. `conditional_entropy` uses the first 64 model-native continuation IDs and no prefix tokens. `conditional_evaluate.evaluate` rejects any schedule or prefix mismatch before loading evaluator models and publishes:
|
| 776 |
+
|
| 777 |
+
```python
|
| 778 |
+
{
|
| 779 |
+
"schema": "dlb-conditional-metrics-v1",
|
| 780 |
+
"protocol": "c64_zs_v1",
|
| 781 |
+
"sample_count": 2048,
|
| 782 |
+
"quality_sample_count": 1024,
|
| 783 |
+
"diversity_prompt_count": 256,
|
| 784 |
+
"samples_sha256": sha256_file(samples_path),
|
| 785 |
+
"prompt_manifest_sha256": manifest_sha,
|
| 786 |
+
"metrics": {
|
| 787 |
+
"conditional_generation_perplexity": generated_ppl,
|
| 788 |
+
"reference_conditional_perplexity": reference_ppl,
|
| 789 |
+
"mauve": mauve_result,
|
| 790 |
+
"grouped_self_bleu": self_bleu_result,
|
| 791 |
+
"continuation_entropy": entropy_result,
|
| 792 |
+
"prefix_exact_match_rate": 1.0,
|
| 793 |
+
},
|
| 794 |
+
}
|
| 795 |
+
```
|
| 796 |
+
|
| 797 |
+
Run: `pytest tests/test_conditional_perplexity.py tests/test_mauve_score.py tests/test_conditional_evaluate.py tests/test_perplexity.py tests/test_self_bleu.py tests/test_entropy.py -q`
|
| 798 |
+
|
| 799 |
+
Expected: all conditional and unconditional evaluation tests pass.
|
| 800 |
+
|
| 801 |
+
- [ ] **Step 5: Commit conditional evaluation**
|
| 802 |
+
|
| 803 |
+
```bash
|
| 804 |
+
git add evaluation/conditional_perplexity.py evaluation/mauve_score.py \
|
| 805 |
+
evaluation/conditional_evaluate.py evaluation/self_bleu.py \
|
| 806 |
+
evaluation/unigram_entropy.py tests/test_conditional_perplexity.py \
|
| 807 |
+
tests/test_mauve_score.py tests/test_conditional_evaluate.py
|
| 808 |
+
git commit -m "feat: evaluate conditional quality and diversity"
|
| 809 |
+
```
|
| 810 |
+
|
| 811 |
+
### Task 9: Conditional timing protocol
|
| 812 |
+
|
| 813 |
+
**Files:**
|
| 814 |
+
|
| 815 |
+
- Create: `src/dlb/conditional_benchmarking.py`
|
| 816 |
+
- Modify: `src/dlb/benchmarking.py`
|
| 817 |
+
- Modify: `src/dlb/timing.py`
|
| 818 |
+
- Modify: `src/dlb/adapters/base.py`
|
| 819 |
+
- Create: `tests/test_conditional_timing.py`
|
| 820 |
+
|
| 821 |
+
**Interfaces:**
|
| 822 |
+
|
| 823 |
+
- Consumes: prompt 0, conditional matrix task, adapter `benchmark_hook`, and existing CUDA timing primitives.
|
| 824 |
+
- Produces: schema `dlb-conditional-timing-v1` with 5 warmups, 32 repeats, batch size 1, full model canvas, and conditional provenance.
|
| 825 |
+
|
| 826 |
+
- [ ] **Step 1: Write failing timer-boundary and metadata tests**
|
| 827 |
+
|
| 828 |
+
```python
|
| 829 |
+
def test_conditional_timer_moves_prompt_before_first_clock(monkeypatch, fake_adapter):
|
| 830 |
+
events = install_timing_spies(monkeypatch, fake_adapter)
|
| 831 |
+
benchmark_conditional_cell(fake_task(), fake_adapter)
|
| 832 |
+
assert events.index("prompt.to:cuda") < events.index("clock:start")
|
| 833 |
+
assert events.count("generate_with_projection") == 5 + 32
|
| 834 |
+
|
| 835 |
+
|
| 836 |
+
def test_owt_timing_records_full_canvas_not_metric_slice():
|
| 837 |
+
metadata = conditional_timing_metadata(fake_task(dataset="owt"))
|
| 838 |
+
assert metadata["model_length"] == 1024
|
| 839 |
+
assert metadata["generated_suffix_length"] == 960
|
| 840 |
+
assert metadata["evaluation_continuation_length"] == 64
|
| 841 |
+
```
|
| 842 |
+
|
| 843 |
+
- [ ] **Step 2: Run focused timing tests and verify missing conditional module**
|
| 844 |
+
|
| 845 |
+
Run: `pytest tests/test_conditional_timing.py -q`
|
| 846 |
+
|
| 847 |
+
Expected: import failure for `dlb.conditional_benchmarking`.
|
| 848 |
+
|
| 849 |
+
- [ ] **Step 3: Implement conditional benchmark rendering and publication**
|
| 850 |
+
|
| 851 |
+
Use a `RunRequest` with `generation_mode="conditional_prefix"`, `sample_count=1`, `prompt_count=1`, `diversity_prompt_count=0`, and `completions_per_diversity_prompt=1`. Load prompt 0 and move it to the target GPU before passing the measured closure to `benchmark`. Reuse the current synchronization and raw-duration logic with `warmups=5`, `repeats=32`; projection/clamp calls stay inside `generate_one`.
|
| 852 |
+
|
| 853 |
+
```python
|
| 854 |
+
metadata.update({
|
| 855 |
+
"schema": "dlb-conditional-timing-v1",
|
| 856 |
+
"protocol": "c64_zs_v1",
|
| 857 |
+
"prompt_id": 0,
|
| 858 |
+
"prefix_length": 64,
|
| 859 |
+
"batch_size": 1,
|
| 860 |
+
"warmups": 5,
|
| 861 |
+
"repeats": 32,
|
| 862 |
+
"conditioning_manifest_sha256": task.conditioning_manifest_sha256,
|
| 863 |
+
})
|
| 864 |
+
```
|
| 865 |
+
|
| 866 |
+
- [ ] **Step 4: Prove unconditional timing remains unchanged**
|
| 867 |
+
|
| 868 |
+
Run: `pytest tests/test_conditional_timing.py tests/test_timing.py -q`
|
| 869 |
+
|
| 870 |
+
Expected: conditional timing passes; existing timing schema, warmup/repeat behavior, and benchmark command tests remain green.
|
| 871 |
+
|
| 872 |
+
- [ ] **Step 5: Commit conditional timing**
|
| 873 |
+
|
| 874 |
+
```bash
|
| 875 |
+
git add src/dlb/conditional_benchmarking.py src/dlb/benchmarking.py \
|
| 876 |
+
src/dlb/timing.py src/dlb/adapters/base.py tests/test_conditional_timing.py
|
| 877 |
+
git commit -m "feat: benchmark conditional generation latency"
|
| 878 |
+
```
|
| 879 |
+
|
| 880 |
+
### Task 10: Strict conditional aggregation
|
| 881 |
+
|
| 882 |
+
**Files:**
|
| 883 |
+
|
| 884 |
+
- Create: `src/dlb/conditional_aggregate.py`
|
| 885 |
+
- Create: `scripts/aggregate_conditional_results.py`
|
| 886 |
+
- Create: `tests/test_conditional_aggregate.py`
|
| 887 |
+
|
| 888 |
+
**Interfaces:**
|
| 889 |
+
|
| 890 |
+
- Consumes: canonical conditional matrix, unsupported inventory, prompts, sample/run metadata, metrics, and timing artifacts.
|
| 891 |
+
- Produces: `ConditionalAggregateReport`, `aggregate_conditional(root, strict=True, partial=False, output_dir=None)`, `results/conditional/summary/results.csv`, `failures.csv`, `unsupported.csv`, `provenance.json`, and `README.md`.
|
| 892 |
+
|
| 893 |
+
- [ ] **Step 1: Write failing completeness and provenance tests**
|
| 894 |
+
|
| 895 |
+
```python
|
| 896 |
+
def test_conditional_aggregate_accepts_complete_bound_cell(complete_conditional_cell):
|
| 897 |
+
report = aggregate_conditional(complete_conditional_cell.root, strict=True)
|
| 898 |
+
assert report.complete is True
|
| 899 |
+
assert report.rows[0]["prefix_exact_match_rate"] == 1.0
|
| 900 |
+
|
| 901 |
+
|
| 902 |
+
@pytest.mark.parametrize("mutation,message", [
|
| 903 |
+
(remove_one_sample, "2048"),
|
| 904 |
+
(duplicate_completion, "schedule"),
|
| 905 |
+
(change_prompt_sha, "prompt manifest"),
|
| 906 |
+
(change_metric_sample_sha, "sample artifact"),
|
| 907 |
+
(change_timing_steps, "step count"),
|
| 908 |
+
])
|
| 909 |
+
def test_conditional_aggregate_fails_closed(complete_conditional_cell, mutation, message):
|
| 910 |
+
mutation(complete_conditional_cell)
|
| 911 |
+
with pytest.raises(IncompleteConditionalMatrixError, match=message):
|
| 912 |
+
aggregate_conditional(complete_conditional_cell.root, strict=True)
|
| 913 |
+
```
|
| 914 |
+
|
| 915 |
+
- [ ] **Step 2: Run focused tests and confirm missing aggregator**
|
| 916 |
+
|
| 917 |
+
Run: `pytest tests/test_conditional_aggregate.py -q`
|
| 918 |
+
|
| 919 |
+
Expected: import failure for `dlb.conditional_aggregate`.
|
| 920 |
+
|
| 921 |
+
- [ ] **Step 3: Implement cell linkage and strict matrix accounting**
|
| 922 |
+
|
| 923 |
+
For each of the 132 tasks, validate sample JSONL first, then require run identity, checkpoint/source/tokenizer/prompt SHA linkage, metric `samples_sha256`, timing protocol/task identity, finite metric values, exact record counts, and prefix rate 1.0. Collect failures with `task_id`, stage, path, and message. Strict mode raises when any task fails; partial mode writes a visibly partial report.
|
| 924 |
+
|
| 925 |
+
- [ ] **Step 4: Publish deterministic summary files atomically**
|
| 926 |
+
|
| 927 |
+
```python
|
| 928 |
+
@dataclass(frozen=True)
|
| 929 |
+
class ConditionalAggregateReport:
|
| 930 |
+
complete: bool
|
| 931 |
+
rows: Sequence[dict[str, object]]
|
| 932 |
+
failures: Sequence[dict[str, object]]
|
| 933 |
+
unsupported: Sequence[dict[str, object]]
|
| 934 |
+
```
|
| 935 |
+
|
| 936 |
+
Sort result rows by model, dataset, and integer steps. CSV columns include conditional PPL, reference PPL, MAUVE, grouped Self-BLEU, entropy, prefix exact match, median/mean latency, checkpoint provenance, and conditioning implementation label.
|
| 937 |
+
|
| 938 |
+
Run: `pytest tests/test_conditional_aggregate.py tests/test_aggregate.py -q`
|
| 939 |
+
|
| 940 |
+
Expected: new strict/partial tests and existing unconditional aggregation tests pass.
|
| 941 |
+
|
| 942 |
+
- [ ] **Step 5: Commit aggregation**
|
| 943 |
+
|
| 944 |
+
```bash
|
| 945 |
+
git add src/dlb/conditional_aggregate.py scripts/aggregate_conditional_results.py \
|
| 946 |
+
tests/test_conditional_aggregate.py
|
| 947 |
+
git commit -m "feat: aggregate conditional benchmark results"
|
| 948 |
+
```
|
| 949 |
+
|
| 950 |
+
### Task 11: One-cell, serial, four-GPU, and smoke launchers
|
| 951 |
+
|
| 952 |
+
**Files:**
|
| 953 |
+
|
| 954 |
+
- Create: `scripts/run_conditional_one.sh`
|
| 955 |
+
- Create: `scripts/run_conditional_all.sh`
|
| 956 |
+
- Create: `scripts/run_conditional_4gpu.sh`
|
| 957 |
+
- Create: `scripts/evaluate_conditional_all.sh`
|
| 958 |
+
- Create: `scripts/evaluate_conditional_4gpu.sh`
|
| 959 |
+
- Create: `scripts/benchmark_conditional_one.sh`
|
| 960 |
+
- Create: `scripts/benchmark_conditional_all.sh`
|
| 961 |
+
- Create: `scripts/benchmark_conditional_4gpu.sh`
|
| 962 |
+
- Create: `scripts/smoke_conditional_all.sh`
|
| 963 |
+
- Create: `scripts/smoke_conditional_4gpu.sh`
|
| 964 |
+
- Modify: `src/dlb/gpu_matrix.py`
|
| 965 |
+
- Create: `tests/test_conditional_scripts.py`
|
| 966 |
+
|
| 967 |
+
**Interfaces:**
|
| 968 |
+
|
| 969 |
+
- Consumes: conditional matrix and CLIs from Tasks 1–10.
|
| 970 |
+
- Produces: safe argv-only launchers with serial/resumable and four-GPU modes plus two-prompt/two-completion smoke coverage.
|
| 971 |
+
|
| 972 |
+
- [ ] **Step 1: Write failing shell contract and dry-run routing tests**
|
| 973 |
+
|
| 974 |
+
```python
|
| 975 |
+
def test_conditional_one_exports_project_src_and_complete_protocol(tmp_path):
|
| 976 |
+
invocation = run_with_fake_conda("scripts/run_conditional_one.sh", tmp_path)
|
| 977 |
+
assert invocation.module == "dlb.runner"
|
| 978 |
+
assert invocation.option("--generation-mode") == "conditional_prefix"
|
| 979 |
+
assert invocation.option("--num-samples") == "2048"
|
| 980 |
+
assert invocation.option("--results-root").endswith("results/conditional")
|
| 981 |
+
|
| 982 |
+
|
| 983 |
+
def test_conditional_gpu_dry_run_routes_each_stage(tmp_path):
|
| 984 |
+
commands = conditional_gpu_dry_run(tmp_path, stages=("generate", "evaluate", "benchmark"))
|
| 985 |
+
assert [Path(command[0]).name for command in commands] == [
|
| 986 |
+
"run_conditional_one.sh", "python", "benchmark_conditional_one.sh"
|
| 987 |
+
]
|
| 988 |
+
assert "evaluation.conditional_evaluate" in commands[1]
|
| 989 |
+
```
|
| 990 |
+
|
| 991 |
+
- [ ] **Step 2: Run tests and verify missing scripts**
|
| 992 |
+
|
| 993 |
+
Run: `pytest tests/test_conditional_scripts.py -q`
|
| 994 |
+
|
| 995 |
+
Expected: failures because the conditional scripts do not exist.
|
| 996 |
+
|
| 997 |
+
- [ ] **Step 3: Implement safe one-cell and serial scripts**
|
| 998 |
+
|
| 999 |
+
Every script uses Bash arrays, validates integer/model/dataset arguments, quotes every path, regenerates or verifies the conditional matrix before launch, and records failures below `results/conditional/logs/`. The one-cell generator first calls runner `--validate-only`, then launches the registry-selected Conda environment with:
|
| 1000 |
+
|
| 1001 |
+
```bash
|
| 1002 |
+
runner_args=(
|
| 1003 |
+
--root "$DLB_ROOT" --model "$model" --dataset "$dataset" --steps "$steps"
|
| 1004 |
+
--num-samples 2048 --seed 42 --generation-mode conditional_prefix
|
| 1005 |
+
--conditioning-config "$DLB_ROOT/configs/conditional.yaml"
|
| 1006 |
+
--conditioning-manifest "$manifest"
|
| 1007 |
+
--results-root "$DLB_ROOT/results/conditional"
|
| 1008 |
+
)
|
| 1009 |
+
```
|
| 1010 |
+
|
| 1011 |
+
- [ ] **Step 4: Implement four-GPU routing and smoke schedule**
|
| 1012 |
+
|
| 1013 |
+
Add `--protocol conditional` to `dlb.gpu_matrix`; this selects `read_conditional_matrix` and conditional scripts while the omitted/default value keeps existing unconditional routing. Smoke launchers use a separately labeled output root, two prompt IDs, completions 0 and 1, and step counts 1 plus each category's maximum; they never publish into production paths.
|
| 1014 |
+
|
| 1015 |
+
Run: `pytest tests/test_conditional_scripts.py tests/test_gpu_matrix.py tests/test_run_one_script.py -q`
|
| 1016 |
+
|
| 1017 |
+
Expected: all conditional script and existing GPU/script tests pass.
|
| 1018 |
+
|
| 1019 |
+
- [ ] **Step 5: Commit launchers**
|
| 1020 |
+
|
| 1021 |
+
```bash
|
| 1022 |
+
git add scripts/run_conditional_one.sh scripts/run_conditional_all.sh \
|
| 1023 |
+
scripts/run_conditional_4gpu.sh scripts/evaluate_conditional_all.sh \
|
| 1024 |
+
scripts/evaluate_conditional_4gpu.sh scripts/benchmark_conditional_one.sh \
|
| 1025 |
+
scripts/benchmark_conditional_all.sh scripts/benchmark_conditional_4gpu.sh \
|
| 1026 |
+
scripts/smoke_conditional_all.sh scripts/smoke_conditional_4gpu.sh \
|
| 1027 |
+
src/dlb/gpu_matrix.py tests/test_conditional_scripts.py
|
| 1028 |
+
git commit -m "feat: add conditional benchmark launchers"
|
| 1029 |
+
```
|
| 1030 |
+
|
| 1031 |
+
### Task 12: Documentation, smoke fixtures, and full regression verification
|
| 1032 |
+
|
| 1033 |
+
**Files:**
|
| 1034 |
+
|
| 1035 |
+
- Modify: `README.md`
|
| 1036 |
+
- Modify: `docs/06_generation_and_evaluation.md`
|
| 1037 |
+
- Modify: `docs/09_four_gpu_local.md`
|
| 1038 |
+
- Create: `tests/fixtures/conditional_samples.jsonl`
|
| 1039 |
+
- Modify: tests from Tasks 1–11 only when full-suite failures expose a contract mismatch.
|
| 1040 |
+
|
| 1041 |
+
**Interfaces:**
|
| 1042 |
+
|
| 1043 |
+
- Consumes: every feature and command added in Tasks 1–11.
|
| 1044 |
+
- Produces: operator instructions, zero-shot/OOD interpretation notes, deterministic fixture coverage, and complete verification evidence.
|
| 1045 |
+
|
| 1046 |
+
- [ ] **Step 1: Add a small deterministic fixture and end-to-end offline test**
|
| 1047 |
+
|
| 1048 |
+
```python
|
| 1049 |
+
def test_conditional_offline_pipeline_from_prompts_to_partial_summary(tmp_path):
|
| 1050 |
+
build_fixture_prompts(tmp_path, prompt_count=2)
|
| 1051 |
+
run_fake_conditional_adapter(tmp_path, completions=2)
|
| 1052 |
+
evaluate_fixture_without_mauve(tmp_path, allow_partial=True)
|
| 1053 |
+
report = aggregate_conditional(tmp_path, strict=False, partial=True)
|
| 1054 |
+
assert len(report.rows) == 1
|
| 1055 |
+
assert report.rows[0]["prefix_exact_match_rate"] == 1.0
|
| 1056 |
+
```
|
| 1057 |
+
|
| 1058 |
+
Run: `pytest tests/test_conditional_prompts.py tests/test_conditional_runner.py tests/test_conditional_evaluate.py tests/test_conditional_aggregate.py -q`
|
| 1059 |
+
|
| 1060 |
+
Expected: the offline prompt → fake generation → evaluation → partial aggregate path passes.
|
| 1061 |
+
|
| 1062 |
+
- [ ] **Step 2: Document exact server workflow and interpretation boundary**
|
| 1063 |
+
|
| 1064 |
+
Document these commands in order:
|
| 1065 |
+
|
| 1066 |
+
```bash
|
| 1067 |
+
python scripts/build_conditional_prompts.py --root "$PWD" --dataset lm1b
|
| 1068 |
+
python scripts/build_conditional_prompts.py --root "$PWD" --dataset owt
|
| 1069 |
+
python scripts/verify_conditional_prompts.py --root "$PWD" --dataset all
|
| 1070 |
+
bash scripts/smoke_conditional_4gpu.sh --gpus 0,1,2,3
|
| 1071 |
+
bash scripts/run_conditional_4gpu.sh --gpus 0,1,2,3
|
| 1072 |
+
bash scripts/evaluate_conditional_4gpu.sh --gpus 0,1,2,3
|
| 1073 |
+
bash scripts/benchmark_conditional_4gpu.sh --gpus 0,1,2,3
|
| 1074 |
+
python scripts/aggregate_conditional_results.py --root "$PWD"
|
| 1075 |
+
```
|
| 1076 |
+
|
| 1077 |
+
State explicitly that FLM/FMLM/LangFlow conditioning is zero-shot/OOD projection, the comparison does not reproduce conditional fine-tuning, OWT generation uses a 960-token suffix canvas, and reported aligned quality metrics use the first 64 suffix tokens.
|
| 1078 |
+
|
| 1079 |
+
- [ ] **Step 3: Run formatting/syntax checks**
|
| 1080 |
+
|
| 1081 |
+
Run: `git diff --check`
|
| 1082 |
+
|
| 1083 |
+
Run: `python -m compileall -q src evaluation scripts`
|
| 1084 |
+
|
| 1085 |
+
Run: `for file in scripts/*conditional*.sh; do bash -n "$file"; done`
|
| 1086 |
+
|
| 1087 |
+
Expected: every command exits 0 with no output indicating an error.
|
| 1088 |
+
|
| 1089 |
+
- [ ] **Step 4: Run the complete regression suite**
|
| 1090 |
+
|
| 1091 |
+
Run: `pytest -q`
|
| 1092 |
+
|
| 1093 |
+
Expected: all existing unconditional tests and all conditional tests pass; no tests are skipped because of an accidental online dependency.
|
| 1094 |
+
|
| 1095 |
+
- [ ] **Step 5: Commit documentation and final verification assets**
|
| 1096 |
+
|
| 1097 |
+
```bash
|
| 1098 |
+
git add README.md docs/06_generation_and_evaluation.md \
|
| 1099 |
+
docs/09_four_gpu_local.md tests/fixtures/conditional_samples.jsonl
|
| 1100 |
+
git commit -m "docs: add conditional benchmark operating guide"
|
| 1101 |
+
```
|
| 1102 |
+
|
| 1103 |
+
## Final server acceptance run
|
| 1104 |
+
|
| 1105 |
+
The local implementation is complete after Task 12. On the checkpoint/data server, run these acceptance gates in order and stop on the first failure:
|
| 1106 |
+
|
| 1107 |
+
1. `python scripts/verify_data.py --root "$PWD"`
|
| 1108 |
+
2. `python scripts/verify_checkpoints.py --root "$PWD"`
|
| 1109 |
+
3. `python scripts/verify_conditional_prompts.py --root "$PWD" --dataset all`
|
| 1110 |
+
4. `bash scripts/smoke_conditional_4gpu.sh --gpus 0,1,2,3`
|
| 1111 |
+
5. Inspect smoke artifacts for all sampler families at one and maximum steps.
|
| 1112 |
+
6. `bash scripts/run_conditional_4gpu.sh --gpus 0,1,2,3`
|
| 1113 |
+
7. `bash scripts/evaluate_conditional_4gpu.sh --gpus 0,1,2,3`
|
| 1114 |
+
8. `bash scripts/benchmark_conditional_4gpu.sh --gpus 0,1,2,3`
|
| 1115 |
+
9. `python scripts/aggregate_conditional_results.py --root "$PWD"`
|
| 1116 |
+
10. Require `complete=true`, `rows=132`, `failures=0`, one RDLM/OWT unsupported record, and prefix exact-match rate `1.0` for every row.
|
results/_invalid/old_di4c_shape_skip_20260825_095622/samples_duo_di4c/steps_32/run_metadata.json
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"checkpoint_lock_id": "recipe:duo_di4c_lm1b:4efe3a3fafa3314f13963e81f828590ac771c101e228735166bd23c21e79b69e",
|
| 3 |
+
"checkpoint_selection": {
|
| 4 |
+
"output": "checkpoints/reference_reproduction/duo_di4c/lm1b",
|
| 5 |
+
"recipe": "duo_di4c_lm1b",
|
| 6 |
+
"sampling_checkpoint": "student_checkpoints/20000.ckpt",
|
| 7 |
+
"sampling_config": "config.yaml",
|
| 8 |
+
"source": "di4c",
|
| 9 |
+
"source_commit": "ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3",
|
| 10 |
+
"teacher_adapter": "uniform_to_absorbing",
|
| 11 |
+
"teacher_family": "uniform_duo"
|
| 12 |
+
},
|
| 13 |
+
"checkpoint_sha256": "60208e9645d0442c20b0d41779b41f498f36bdefa9746bee5cd0b691403f68c2",
|
| 14 |
+
"checkpoint_teacher_family": "uniform_duo",
|
| 15 |
+
"command": [
|
| 16 |
+
"/opt/anaconda3/envs/dlb-di4c/bin/python",
|
| 17 |
+
"-B",
|
| 18 |
+
"-u",
|
| 19 |
+
"/home/ruizzhao/diffusion_baseline/adapters/sample_di4c.py",
|
| 20 |
+
"--upstream-root",
|
| 21 |
+
"/home/ruizzhao/diffusion_baseline/upstreams/di4c/sdtt",
|
| 22 |
+
"--checkpoint",
|
| 23 |
+
"/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b/student_checkpoints/20000.ckpt",
|
| 24 |
+
"--config",
|
| 25 |
+
"/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b/config.yaml",
|
| 26 |
+
"--checkpoint-sha256",
|
| 27 |
+
"7104cbbd71fa2b62e95f965ced783f6368e6f3b8bbfaaee23e6ba5d47f0c9d66",
|
| 28 |
+
"--config-sha256",
|
| 29 |
+
"0626f28a7b07365b945ee55770d1cf59fd6c9ab2ee2877e54eaa075dea8a9eda",
|
| 30 |
+
"--data-config",
|
| 31 |
+
"/home/ruizzhao/diffusion_baseline/artifacts/data.yaml",
|
| 32 |
+
"--downloads-manifest",
|
| 33 |
+
"/home/ruizzhao/diffusion_baseline/data/manifests/downloads.json",
|
| 34 |
+
"--tokenizer-snapshot",
|
| 35 |
+
"/home/ruizzhao/diffusion_baseline/data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594",
|
| 36 |
+
"--output",
|
| 37 |
+
"/home/ruizzhao/diffusion_baseline/results/samples/lm1b/duo_di4c/steps_32/upstream_token_ids.json",
|
| 38 |
+
"--sample-count",
|
| 39 |
+
"1024",
|
| 40 |
+
"--batch-size",
|
| 41 |
+
"16",
|
| 42 |
+
"--num-steps",
|
| 43 |
+
"32",
|
| 44 |
+
"--seq-len",
|
| 45 |
+
"128",
|
| 46 |
+
"--seed",
|
| 47 |
+
"42",
|
| 48 |
+
"--sampler",
|
| 49 |
+
"ancestral",
|
| 50 |
+
"--teacher-family",
|
| 51 |
+
"uniform_duo",
|
| 52 |
+
"--dataset",
|
| 53 |
+
"lm1b",
|
| 54 |
+
"--offline",
|
| 55 |
+
"true",
|
| 56 |
+
"--allow-missing-embedded-config",
|
| 57 |
+
"true"
|
| 58 |
+
],
|
| 59 |
+
"command_sha256": "161b06b6ded77905ed223d323fcb3808dd9ade33c1e34fbe1479c334f6d4c8a4",
|
| 60 |
+
"config_sha256": "be62fffbe55e7a17bc51113d6dfd4cc680fad40df9e0b5dbdfef70da22081932",
|
| 61 |
+
"environment": {
|
| 62 |
+
"cuda_version": null,
|
| 63 |
+
"cuda_visible_devices": "7",
|
| 64 |
+
"hostname": "DS210029",
|
| 65 |
+
"platform": "Linux-5.15.0-186-generic-x86_64-with-glibc2.35",
|
| 66 |
+
"python": "3.10.20",
|
| 67 |
+
"torch_version": "2.5.1"
|
| 68 |
+
},
|
| 69 |
+
"finished_at": "2026-08-21T02:07:49.732328Z",
|
| 70 |
+
"identity": {
|
| 71 |
+
"adapter_identity": "dlb.adapters.di4c:v3|dlb.adapters.di4c:Di4CAdapter|5b83af37672cba7bfa98d900022b1f74a140106369c8610c88338213eaee43b4",
|
| 72 |
+
"checkpoint_lock_id": "recipe:duo_di4c_lm1b:4efe3a3fafa3314f13963e81f828590ac771c101e228735166bd23c21e79b69e",
|
| 73 |
+
"checkpoint_selection": {
|
| 74 |
+
"output": "checkpoints/reference_reproduction/duo_di4c/lm1b",
|
| 75 |
+
"recipe": "duo_di4c_lm1b",
|
| 76 |
+
"sampling_checkpoint": "student_checkpoints/20000.ckpt",
|
| 77 |
+
"sampling_config": "config.yaml",
|
| 78 |
+
"source": "di4c",
|
| 79 |
+
"source_commit": "ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3",
|
| 80 |
+
"teacher_adapter": "uniform_to_absorbing",
|
| 81 |
+
"teacher_family": "uniform_duo"
|
| 82 |
+
},
|
| 83 |
+
"checkpoint_sha256": "60208e9645d0442c20b0d41779b41f498f36bdefa9746bee5cd0b691403f68c2",
|
| 84 |
+
"checkpoint_teacher_family": "uniform_duo",
|
| 85 |
+
"command_sha256": "161b06b6ded77905ed223d323fcb3808dd9ade33c1e34fbe1479c334f6d4c8a4",
|
| 86 |
+
"config_sha256": "be62fffbe55e7a17bc51113d6dfd4cc680fad40df9e0b5dbdfef70da22081932",
|
| 87 |
+
"dataset_id": "lm1b",
|
| 88 |
+
"device": null,
|
| 89 |
+
"environment": "dlb-di4c",
|
| 90 |
+
"model_id": "duo_di4c",
|
| 91 |
+
"results_root": null,
|
| 92 |
+
"run_id": "duo_di4c-lm1b-steps-32",
|
| 93 |
+
"sample_count": 1024,
|
| 94 |
+
"seed": 42,
|
| 95 |
+
"source_sha256": "ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3",
|
| 96 |
+
"step_count": 32
|
| 97 |
+
},
|
| 98 |
+
"source_sha256": "ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3",
|
| 99 |
+
"started_at": "2026-08-21T02:07:15.248756Z",
|
| 100 |
+
"status": "succeeded",
|
| 101 |
+
"stderr_log": "/home/ruizzhao/diffusion_baseline/results/samples/lm1b/duo_di4c/steps_32/stderr.log",
|
| 102 |
+
"stdout_log": "/home/ruizzhao/diffusion_baseline/results/samples/lm1b/duo_di4c/steps_32/stdout.log"
|
| 103 |
+
}
|
results/_invalid/old_di4c_shape_skip_20260825_095622/samples_mdlm_di4c/steps_32/samples.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
results/_invalid/old_di4c_shape_skip_20260825_095622/samples_mdlm_di4c/steps_4/run_metadata.json
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"checkpoint_lock_id": "recipe:mdlm_di4c_lm1b:4efe3a3fafa3314f13963e81f828590ac771c101e228735166bd23c21e79b69e",
|
| 3 |
+
"checkpoint_selection": {
|
| 4 |
+
"output": "checkpoints/reference_reproduction/mdlm_di4c/lm1b",
|
| 5 |
+
"recipe": "mdlm_di4c_lm1b",
|
| 6 |
+
"sampling_checkpoint": "student_checkpoints/20000.ckpt",
|
| 7 |
+
"sampling_config": "config.yaml",
|
| 8 |
+
"source": "di4c",
|
| 9 |
+
"source_commit": "ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3",
|
| 10 |
+
"teacher_adapter": "masked_to_absorbing",
|
| 11 |
+
"teacher_family": "masked_mdlm"
|
| 12 |
+
},
|
| 13 |
+
"checkpoint_sha256": "d6b14edd7f897871ed6381527f604493ac680712de055a3757e4e73dccd2e777",
|
| 14 |
+
"checkpoint_teacher_family": "masked_mdlm",
|
| 15 |
+
"command": [
|
| 16 |
+
"/opt/anaconda3/envs/dlb-di4c/bin/python",
|
| 17 |
+
"-B",
|
| 18 |
+
"-u",
|
| 19 |
+
"/home/ruizzhao/diffusion_baseline/adapters/sample_di4c.py",
|
| 20 |
+
"--upstream-root",
|
| 21 |
+
"/home/ruizzhao/diffusion_baseline/upstreams/di4c/sdtt",
|
| 22 |
+
"--checkpoint",
|
| 23 |
+
"/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/mdlm_di4c/lm1b/student_checkpoints/20000.ckpt",
|
| 24 |
+
"--config",
|
| 25 |
+
"/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/mdlm_di4c/lm1b/config.yaml",
|
| 26 |
+
"--checkpoint-sha256",
|
| 27 |
+
"29ea21a518da604466ec425458155a3880ed6c03bdaaf7d00ad958a761c6c46d",
|
| 28 |
+
"--config-sha256",
|
| 29 |
+
"afaecee89f9071bb9f2761b5839927858cb24e8360c77a79b272767aad09a488",
|
| 30 |
+
"--data-config",
|
| 31 |
+
"/home/ruizzhao/diffusion_baseline/artifacts/data.yaml",
|
| 32 |
+
"--downloads-manifest",
|
| 33 |
+
"/home/ruizzhao/diffusion_baseline/data/manifests/downloads.json",
|
| 34 |
+
"--tokenizer-snapshot",
|
| 35 |
+
"/home/ruizzhao/diffusion_baseline/data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594",
|
| 36 |
+
"--output",
|
| 37 |
+
"/home/ruizzhao/diffusion_baseline/results/samples/lm1b/mdlm_di4c/steps_4/upstream_token_ids.json",
|
| 38 |
+
"--sample-count",
|
| 39 |
+
"1024",
|
| 40 |
+
"--batch-size",
|
| 41 |
+
"16",
|
| 42 |
+
"--num-steps",
|
| 43 |
+
"4",
|
| 44 |
+
"--seq-len",
|
| 45 |
+
"128",
|
| 46 |
+
"--seed",
|
| 47 |
+
"42",
|
| 48 |
+
"--sampler",
|
| 49 |
+
"ancestral",
|
| 50 |
+
"--teacher-family",
|
| 51 |
+
"masked_mdlm",
|
| 52 |
+
"--dataset",
|
| 53 |
+
"lm1b",
|
| 54 |
+
"--offline",
|
| 55 |
+
"true",
|
| 56 |
+
"--allow-missing-embedded-config",
|
| 57 |
+
"true"
|
| 58 |
+
],
|
| 59 |
+
"command_sha256": "d76b9ff0b9cff4a718629ac7eed9456bc6c97c8e687bf98b78fb25117121f21b",
|
| 60 |
+
"config_sha256": "be62fffbe55e7a17bc51113d6dfd4cc680fad40df9e0b5dbdfef70da22081932",
|
| 61 |
+
"environment": {
|
| 62 |
+
"cuda_version": null,
|
| 63 |
+
"cuda_visible_devices": "7",
|
| 64 |
+
"hostname": "DS210029",
|
| 65 |
+
"platform": "Linux-5.15.0-186-generic-x86_64-with-glibc2.35",
|
| 66 |
+
"python": "3.10.20",
|
| 67 |
+
"torch_version": "2.5.1"
|
| 68 |
+
},
|
| 69 |
+
"finished_at": "2026-08-21T02:08:57.175679Z",
|
| 70 |
+
"identity": {
|
| 71 |
+
"adapter_identity": "dlb.adapters.di4c:v3|dlb.adapters.di4c:Di4CAdapter|5b83af37672cba7bfa98d900022b1f74a140106369c8610c88338213eaee43b4",
|
| 72 |
+
"checkpoint_lock_id": "recipe:mdlm_di4c_lm1b:4efe3a3fafa3314f13963e81f828590ac771c101e228735166bd23c21e79b69e",
|
| 73 |
+
"checkpoint_selection": {
|
| 74 |
+
"output": "checkpoints/reference_reproduction/mdlm_di4c/lm1b",
|
| 75 |
+
"recipe": "mdlm_di4c_lm1b",
|
| 76 |
+
"sampling_checkpoint": "student_checkpoints/20000.ckpt",
|
| 77 |
+
"sampling_config": "config.yaml",
|
| 78 |
+
"source": "di4c",
|
| 79 |
+
"source_commit": "ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3",
|
| 80 |
+
"teacher_adapter": "masked_to_absorbing",
|
| 81 |
+
"teacher_family": "masked_mdlm"
|
| 82 |
+
},
|
| 83 |
+
"checkpoint_sha256": "d6b14edd7f897871ed6381527f604493ac680712de055a3757e4e73dccd2e777",
|
| 84 |
+
"checkpoint_teacher_family": "masked_mdlm",
|
| 85 |
+
"command_sha256": "d76b9ff0b9cff4a718629ac7eed9456bc6c97c8e687bf98b78fb25117121f21b",
|
| 86 |
+
"config_sha256": "be62fffbe55e7a17bc51113d6dfd4cc680fad40df9e0b5dbdfef70da22081932",
|
| 87 |
+
"dataset_id": "lm1b",
|
| 88 |
+
"device": null,
|
| 89 |
+
"environment": "dlb-di4c",
|
| 90 |
+
"model_id": "mdlm_di4c",
|
| 91 |
+
"results_root": null,
|
| 92 |
+
"run_id": "mdlm_di4c-lm1b-steps-4",
|
| 93 |
+
"sample_count": 1024,
|
| 94 |
+
"seed": 42,
|
| 95 |
+
"source_sha256": "ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3",
|
| 96 |
+
"step_count": 4
|
| 97 |
+
},
|
| 98 |
+
"source_sha256": "ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3",
|
| 99 |
+
"started_at": "2026-08-21T02:08:41.813188Z",
|
| 100 |
+
"status": "succeeded",
|
| 101 |
+
"stderr_log": "/home/ruizzhao/diffusion_baseline/results/samples/lm1b/mdlm_di4c/steps_4/stderr.log",
|
| 102 |
+
"stdout_log": "/home/ruizzhao/diffusion_baseline/results/samples/lm1b/mdlm_di4c/steps_4/stdout.log"
|
| 103 |
+
}
|
results/_invalid/old_di4c_shape_skip_20260825_095622/timing_mdlm_di4c/steps_4/timing.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"metadata":{"adapter_identity":"dlb.adapters.di4c:v3|dlb.adapters.di4c:Di4CAdapter|5b83af37672cba7bfa98d900022b1f74a140106369c8610c88338213eaee43b4","attempt_id":"3de92da2acbc4ab38e115eebb5e5cff5","checkpoint_lock_id":"recipe:mdlm_di4c_lm1b:4efe3a3fafa3314f13963e81f828590ac771c101e228735166bd23c21e79b69e","checkpoint_selection":{"output":"checkpoints/reference_reproduction/mdlm_di4c/lm1b","recipe":"mdlm_di4c_lm1b","sampling_checkpoint":"student_checkpoints/20000.ckpt","sampling_config":"config.yaml","source":"di4c","source_commit":"ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3","teacher_adapter":"masked_to_absorbing","teacher_family":"masked_mdlm"},"checkpoint_sha256":"d6b14edd7f897871ed6381527f604493ac680712de055a3757e4e73dccd2e777","checkpoint_teacher_family":"masked_mdlm","config_sha256":"be62fffbe55e7a17bc51113d6dfd4cc680fad40df9e0b5dbdfef70da22081932","cuda_runtime_version":"12.4","dataset":"lm1b","environment":"dlb-di4c","gpu_compute_capability":"9.0","gpu_index":0,"gpu_name":"NVIDIA H200 NVL","model":"mdlm_di4c","nvidia_driver_version":"575.57.08","parameter_precision":"fp32","precision":"bf16-mixed","precision_evidence":"static_policy_bound_to_checkpoint_and_runtime_code_not_runtime_autocast_observation","precision_policy":"di4c:pinned_bf16_mixed_config_and_internal_autocast","precision_policy_binding":{"adapter_identity":"dlb.adapters.di4c:v3|dlb.adapters.di4c:Di4CAdapter|5b83af37672cba7bfa98d900022b1f74a140106369c8610c88338213eaee43b4","checkpoint_lock_id":"recipe:mdlm_di4c_lm1b:4efe3a3fafa3314f13963e81f828590ac771c101e228735166bd23c21e79b69e","checkpoint_selection":{"output":"checkpoints/reference_reproduction/mdlm_di4c/lm1b","recipe":"mdlm_di4c_lm1b","sampling_checkpoint":"student_checkpoints/20000.ckpt","sampling_config":"config.yaml","source":"di4c","source_commit":"ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3","teacher_adapter":"masked_to_absorbing","teacher_family":"masked_mdlm"},"checkpoint_sha256":"d6b14edd7f897871ed6381527f604493ac680712de055a3757e4e73dccd2e777","dataset":"lm1b","experiment_config_sha256":"be62fffbe55e7a17bc51113d6dfd4cc680fad40df9e0b5dbdfef70da22081932","model":"mdlm_di4c","source_commit":"ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3","upstream":"di4c"},"pytorch_compiled_cuda_toolkit":"12.4","requested_precision":"author","runtime_code_binding":[{"kind":"sampling_callable","path":"/home/ruizzhao/diffusion_baseline/adapters/_distilled_runtime.py","sha256":"a81d2f1a09bc895c9775f5b583ec4e4b8245d6e7b518728e163a03b872554992","symbol":"benchmark_model.<locals>.<lambda>"},{"kind":"loaded_module","path":"/home/ruizzhao/diffusion_baseline/upstreams/di4c/sdtt/src/sdtt/core/diffusion/noise_schedule.py","sha256":"b2c06e11a4947b306db1c19eac181b99bbc6b36520526ebb36639696c95dcb21","symbol":"LogLinearNoise"},{"kind":"loaded_module","path":"/home/ruizzhao/diffusion_baseline/upstreams/di4c/sdtt/src/sdtt/core/distill/multi_round_sdtt.py","sha256":"75c61fbd1d3e1b87e10b25d67c0143ba0995753f579e0282bc1e3559c145704c","symbol":"MultiRoundSDTT"},{"kind":"loaded_module","path":"/home/ruizzhao/diffusion_baseline/upstreams/di4c/sdtt/src/sdtt/models/dit_orig.py","sha256":"c2e7ced4ccbb92bcf9e6fdc0fc2b41130447d8d15e5d696e7f3a927177578912","symbol":"DDiTBlock"},{"kind":"loaded_module","path":"/home/ruizzhao/diffusion_baseline/upstreams/di4c/sdtt/src/sdtt/models/dit_orig.py","sha256":"c2e7ced4ccbb92bcf9e6fdc0fc2b41130447d8d15e5d696e7f3a927177578912","symbol":"DDitFinalLayer"},{"kind":"loaded_module","path":"/home/ruizzhao/diffusion_baseline/upstreams/di4c/sdtt/src/sdtt/models/dit_orig.py","sha256":"c2e7ced4ccbb92bcf9e6fdc0fc2b41130447d8d15e5d696e7f3a927177578912","symbol":"DIT"},{"kind":"loaded_module","path":"/home/ruizzhao/diffusion_baseline/upstreams/di4c/sdtt/src/sdtt/models/dit_orig.py","sha256":"c2e7ced4ccbb92bcf9e6fdc0fc2b41130447d8d15e5d696e7f3a927177578912","symbol":"EmbeddingLayer"},{"kind":"loaded_module","path":"/home/ruizzhao/diffusion_baseline/upstreams/di4c/sdtt/src/sdtt/models/dit_orig.py","sha256":"c2e7ced4ccbb92bcf9e6fdc0fc2b41130447d8d15e5d696e7f3a927177578912","symbol":"LayerNorm"},{"kind":"loaded_module","path":"/home/ruizzhao/diffusion_baseline/upstreams/di4c/sdtt/src/sdtt/models/dit_orig.py","sha256":"c2e7ced4ccbb92bcf9e6fdc0fc2b41130447d8d15e5d696e7f3a927177578912","symbol":"Rotary"},{"kind":"loaded_module","path":"/home/ruizzhao/diffusion_baseline/upstreams/di4c/sdtt/src/sdtt/models/dit_orig.py","sha256":"c2e7ced4ccbb92bcf9e6fdc0fc2b41130447d8d15e5d696e7f3a927177578912","symbol":"TimestepEmbedder"},{"kind":"loaded_module","path":"/opt/anaconda3/envs/dlb-di4c/lib/python3.10/site-packages/torch/nn/modules/activation.py","sha256":"1e748e0df0d70491fe82f0178f32126ee6ee138a53de7fc76463af10c75f5477","symbol":"GELU"},{"kind":"loaded_module","path":"/opt/anaconda3/envs/dlb-di4c/lib/python3.10/site-packages/torch/nn/modules/activation.py","sha256":"1e748e0df0d70491fe82f0178f32126ee6ee138a53de7fc76463af10c75f5477","symbol":"SiLU"},{"kind":"loaded_module","path":"/opt/anaconda3/envs/dlb-di4c/lib/python3.10/site-packages/torch/nn/modules/container.py","sha256":"33af8d1fe4fb542cbba5542c5079b1bd8ef8a6493abbef329411f71b2071d24f","symbol":"ModuleList"},{"kind":"loaded_module","path":"/opt/anaconda3/envs/dlb-di4c/lib/python3.10/site-packages/torch/nn/modules/container.py","sha256":"33af8d1fe4fb542cbba5542c5079b1bd8ef8a6493abbef329411f71b2071d24f","symbol":"Sequential"},{"kind":"loaded_module","path":"/opt/anaconda3/envs/dlb-di4c/lib/python3.10/site-packages/torch/nn/modules/dropout.py","sha256":"1fc9974c3cf2078dd4bc257f73afb6a6714adfec2e9d099d47cf923e766d3490","symbol":"Dropout"},{"kind":"loaded_module","path":"/opt/anaconda3/envs/dlb-di4c/lib/python3.10/site-packages/torch/nn/modules/linear.py","sha256":"d7f5cadf773deb5ab4bcc9f57414e8239b2859c70d2aa6f141576de1ec918823","symbol":"Linear"}],"seed":42,"source_commit":"ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3","steps":4,"synchronization_policy":"torch.cuda.synchronize_before_start_and_after_generate"},"schema":"dlb-generation-timing-v1","timing":{"batch_size":1,"exclusions":["model_and_checkpoint_loading","first_compilation","token_decoding","metrics","file_io"],"median_seconds":0.028812396456487477,"mode":"primary_latency","num_timed_samples":32,"raw_durations_seconds":[0.028952325927093625,0.02889296505600214,0.02881482499651611,0.028835597913712263,0.028863610001280904,0.028826443012803793,0.028941920027136803,0.028878462966531515,0.028866074047982693,0.028866023989394307,0.028870980953797698,0.028861936880275607,0.028891792986541986,0.028822577092796564,0.028789767064154148,0.028745889198035002,0.028809967916458845,0.02874415717087686,0.028673017164692283,0.02875470300205052,0.028752220096066594,0.030406136997044086,0.02901422092691064,0.028689061990007758,0.028658536029979587,0.02869494096376002,0.028659117175266147,0.028743675909936428,0.028751909034326673,0.02867855690419674,0.0286307530477643,0.02864584606140852],"repeats":32,"seconds_per_sample":0.028844625390775036,"standard_deviation_convention":"population","standard_deviation_seconds":0.00029661149222713547,"warmups":5}}
|
results/conditional/samples/lm1b/duo_di4c/steps_4/upstream_token_ids.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
results/conditional/samples/lm1b/langflow/steps_32/samples.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
results/conditional/samples/lm1b/mdlm_di4c/steps_1/upstream_token_ids.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
results/conditional/samples/lm1b/mdlm_di4c/steps_2/upstream_token_ids.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
results/conditional/samples/lm1b/mdlm_di4c/steps_4/samples.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
results/conditional/samples/lm1b/mdlm_sdtt/steps_16/samples.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
results/elf-gpu01/logs/20260913T065838-1485984/owt-prefix__c64-text-t5__steps_1__seed_42__compile_0__generate.log
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
["/opt/anaconda3/envs/dlb-elf/bin/python", "-B", "/home/ruizzhao/diffusion_baseline/scripts/run_elf.py", "generate", "--root", "/home/ruizzhao/diffusion_baseline", "--task", "owt-prefix", "--steps", "1", "--seed", "42", "--batch-size", "8", "--num-samples", "1024", "--output", "/home/ruizzhao/diffusion_baseline/results/elf-gpu01/quality/owt-prefix/c64-text-t5/steps_1/seed_42/compile_0", "--input", "/home/ruizzhao/diffusion_baseline/data/elf/owt-prefix.jsonl", "--diversity"]
|
| 2 |
+
Checking 1024 conditions before loading ELF weights: /home/ruizzhao/diffusion_baseline/data/elf/owt-prefix.jsonl
|
| 3 |
+
ELF generate: task=owt-prefix, steps=1, split=unconditional, output=/home/ruizzhao/diffusion_baseline/results/elf-gpu01/quality/owt-prefix/c64-text-t5/steps_1/seed_42/compile_0
|
| 4 |
+
8/2048 samples
|
| 5 |
+
16/2048 samples
|
| 6 |
+
24/2048 samples
|
| 7 |
+
32/2048 samples
|
| 8 |
+
40/2048 samples
|
| 9 |
+
48/2048 samples
|
| 10 |
+
56/2048 samples
|
| 11 |
+
64/2048 samples
|
| 12 |
+
72/2048 samples
|
| 13 |
+
80/2048 samples
|
| 14 |
+
88/2048 samples
|
| 15 |
+
96/2048 samples
|
| 16 |
+
104/2048 samples
|
| 17 |
+
112/2048 samples
|
| 18 |
+
120/2048 samples
|
| 19 |
+
128/2048 samples
|
| 20 |
+
136/2048 samples
|
| 21 |
+
144/2048 samples
|
| 22 |
+
152/2048 samples
|
| 23 |
+
160/2048 samples
|
| 24 |
+
168/2048 samples
|
| 25 |
+
176/2048 samples
|
| 26 |
+
184/2048 samples
|
| 27 |
+
192/2048 samples
|
| 28 |
+
200/2048 samples
|
| 29 |
+
208/2048 samples
|
| 30 |
+
216/2048 samples
|
| 31 |
+
224/2048 samples
|
| 32 |
+
232/2048 samples
|
| 33 |
+
240/2048 samples
|
| 34 |
+
248/2048 samples
|
| 35 |
+
256/2048 samples
|
| 36 |
+
264/2048 samples
|
| 37 |
+
272/2048 samples
|
| 38 |
+
280/2048 samples
|
| 39 |
+
288/2048 samples
|
| 40 |
+
296/2048 samples
|
| 41 |
+
304/2048 samples
|
| 42 |
+
312/2048 samples
|
| 43 |
+
320/2048 samples
|
| 44 |
+
328/2048 samples
|
| 45 |
+
336/2048 samples
|
| 46 |
+
344/2048 samples
|
| 47 |
+
352/2048 samples
|
| 48 |
+
360/2048 samples
|
| 49 |
+
368/2048 samples
|
| 50 |
+
376/2048 samples
|
| 51 |
+
384/2048 samples
|
| 52 |
+
392/2048 samples
|
| 53 |
+
400/2048 samples
|
| 54 |
+
408/2048 samples
|
| 55 |
+
416/2048 samples
|
| 56 |
+
424/2048 samples
|
| 57 |
+
432/2048 samples
|
| 58 |
+
440/2048 samples
|
| 59 |
+
448/2048 samples
|
| 60 |
+
456/2048 samples
|
| 61 |
+
464/2048 samples
|
| 62 |
+
472/2048 samples
|
| 63 |
+
480/2048 samples
|
| 64 |
+
488/2048 samples
|
| 65 |
+
496/2048 samples
|
| 66 |
+
504/2048 samples
|
| 67 |
+
512/2048 samples
|
| 68 |
+
520/2048 samples
|
| 69 |
+
528/2048 samples
|
| 70 |
+
536/2048 samples
|
| 71 |
+
544/2048 samples
|
| 72 |
+
552/2048 samples
|
| 73 |
+
560/2048 samples
|
| 74 |
+
568/2048 samples
|
| 75 |
+
576/2048 samples
|
| 76 |
+
584/2048 samples
|
| 77 |
+
592/2048 samples
|
| 78 |
+
600/2048 samples
|
| 79 |
+
608/2048 samples
|
| 80 |
+
616/2048 samples
|
| 81 |
+
624/2048 samples
|
| 82 |
+
632/2048 samples
|
| 83 |
+
640/2048 samples
|
| 84 |
+
648/2048 samples
|
| 85 |
+
656/2048 samples
|
| 86 |
+
664/2048 samples
|
| 87 |
+
672/2048 samples
|
| 88 |
+
680/2048 samples
|
| 89 |
+
688/2048 samples
|
| 90 |
+
696/2048 samples
|
| 91 |
+
704/2048 samples
|
| 92 |
+
712/2048 samples
|
| 93 |
+
720/2048 samples
|
| 94 |
+
728/2048 samples
|
| 95 |
+
736/2048 samples
|
| 96 |
+
744/2048 samples
|
| 97 |
+
752/2048 samples
|
| 98 |
+
760/2048 samples
|
| 99 |
+
768/2048 samples
|
| 100 |
+
776/2048 samples
|
| 101 |
+
784/2048 samples
|
| 102 |
+
792/2048 samples
|
| 103 |
+
800/2048 samples
|
| 104 |
+
808/2048 samples
|
| 105 |
+
816/2048 samples
|
| 106 |
+
824/2048 samples
|
| 107 |
+
832/2048 samples
|
| 108 |
+
840/2048 samples
|
| 109 |
+
848/2048 samples
|
| 110 |
+
856/2048 samples
|
| 111 |
+
864/2048 samples
|
| 112 |
+
872/2048 samples
|
| 113 |
+
880/2048 samples
|
| 114 |
+
888/2048 samples
|
| 115 |
+
896/2048 samples
|
| 116 |
+
904/2048 samples
|
| 117 |
+
912/2048 samples
|
| 118 |
+
920/2048 samples
|
| 119 |
+
928/2048 samples
|
| 120 |
+
936/2048 samples
|
| 121 |
+
944/2048 samples
|
| 122 |
+
952/2048 samples
|
| 123 |
+
960/2048 samples
|
| 124 |
+
968/2048 samples
|
| 125 |
+
976/2048 samples
|
| 126 |
+
984/2048 samples
|
| 127 |
+
992/2048 samples
|
| 128 |
+
1000/2048 samples
|
| 129 |
+
1008/2048 samples
|
| 130 |
+
1016/2048 samples
|
| 131 |
+
1024/2048 samples
|
| 132 |
+
1032/2048 samples
|
| 133 |
+
1040/2048 samples
|
| 134 |
+
1048/2048 samples
|
| 135 |
+
1056/2048 samples
|
| 136 |
+
1064/2048 samples
|
| 137 |
+
1072/2048 samples
|
| 138 |
+
1080/2048 samples
|
| 139 |
+
1088/2048 samples
|
| 140 |
+
1096/2048 samples
|
| 141 |
+
1104/2048 samples
|
| 142 |
+
1112/2048 samples
|
| 143 |
+
1120/2048 samples
|
| 144 |
+
1128/2048 samples
|
| 145 |
+
1136/2048 samples
|
| 146 |
+
1144/2048 samples
|
| 147 |
+
1152/2048 samples
|
| 148 |
+
1160/2048 samples
|
| 149 |
+
1168/2048 samples
|
| 150 |
+
1176/2048 samples
|
| 151 |
+
1184/2048 samples
|
| 152 |
+
1192/2048 samples
|
| 153 |
+
1200/2048 samples
|
| 154 |
+
1208/2048 samples
|
| 155 |
+
1216/2048 samples
|
| 156 |
+
1224/2048 samples
|
| 157 |
+
1232/2048 samples
|
| 158 |
+
1240/2048 samples
|
| 159 |
+
1248/2048 samples
|
| 160 |
+
1256/2048 samples
|
| 161 |
+
1264/2048 samples
|
| 162 |
+
1272/2048 samples
|
| 163 |
+
1280/2048 samples
|
| 164 |
+
1288/2048 samples
|
| 165 |
+
1296/2048 samples
|
| 166 |
+
1304/2048 samples
|
| 167 |
+
1312/2048 samples
|
| 168 |
+
1320/2048 samples
|
| 169 |
+
1328/2048 samples
|
| 170 |
+
1336/2048 samples
|
| 171 |
+
1344/2048 samples
|
| 172 |
+
1352/2048 samples
|
| 173 |
+
1360/2048 samples
|
| 174 |
+
1368/2048 samples
|
| 175 |
+
1376/2048 samples
|
| 176 |
+
1384/2048 samples
|
| 177 |
+
1392/2048 samples
|
| 178 |
+
1400/2048 samples
|
| 179 |
+
1408/2048 samples
|
| 180 |
+
1416/2048 samples
|
| 181 |
+
1424/2048 samples
|
| 182 |
+
1432/2048 samples
|
| 183 |
+
1440/2048 samples
|
| 184 |
+
1448/2048 samples
|
| 185 |
+
1456/2048 samples
|
| 186 |
+
1464/2048 samples
|
| 187 |
+
1472/2048 samples
|
| 188 |
+
1480/2048 samples
|
| 189 |
+
1488/2048 samples
|
| 190 |
+
1496/2048 samples
|
| 191 |
+
1504/2048 samples
|
| 192 |
+
1512/2048 samples
|
| 193 |
+
1520/2048 samples
|
| 194 |
+
1528/2048 samples
|
| 195 |
+
1536/2048 samples
|
| 196 |
+
1544/2048 samples
|
| 197 |
+
1552/2048 samples
|
| 198 |
+
1560/2048 samples
|
| 199 |
+
1568/2048 samples
|
| 200 |
+
1576/2048 samples
|
| 201 |
+
1584/2048 samples
|
| 202 |
+
1592/2048 samples
|
| 203 |
+
1600/2048 samples
|
| 204 |
+
1608/2048 samples
|
| 205 |
+
1616/2048 samples
|
| 206 |
+
1624/2048 samples
|
| 207 |
+
1632/2048 samples
|
| 208 |
+
1640/2048 samples
|
| 209 |
+
1648/2048 samples
|
| 210 |
+
1656/2048 samples
|
| 211 |
+
1664/2048 samples
|
| 212 |
+
1672/2048 samples
|
| 213 |
+
1680/2048 samples
|
| 214 |
+
1688/2048 samples
|
| 215 |
+
1696/2048 samples
|
| 216 |
+
1704/2048 samples
|
| 217 |
+
1712/2048 samples
|
| 218 |
+
1720/2048 samples
|
| 219 |
+
1728/2048 samples
|
| 220 |
+
1736/2048 samples
|
| 221 |
+
1744/2048 samples
|
| 222 |
+
1752/2048 samples
|
| 223 |
+
1760/2048 samples
|
| 224 |
+
1768/2048 samples
|
| 225 |
+
1776/2048 samples
|
| 226 |
+
1784/2048 samples
|
| 227 |
+
1792/2048 samples
|
| 228 |
+
1800/2048 samples
|
| 229 |
+
1808/2048 samples
|
| 230 |
+
1816/2048 samples
|
| 231 |
+
1824/2048 samples
|
| 232 |
+
1832/2048 samples
|
| 233 |
+
1840/2048 samples
|
| 234 |
+
1848/2048 samples
|
| 235 |
+
1856/2048 samples
|
| 236 |
+
1864/2048 samples
|
| 237 |
+
1872/2048 samples
|
| 238 |
+
1880/2048 samples
|
| 239 |
+
1888/2048 samples
|
| 240 |
+
1896/2048 samples
|
| 241 |
+
1904/2048 samples
|
| 242 |
+
1912/2048 samples
|
| 243 |
+
1920/2048 samples
|
| 244 |
+
1928/2048 samples
|
| 245 |
+
1936/2048 samples
|
| 246 |
+
1944/2048 samples
|
| 247 |
+
1952/2048 samples
|
| 248 |
+
1960/2048 samples
|
| 249 |
+
1968/2048 samples
|
| 250 |
+
1976/2048 samples
|
| 251 |
+
1984/2048 samples
|
| 252 |
+
1992/2048 samples
|
| 253 |
+
2000/2048 samples
|
| 254 |
+
2008/2048 samples
|
| 255 |
+
2016/2048 samples
|
| 256 |
+
2024/2048 samples
|
| 257 |
+
2032/2048 samples
|
| 258 |
+
2040/2048 samples
|
| 259 |
+
2048/2048 samples
|
results/elf-gpu01/quality/owt-prefix/c64-text-t5/steps_1/seed_42/compile_0/request.json
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema": "dlb-elf-v1",
|
| 3 |
+
"task": "owt-prefix",
|
| 4 |
+
"model": "ELF-B",
|
| 5 |
+
"source_commit": "b29d8833609e9ab7f67cd9da39435ac5cea04837",
|
| 6 |
+
"checkpoint_task": "owt",
|
| 7 |
+
"checkpoint_selection": "ema_params1",
|
| 8 |
+
"checkpoint_sha256": "52c21bfb7f920e785e0e780645c411c94e4541af4cee8f85b6f555f826d4c283",
|
| 9 |
+
"assets_manifest_sha256": "213c6827168523e800868db2d2ee2f3c4ed3d030fdbd15fc502c907e429bee0a",
|
| 10 |
+
"source_config_sha256": "6bbe69ad55e5d890affe61d5b89ae03ff4087c8933cdc36832ffeaa5e16375b5",
|
| 11 |
+
"runner_sha256": "19dda543105c787524473128fbac2bc8fd7efd58fa95fa4f1954b9155ebb8a65",
|
| 12 |
+
"helper_sha256": "ff79c244d7b893bb7353677c4bd7e206f6a486193b2bbfd150282dbee95d307c",
|
| 13 |
+
"lock_sha256": "b801cd97165cfd78a9f41d6dcbaa4bbf51c60df794237e8a981f8b1520b896a5",
|
| 14 |
+
"steps": 1,
|
| 15 |
+
"sampling": {
|
| 16 |
+
"sampling_method": "sde",
|
| 17 |
+
"cfg": 1.0,
|
| 18 |
+
"sc_cfg": 3.0,
|
| 19 |
+
"gamma": 2.0
|
| 20 |
+
},
|
| 21 |
+
"seed": 42,
|
| 22 |
+
"batch_size": 8,
|
| 23 |
+
"precision": "fp32 parameters/latents; official bf16 forward autocast; fp32 output heads",
|
| 24 |
+
"matmul_precision": "high",
|
| 25 |
+
"compile": false,
|
| 26 |
+
"model_canvas_t5_tokens": 1024,
|
| 27 |
+
"input": {
|
| 28 |
+
"schema": "elf-input-v1",
|
| 29 |
+
"count": 1024,
|
| 30 |
+
"sha256": "3df4bc0c58454494944c86e374388f83ef3a4c2168a6c06f6b70a217cc8e3958",
|
| 31 |
+
"task": "owt-prefix",
|
| 32 |
+
"protocol": "c64_text_t5_v1",
|
| 33 |
+
"source_sha256": "68907c7d4eb5104dbaea6d136a951635a11eb46c0f3e8f417d5f406e1ca834fc"
|
| 34 |
+
},
|
| 35 |
+
"protocol": "c64_text_t5_v1",
|
| 36 |
+
"conditioning": "zero_shot_ood_native_projection",
|
| 37 |
+
"condition_tokenization_policy": "c64_raw_no_special_tokens_v1",
|
| 38 |
+
"elf_forward_calls_per_sample": 2,
|
| 39 |
+
"condition_encoder_calls_per_sample": 1,
|
| 40 |
+
"untimed_preflight": {
|
| 41 |
+
"calls": 2,
|
| 42 |
+
"prefix_checked_calls": 2
|
| 43 |
+
},
|
| 44 |
+
"gpu_name": "NVIDIA H200 NVL",
|
| 45 |
+
"gpu_index": 0,
|
| 46 |
+
"gpu_compute_capability": "9.0",
|
| 47 |
+
"cuda_runtime_version": "12.4",
|
| 48 |
+
"pytorch_compiled_cuda_toolkit": "12.4",
|
| 49 |
+
"nvidia_driver_version": "575.57.08",
|
| 50 |
+
"synchronization_policy": "torch.cuda.synchronize_before_start_and_after_generate"
|
| 51 |
+
}
|
results/elf-gpu01/quality/owt/unconditional/steps_1024/seed_42/compile_0/samples.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
results/elf-gpu01/quality/wmt14/validation/steps_2/seed_42/compile_0/samples.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
results/elf-gpu01/quality/wmt14/validation/steps_4/seed_42/compile_0/samples.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
results/elf-gpu01/quality/xsum/validation/steps_4/seed_42/compile_0/request.json
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema": "dlb-elf-v1",
|
| 3 |
+
"task": "xsum",
|
| 4 |
+
"model": "ELF-B",
|
| 5 |
+
"source_commit": "b29d8833609e9ab7f67cd9da39435ac5cea04837",
|
| 6 |
+
"checkpoint_task": "xsum",
|
| 7 |
+
"checkpoint_selection": "ema_params1",
|
| 8 |
+
"checkpoint_sha256": "fb2cb9a2aba0328d0de30d53800c0976a019303b9a31f16051981982ed8795ae",
|
| 9 |
+
"assets_manifest_sha256": "213c6827168523e800868db2d2ee2f3c4ed3d030fdbd15fc502c907e429bee0a",
|
| 10 |
+
"source_config_sha256": "8ee817a563626df6f98bb96b91df378d3c459cc002923a5b4a28b3b17bf41ca5",
|
| 11 |
+
"runner_sha256": "19dda543105c787524473128fbac2bc8fd7efd58fa95fa4f1954b9155ebb8a65",
|
| 12 |
+
"helper_sha256": "ff79c244d7b893bb7353677c4bd7e206f6a486193b2bbfd150282dbee95d307c",
|
| 13 |
+
"lock_sha256": "b801cd97165cfd78a9f41d6dcbaa4bbf51c60df794237e8a981f8b1520b896a5",
|
| 14 |
+
"steps": 4,
|
| 15 |
+
"sampling": {
|
| 16 |
+
"sampling_method": "ode",
|
| 17 |
+
"cfg": 2.0,
|
| 18 |
+
"sc_cfg": 1.0,
|
| 19 |
+
"gamma": 0.0
|
| 20 |
+
},
|
| 21 |
+
"seed": 42,
|
| 22 |
+
"batch_size": 8,
|
| 23 |
+
"precision": "fp32 parameters/latents; official bf16 forward autocast; fp32 output heads",
|
| 24 |
+
"matmul_precision": "high",
|
| 25 |
+
"compile": false,
|
| 26 |
+
"model_canvas_t5_tokens": 1088,
|
| 27 |
+
"input": {
|
| 28 |
+
"schema": "elf-input-v1",
|
| 29 |
+
"count": 11332,
|
| 30 |
+
"sha256": "98d21431c1df775e0f5585f509fa5d412b64e43b7a04db08e8f458c301c110ed",
|
| 31 |
+
"task": "xsum",
|
| 32 |
+
"split": "validation",
|
| 33 |
+
"asset": {
|
| 34 |
+
"repo_id": "EdinburghNLP/xsum",
|
| 35 |
+
"revision": "7d4d486c2f8ef850b1a11aead99b894ff3dd7da9",
|
| 36 |
+
"repo_type": "dataset",
|
| 37 |
+
"patterns": [
|
| 38 |
+
"data/validation-*.parquet",
|
| 39 |
+
"data/test-*.parquet"
|
| 40 |
+
]
|
| 41 |
+
},
|
| 42 |
+
"reader": "pyarrow_parquet_v1"
|
| 43 |
+
},
|
| 44 |
+
"protocol": "elf_native_v1",
|
| 45 |
+
"conditioning": "task_trained",
|
| 46 |
+
"condition_tokenization_policy": "released_ids_or_raw_plus_eos_then_source_cap_v2",
|
| 47 |
+
"elf_forward_calls_per_sample": 9,
|
| 48 |
+
"condition_encoder_calls_per_sample": 1,
|
| 49 |
+
"untimed_preflight": {
|
| 50 |
+
"calls": 9,
|
| 51 |
+
"prefix_checked_calls": 9
|
| 52 |
+
},
|
| 53 |
+
"gpu_name": "NVIDIA H200 NVL",
|
| 54 |
+
"gpu_index": 0,
|
| 55 |
+
"gpu_compute_capability": "9.0",
|
| 56 |
+
"cuda_runtime_version": "12.4",
|
| 57 |
+
"pytorch_compiled_cuda_toolkit": "12.4",
|
| 58 |
+
"nvidia_driver_version": "575.57.08",
|
| 59 |
+
"synchronization_policy": "torch.cuda.synchronize_before_start_and_after_generate"
|
| 60 |
+
}
|
results/elf-h200-gpu2/logs/smoke_xsum.log
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2/2 samples
|
| 2 |
+
2/2 samples
|
results/elf-h200-gpu2/quality/wmt14/validation/steps_16/seed_42/compile_0/samples.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
results/elf-owt-baseline-eval-owt-prefix/steps_32/metrics.json
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"protocol": "elf_owt_public_baseline_eval_v1",
|
| 3 |
+
"task": "owt-prefix",
|
| 4 |
+
"steps": 32,
|
| 5 |
+
"source_run": "/home/ruizzhao/diffusion_baseline/results/elf-gpu01/quality/owt-prefix/c64-text-t5/steps_32/seed_42/compile_0",
|
| 6 |
+
"samples_sha256": "d0d9f5833ccf8a9c2e38e3707ba118e4a93f8b2646920cb1fde7e91b8132ff9e",
|
| 7 |
+
"generation_manifest_sha256": "555290477f6dcf6a67cc5343193352be2ea5c18594786411b26bd0517212dc5d",
|
| 8 |
+
"assets_manifest_sha256": "213c6827168523e800868db2d2ee2f3c4ed3d030fdbd15fc502c907e429bee0a",
|
| 9 |
+
"evaluator": {
|
| 10 |
+
"evaluate_elf_owt_baseline.py": "14543ff030dcc81b94327737f61d0160f7cf56f2805a36c0c02708e4afe6268d",
|
| 11 |
+
"generative_perplexity.py": "d2f0a6b7513dd8267a6af63ca269dbf80f2bf449e9074ee3bd9b3c8340d30377",
|
| 12 |
+
"conditional_perplexity.py": "8344d2b60221c95ed979ea9ddc241a5430b2f41b27ab6e7d9d3aabcd5eefbb30",
|
| 13 |
+
"conditional_evaluate.py": "91db381ab153f7f8079d6e6048d7b05cc19b755f3e8a39284d63694147a74698",
|
| 14 |
+
"unigram_entropy.py": "22bc98226a73ff232f1bd159d2a5efd851689f54a715ba037c9bc1e2c2d90a70",
|
| 15 |
+
"self_bleu.py": "60c0b3d52034ca67d9cf517871dc804bf16d6b9f03e6a3fc5bb3b164f726b4d9"
|
| 16 |
+
},
|
| 17 |
+
"batch_size": 8,
|
| 18 |
+
"prompts_sha256": "68907c7d4eb5104dbaea6d136a951635a11eb46c0f3e8f417d5f406e1ca834fc",
|
| 19 |
+
"sample_count": 2048,
|
| 20 |
+
"prompt_count": 1024,
|
| 21 |
+
"metrics": {
|
| 22 |
+
"entropy_gpt2_nats": 3.725415810106105,
|
| 23 |
+
"conditional_ppl": {
|
| 24 |
+
"perplexity": 60.56163680457397,
|
| 25 |
+
"total_nll": 538236.2602539062,
|
| 26 |
+
"valid_token_count": 131160,
|
| 27 |
+
"sample_count": 2048,
|
| 28 |
+
"batch_size": 8,
|
| 29 |
+
"context_length": 1024,
|
| 30 |
+
"truncation": "right",
|
| 31 |
+
"model_id": "gpt2-large",
|
| 32 |
+
"model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
|
| 33 |
+
"tokenizer_id": "gpt2",
|
| 34 |
+
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
|
| 35 |
+
"aggregation": "sum_nll_over_sum_valid_next_tokens"
|
| 36 |
+
},
|
| 37 |
+
"grouped_self_bleu": {
|
| 38 |
+
"score": 0.03318342181743137,
|
| 39 |
+
"prompt_count": 256,
|
| 40 |
+
"completions_per_prompt": 5,
|
| 41 |
+
"sample_count": 1280,
|
| 42 |
+
"tokenization": "gpt2",
|
| 43 |
+
"ngram_order": 4,
|
| 44 |
+
"weights": [
|
| 45 |
+
0.25,
|
| 46 |
+
0.25,
|
| 47 |
+
0.25,
|
| 48 |
+
0.25
|
| 49 |
+
],
|
| 50 |
+
"smoothing": "chen_cherry_method1",
|
| 51 |
+
"aggregation": "mean_prompt_group_self_bleu",
|
| 52 |
+
"slice": "generated_source_tokens_64_128_only"
|
| 53 |
+
}
|
| 54 |
+
},
|
| 55 |
+
"valid": true,
|
| 56 |
+
"length_diagnostics": {
|
| 57 |
+
"mean": 868.18603515625,
|
| 58 |
+
"min": 631,
|
| 59 |
+
"max": 985,
|
| 60 |
+
"below_64_count": 0,
|
| 61 |
+
"above_1024_count": 0
|
| 62 |
+
},
|
| 63 |
+
"short_sample_ids": [],
|
| 64 |
+
"native_tokenizer": "t5-small",
|
| 65 |
+
"scoring_tokenizer": "gpt2",
|
| 66 |
+
"scoring_limit": 64,
|
| 67 |
+
"completion_scope": "all_saved_completions",
|
| 68 |
+
"output_policy": "released ELF EOS-trimmed decoded text; GPT-2 retokenized; no fabricated padding",
|
| 69 |
+
"ppl_policy": "existing compute_conditional_gen_ppl; canonical prefix IDs; decode joined IDs then retokenize; prompt loss excluded"
|
| 70 |
+
}
|
results/elf-timing-gpu0-20260913T083438-1652790/environment/owt-prefix_validation_s32-after.txt
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Sun Sep 13 16:43:31 2026
|
| 2 |
+
+-----------------------------------------------------------------------------------------+
|
| 3 |
+
| NVIDIA-SMI 575.57.08 Driver Version: 575.57.08 CUDA Version: 12.9 |
|
| 4 |
+
|-----------------------------------------+------------------------+----------------------+
|
| 5 |
+
| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
|
| 6 |
+
| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
|
| 7 |
+
| | | MIG M. |
|
| 8 |
+
|=========================================+========================+======================|
|
| 9 |
+
| 0 NVIDIA H200 NVL Off | 00000000:06:00.0 Off | 0 |
|
| 10 |
+
| N/A 36C P0 101W / 600W | 0MiB / 143771MiB | 0% Default |
|
| 11 |
+
| | | Disabled |
|
| 12 |
+
+-----------------------------------------+------------------------+----------------------+
|
| 13 |
+
|
| 14 |
+
+-----------------------------------------------------------------------------------------+
|
| 15 |
+
| Processes: |
|
| 16 |
+
| GPU GI CI PID Type Process name GPU Memory |
|
| 17 |
+
| ID ID Usage |
|
| 18 |
+
|=========================================================================================|
|
| 19 |
+
| No running processes found |
|
| 20 |
+
+-----------------------------------------------------------------------------------------+
|
results/elf-timing-gpu1-20260915T091449-470600/environment/owt-prefix_validation_s1024-after.txt
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Tue Sep 15 17:29:09 2026
|
| 2 |
+
+-----------------------------------------------------------------------------------------+
|
| 3 |
+
| NVIDIA-SMI 575.57.08 Driver Version: 575.57.08 CUDA Version: 12.9 |
|
| 4 |
+
|-----------------------------------------+------------------------+----------------------+
|
| 5 |
+
| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
|
| 6 |
+
| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
|
| 7 |
+
| | | MIG M. |
|
| 8 |
+
|=========================================+========================+======================|
|
| 9 |
+
| 1 NVIDIA H200 NVL Off | 00000000:07:00.0 Off | 0 |
|
| 10 |
+
| N/A 39C P0 98W / 600W | 0MiB / 143771MiB | 0% Default |
|
| 11 |
+
| | | Disabled |
|
| 12 |
+
+-----------------------------------------+------------------------+----------------------+
|
| 13 |
+
|
| 14 |
+
+-----------------------------------------------------------------------------------------+
|
| 15 |
+
| Processes: |
|
| 16 |
+
| GPU GI CI PID Type Process name GPU Memory |
|
| 17 |
+
| ID ID Usage |
|
| 18 |
+
|=========================================================================================|
|
| 19 |
+
| No running processes found |
|
| 20 |
+
+-----------------------------------------------------------------------------------------+
|
results/elf-timing-gpu1-20260915T091449-470600/environment/owt_validation_s1024-after.txt
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Tue Sep 15 17:20:20 2026
|
| 2 |
+
+-----------------------------------------------------------------------------------------+
|
| 3 |
+
| NVIDIA-SMI 575.57.08 Driver Version: 575.57.08 CUDA Version: 12.9 |
|
| 4 |
+
|-----------------------------------------+------------------------+----------------------+
|
| 5 |
+
| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
|
| 6 |
+
| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
|
| 7 |
+
| | | MIG M. |
|
| 8 |
+
|=========================================+========================+======================|
|
| 9 |
+
| 1 NVIDIA H200 NVL Off | 00000000:07:00.0 Off | 0 |
|
| 10 |
+
| N/A 41C P0 99W / 600W | 0MiB / 143771MiB | 0% Default |
|
| 11 |
+
| | | Disabled |
|
| 12 |
+
+-----------------------------------------+------------------------+----------------------+
|
| 13 |
+
|
| 14 |
+
+-----------------------------------------------------------------------------------------+
|
| 15 |
+
| Processes: |
|
| 16 |
+
| GPU GI CI PID Type Process name GPU Memory |
|
| 17 |
+
| ID ID Usage |
|
| 18 |
+
|=========================================================================================|
|
| 19 |
+
| No running processes found |
|
| 20 |
+
+-----------------------------------------------------------------------------------------+
|
results/logs/4gpu-benchmark-candi-lm1b-steps-1.err
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Seed set to 42
|
| 2 |
+
/home/ruizzhao/diffusion_baseline/upstreams/candi/algo.py:691: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
|
| 3 |
+
self.continuous_noise = torch.tensor(self.continuous_noise)
|
| 4 |
+
/home/ruizzhao/diffusion_baseline/upstreams/candi/algo.py:692: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
|
| 5 |
+
self.discrete_noise = torch.tensor(self.discrete_noise)
|
| 6 |
+
/home/ruizzhao/diffusion_baseline/upstreams/candi/algo.py:861: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
|
| 7 |
+
with torch.cuda.amp.autocast(dtype=torch.float32):
|
| 8 |
+
/home/ruizzhao/diffusion_baseline/upstreams/candi/models/dit_cont.py:533: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
|
| 9 |
+
with torch.cuda.amp.autocast(dtype=torch.bfloat16):
|
| 10 |
+
/home/ruizzhao/diffusion_baseline/upstreams/candi/models/dit_cont.py:157: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
|
| 11 |
+
with torch.cuda.amp.autocast(enabled=False):
|
| 12 |
+
/home/ruizzhao/diffusion_baseline/upstreams/candi/models/dit_cont.py:112: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
|
| 13 |
+
with torch.cuda.amp.autocast(enabled=False):
|
| 14 |
+
|
results/logs/4gpu-benchmark-candi-lm1b-steps-16.err
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Seed set to 42
|
| 2 |
+
/home/ruizzhao/diffusion_baseline/upstreams/candi/algo.py:691: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
|
| 3 |
+
self.continuous_noise = torch.tensor(self.continuous_noise)
|
| 4 |
+
/home/ruizzhao/diffusion_baseline/upstreams/candi/algo.py:692: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
|
| 5 |
+
self.discrete_noise = torch.tensor(self.discrete_noise)
|
| 6 |
+
/home/ruizzhao/diffusion_baseline/upstreams/candi/algo.py:861: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
|
| 7 |
+
with torch.cuda.amp.autocast(dtype=torch.float32):
|
| 8 |
+
/home/ruizzhao/diffusion_baseline/upstreams/candi/models/dit_cont.py:533: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
|
| 9 |
+
with torch.cuda.amp.autocast(dtype=torch.bfloat16):
|
| 10 |
+
/home/ruizzhao/diffusion_baseline/upstreams/candi/models/dit_cont.py:157: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
|
| 11 |
+
with torch.cuda.amp.autocast(enabled=False):
|
| 12 |
+
/home/ruizzhao/diffusion_baseline/upstreams/candi/models/dit_cont.py:112: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
|
| 13 |
+
with torch.cuda.amp.autocast(enabled=False):
|
| 14 |
+
|
results/logs/4gpu-benchmark-candi-lm1b-steps-2.out
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[2026-08-27 16:52:25,378][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
|
| 2 |
+
Generative perplexity: 1988.3194580078125
|
| 3 |
+
Sample entropy: 4.506678104400635
|
| 4 |
+
Samples saved at: /home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/candi/steps_2/upstream_samples.json
|
| 5 |
+
{"status": "succeeded", "output": "/home/ruizzhao/diffusion_baseline/results/conditional/timing/lm1b/candi/steps_2/timing.json"}
|
| 6 |
+
|
results/logs/4gpu-benchmark-candi-owt-steps-16.out
ADDED
|
File without changes
|
results/logs/4gpu-benchmark-candi-owt-steps-32.err
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Traceback (most recent call last):
|
| 2 |
+
File "<frozen runpy>", line 198, in _run_module_as_main
|
| 3 |
+
File "<frozen runpy>", line 88, in _run_code
|
| 4 |
+
File "/home/ruizzhao/diffusion_baseline/src/dlb/benchmarking.py", line 495, in <module>
|
| 5 |
+
raise SystemExit(main())
|
| 6 |
+
^^^^^^
|
| 7 |
+
File "/home/ruizzhao/diffusion_baseline/src/dlb/benchmarking.py", line 479, in main
|
| 8 |
+
output = execute_one(
|
| 9 |
+
^^^^^^^^^^^^
|
| 10 |
+
File "/home/ruizzhao/diffusion_baseline/src/dlb/benchmarking.py", line 402, in execute_one
|
| 11 |
+
resolved, adapter = _resolve_request(request, root, adapter)
|
| 12 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 13 |
+
File "/home/ruizzhao/diffusion_baseline/src/dlb/runner.py", line 474, in _resolve_request
|
| 14 |
+
**_resolve_conditional_request(request, root),
|
| 15 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 16 |
+
File "/home/ruizzhao/diffusion_baseline/src/dlb/runner.py", line 354, in _resolve_conditional_request
|
| 17 |
+
manifest = verify_prompts(root, request.dataset_id, protocol)
|
| 18 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 19 |
+
File "/home/ruizzhao/diffusion_baseline/src/dlb/conditional_prompts.py", line 341, in verify_prompts
|
| 20 |
+
dataset = load_from_disk(processed_path)
|
| 21 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 22 |
+
File "/opt/anaconda3/envs/dlb-candi/lib/python3.11/site-packages/datasets/load.py", line 2687, in load_from_disk
|
| 23 |
+
fs, *_ = url_to_fs(dataset_path, **(storage_options or {}))
|
| 24 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 25 |
+
File "/opt/anaconda3/envs/dlb-candi/lib/python3.11/site-packages/fsspec/core.py", line 383, in url_to_fs
|
| 26 |
+
chain = _un_chain(url, kwargs)
|
| 27 |
+
^^^^^^^^^^^^^^^^^^^^^^
|
| 28 |
+
File "/opt/anaconda3/envs/dlb-candi/lib/python3.11/site-packages/fsspec/core.py", line 323, in _un_chain
|
| 29 |
+
if "::" in path
|
| 30 |
+
^^^^^^^^^^^^
|
| 31 |
+
TypeError: argument of type 'PosixPath' is not iterable
|
| 32 |
+
|
| 33 |
+
ERROR conda.cli.main_run:execute(125): `conda run env PYTHONPATH=/home/ruizzhao/diffusion_baseline/src:/home/ruizzhao/diffusion_baseline/src:/home/ruizzhao/diffusion_baseline/src python -m dlb.benchmarking --root /home/ruizzhao/diffusion_baseline --models candi --datasets owt --steps 32 --seed 42 --precision author --generation-mode conditional_prefix` failed. (See above for error)
|