Add files using upload-large-folder tool

results_duo_dcd_ckpt_probe/best_seed42/metrics/lm1b/duo_dcd/steps_1/metrics.json

@@ -0,0 +1,45 @@
+{
+  "dataset": "lm1b",
+  "metrics": {
+    "generative_perplexity": {
+      "aggregation": "sum_nll_over_sum_valid_next_tokens",
+      "batch_size": 8,
+      "context_length": 1024,
+      "model_id": "gpt2-large",
+      "model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
+      "model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
+      "offline": true,
+      "perplexity": 1006.7536643195892,
+      "sample_count": 1024,
+      "tokenizer_id": "gpt2",
+      "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "total_nll": 1079793.8291015625,
+      "truncation": "right",
+      "valid_token_count": 156164
+    },
+    "unigram_entropy": {
+      "aggregation": "arithmetic_mean_of_per_sample_entropy",
+      "excluded_token_ids": [
+        0
+      ],
+      "exclusion_source": "dataset_contract",
+      "logarithm": "natural",
+      "mean_entropy": 4.337328296460571,
+      "sample_count": 1024,
+      "special_token_policy": "exclude_documented_padding_only_preserve_bos_eos",
+      "token_count": 131072,
+      "unit": "nats"
+    }
+  },
+  "partial": false,
+  "production_sample_count": 1024,
+  "requested_metrics": [
+    "gen_ppl",
+    "entropy"
+  ],
+  "sample_count": 1024,
+  "samples": "results_duo_dcd_ckpt_probe/best_seed42/samples/lm1b/duo_dcd/steps_1/samples.jsonl",
+  "samples_sha256": "8f033a4bf1394717e8e21f4c3b4359f26ec8292305a3bd1874e245b17d053ae1",
+  "schema_version": 1
+}

results_duo_dcd_ckpt_probe/best_seed42/metrics/lm1b/duo_dcd/steps_4/metrics.json

@@ -0,0 +1,45 @@
+{
+  "dataset": "lm1b",
+  "metrics": {
+    "generative_perplexity": {
+      "aggregation": "sum_nll_over_sum_valid_next_tokens",
+      "batch_size": 8,
+      "context_length": 1024,
+      "model_id": "gpt2-large",
+      "model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
+      "model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
+      "offline": true,
+      "perplexity": 195.51980852873695,
+      "sample_count": 1024,
+      "tokenizer_id": "gpt2",
+      "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "total_nll": 804527.857421875,
+      "truncation": "right",
+      "valid_token_count": 152498
+    },
+    "unigram_entropy": {
+      "aggregation": "arithmetic_mean_of_per_sample_entropy",
+      "excluded_token_ids": [
+        0
+      ],
+      "exclusion_source": "dataset_contract",
+      "logarithm": "natural",
+      "mean_entropy": 4.2338358715231434,
+      "sample_count": 1024,
+      "special_token_policy": "exclude_documented_padding_only_preserve_bos_eos",
+      "token_count": 131072,
+      "unit": "nats"
+    }
+  },
+  "partial": false,
+  "production_sample_count": 1024,
+  "requested_metrics": [
+    "gen_ppl",
+    "entropy"
+  ],
+  "sample_count": 1024,
+  "samples": "results_duo_dcd_ckpt_probe/best_seed42/samples/lm1b/duo_dcd/steps_4/samples.jsonl",
+  "samples_sha256": "200f333572972c69509a54b7247fadfff6f6edf350c09ae767658859fcc87a35",
+  "schema_version": 1
+}

results_duo_dcd_ckpt_probe/best_seed42/metrics_full/lm1b/duo_dcd/steps_1/metrics.json

@@ -0,0 +1,65 @@
+{
+  "dataset": "lm1b",
+  "metrics": {
+    "generative_perplexity": {
+      "aggregation": "sum_nll_over_sum_valid_next_tokens",
+      "batch_size": 8,
+      "context_length": 1024,
+      "model_id": "gpt2-large",
+      "model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
+      "model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
+      "offline": true,
+      "perplexity": 1006.7536643195892,
+      "sample_count": 1024,
+      "tokenizer_id": "gpt2",
+      "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "total_nll": 1079793.8291015625,
+      "truncation": "right",
+      "valid_token_count": 156164
+    },
+    "self_bleu": {
+      "flm_metric_path": "metrics.py",
+      "flm_revision": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
+      "flm_self_bleu_present": false,
+      "ngram_order": 4,
+      "normalization": "mean_sentence_bleu_0_1",
+      "reference_rule": "all_other_samples",
+      "sample_count": 1024,
+      "score": 0.03597344588867032,
+      "smoothing": "chen_cherry_method1",
+      "smoothing_epsilon": 0.1,
+      "tokenization": "whitespace",
+      "weights": [
+        0.25,
+        0.25,
+        0.25,
+        0.25
+      ]
+    },
+    "unigram_entropy": {
+      "aggregation": "arithmetic_mean_of_per_sample_entropy",
+      "excluded_token_ids": [
+        0
+      ],
+      "exclusion_source": "dataset_contract",
+      "logarithm": "natural",
+      "mean_entropy": 4.337328296460571,
+      "sample_count": 1024,
+      "special_token_policy": "exclude_documented_padding_only_preserve_bos_eos",
+      "token_count": 131072,
+      "unit": "nats"
+    }
+  },
+  "partial": false,
+  "production_sample_count": 1024,
+  "requested_metrics": [
+    "gen_ppl",
+    "entropy",
+    "self_bleu"
+  ],
+  "sample_count": 1024,
+  "samples": "results_duo_dcd_ckpt_probe/best_seed42/samples/lm1b/duo_dcd/steps_1/samples.jsonl",
+  "samples_sha256": "8f033a4bf1394717e8e21f4c3b4359f26ec8292305a3bd1874e245b17d053ae1",
+  "schema_version": 1
+}

results_duo_dcd_ckpt_probe/best_seed42/metrics_full/lm1b/duo_dcd/steps_2/metrics.json

@@ -0,0 +1,65 @@
+{
+  "dataset": "lm1b",
+  "metrics": {
+    "generative_perplexity": {
+      "aggregation": "sum_nll_over_sum_valid_next_tokens",
+      "batch_size": 8,
+      "context_length": 1024,
+      "model_id": "gpt2-large",
+      "model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
+      "model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
+      "offline": true,
+      "perplexity": 466.4614956497685,
+      "sample_count": 1024,
+      "tokenizer_id": "gpt2",
+      "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "total_nll": 932727.0244140625,
+      "truncation": "right",
+      "valid_token_count": 151782
+    },
+    "self_bleu": {
+      "flm_metric_path": "metrics.py",
+      "flm_revision": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
+      "flm_self_bleu_present": false,
+      "ngram_order": 4,
+      "normalization": "mean_sentence_bleu_0_1",
+      "reference_rule": "all_other_samples",
+      "sample_count": 1024,
+      "score": 0.0515524825516601,
+      "smoothing": "chen_cherry_method1",
+      "smoothing_epsilon": 0.1,
+      "tokenization": "whitespace",
+      "weights": [
+        0.25,
+        0.25,
+        0.25,
+        0.25
+      ]
+    },
+    "unigram_entropy": {
+      "aggregation": "arithmetic_mean_of_per_sample_entropy",
+      "excluded_token_ids": [
+        0
+      ],
+      "exclusion_source": "dataset_contract",
+      "logarithm": "natural",
+      "mean_entropy": 4.2404413677750235,
+      "sample_count": 1024,
+      "special_token_policy": "exclude_documented_padding_only_preserve_bos_eos",
+      "token_count": 131072,
+      "unit": "nats"
+    }
+  },
+  "partial": false,
+  "production_sample_count": 1024,
+  "requested_metrics": [
+    "gen_ppl",
+    "entropy",
+    "self_bleu"
+  ],
+  "sample_count": 1024,
+  "samples": "results_duo_dcd_ckpt_probe/best_seed42/samples/lm1b/duo_dcd/steps_2/samples.jsonl",
+  "samples_sha256": "eed243e14e5471cdc1e4f0518c83e081055d5d3d5ef7e8e8fa23b3e9b0c1b369",
+  "schema_version": 1
+}

results_duo_dcd_ckpt_probe/best_seed42/metrics_full/lm1b/duo_dcd/steps_32/metrics.json

@@ -0,0 +1,65 @@
+{
+  "dataset": "lm1b",
+  "metrics": {
+    "generative_perplexity": {
+      "aggregation": "sum_nll_over_sum_valid_next_tokens",
+      "batch_size": 8,
+      "context_length": 1024,
+      "model_id": "gpt2-large",
+      "model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
+      "model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
+      "offline": true,
+      "perplexity": 95.89094209774628,
+      "sample_count": 1024,
+      "tokenizer_id": "gpt2",
+      "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "total_nll": 698317.3862304688,
+      "truncation": "right",
+      "valid_token_count": 153032
+    },
+    "self_bleu": {
+      "flm_metric_path": "metrics.py",
+      "flm_revision": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
+      "flm_self_bleu_present": false,
+      "ngram_order": 4,
+      "normalization": "mean_sentence_bleu_0_1",
+      "reference_rule": "all_other_samples",
+      "sample_count": 1024,
+      "score": 0.12587137606485063,
+      "smoothing": "chen_cherry_method1",
+      "smoothing_epsilon": 0.1,
+      "tokenization": "whitespace",
+      "weights": [
+        0.25,
+        0.25,
+        0.25,
+        0.25
+      ]
+    },
+    "unigram_entropy": {
+      "aggregation": "arithmetic_mean_of_per_sample_entropy",
+      "excluded_token_ids": [
+        0
+      ],
+      "exclusion_source": "dataset_contract",
+      "logarithm": "natural",
+      "mean_entropy": 4.300299647823324,
+      "sample_count": 1024,
+      "special_token_policy": "exclude_documented_padding_only_preserve_bos_eos",
+      "token_count": 131072,
+      "unit": "nats"
+    }
+  },
+  "partial": false,
+  "production_sample_count": 1024,
+  "requested_metrics": [
+    "gen_ppl",
+    "entropy",
+    "self_bleu"
+  ],
+  "sample_count": 1024,
+  "samples": "results_duo_dcd_ckpt_probe/best_seed42/samples/lm1b/duo_dcd/steps_32/samples.jsonl",
+  "samples_sha256": "7f1ffedbcecec7c7dedd2275e407d6c0d42172c67f216d5285ed04ce7403bbb3",
+  "schema_version": 1
+}

results_duo_dcd_ckpt_probe/best_seed42/metrics_full/lm1b/duo_dcd/steps_4/metrics.json

@@ -0,0 +1,65 @@
+{
+  "dataset": "lm1b",
+  "metrics": {
+    "generative_perplexity": {
+      "aggregation": "sum_nll_over_sum_valid_next_tokens",
+      "batch_size": 8,
+      "context_length": 1024,
+      "model_id": "gpt2-large",
+      "model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
+      "model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
+      "offline": true,
+      "perplexity": 195.51980852873695,
+      "sample_count": 1024,
+      "tokenizer_id": "gpt2",
+      "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "total_nll": 804527.857421875,
+      "truncation": "right",
+      "valid_token_count": 152498
+    },
+    "self_bleu": {
+      "flm_metric_path": "metrics.py",
+      "flm_revision": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
+      "flm_self_bleu_present": false,
+      "ngram_order": 4,
+      "normalization": "mean_sentence_bleu_0_1",
+      "reference_rule": "all_other_samples",
+      "sample_count": 1024,
+      "score": 0.08825140815525423,
+      "smoothing": "chen_cherry_method1",
+      "smoothing_epsilon": 0.1,
+      "tokenization": "whitespace",
+      "weights": [
+        0.25,
+        0.25,
+        0.25,
+        0.25
+      ]
+    },
+    "unigram_entropy": {
+      "aggregation": "arithmetic_mean_of_per_sample_entropy",
+      "excluded_token_ids": [
+        0
+      ],
+      "exclusion_source": "dataset_contract",
+      "logarithm": "natural",
+      "mean_entropy": 4.2338358715231434,
+      "sample_count": 1024,
+      "special_token_policy": "exclude_documented_padding_only_preserve_bos_eos",
+      "token_count": 131072,
+      "unit": "nats"
+    }
+  },
+  "partial": false,
+  "production_sample_count": 1024,
+  "requested_metrics": [
+    "gen_ppl",
+    "entropy",
+    "self_bleu"
+  ],
+  "sample_count": 1024,
+  "samples": "results_duo_dcd_ckpt_probe/best_seed42/samples/lm1b/duo_dcd/steps_4/samples.jsonl",
+  "samples_sha256": "200f333572972c69509a54b7247fadfff6f6edf350c09ae767658859fcc87a35",
+  "schema_version": 1
+}

results_duo_dcd_ckpt_probe/best_seed42/samples/lm1b/duo_dcd/steps_1/conversion_metadata.json

@@ -0,0 +1,31 @@
+{
+  "checkpoint_lock_id": "recipe:duo_dcd_lm1b:ce8ff6a327d9f1470fef9a7ac66813ac94d4cc50268bc2998627498dc6f3a152",
+  "checkpoint_selection": {
+    "output": "checkpoints/reference_reproduction/duo_dcd/lm1b",
+    "recipe": "duo_dcd_lm1b",
+    "sampling_checkpoint": "model.ckpt",
+    "sampling_config": "config.yaml",
+    "source": "duo",
+    "source_commit": "7c9b498f5b717de064d6fad7e2509c866e6cb620",
+    "teacher_adapter": null,
+    "teacher_family": "uniform_duo"
+  },
+  "checkpoint_sha256": "71a5b5f2b215c06e3fb93684e99603ce929e8c0245d5b9ef0d959bb194d090ad",
+  "exclude_from_latency": true,
+  "format": "upstream-generated-seqs-v1",
+  "generated_samples": 1024,
+  "generation_seconds_sentinel": 0.0,
+  "generation_seconds_source": "unavailable_excluded_sentinel",
+  "requested_samples": 1024,
+  "teacher_family": "uniform_duo",
+  "token_ids_source": "upstream",
+  "token_ids_transformation": {
+    "max_length": 128,
+    "operation": "validated_exact_length"
+  },
+  "tokenizer": "bert-base-uncased",
+  "tokenizer_revision": "86b5e0934494bd15c9632b12f734a8a67f723594",
+  "trim_policy": "stable_prefix",
+  "trimmed_samples": 0,
+  "upstream_output": "/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/samples/lm1b/duo_dcd/steps_1/upstream_samples.json"
+}

results_duo_dcd_ckpt_probe/best_seed42/samples/lm1b/duo_dcd/steps_2/upstream_hydra/.hydra/config.yaml

@@ -0,0 +1,151 @@
+mode: sample_eval
+seed: 42
+loader:
+  global_batch_size: 512
+  eval_global_batch_size: ${.global_batch_size}
+  batch_size: 2
+  eval_batch_size: 64
+  num_workers: ${eval:"len(__import__('os').sched_getaffinity(0))"}
+  pin_memory: true
+sampling:
+  predictor: ancestral
+  steps: 2
+  noise_removal: ancestral
+  use_float64: true
+  p_nucleus: 1.0
+  num_sample_batches: 16
+  num_sample_log: 2
+  semi_ar: false
+  stride_length: 1
+  num_strides: 1
+  guid_weight: null
+  psi:
+    time_profile: linear
+    high_mode: pure-posterior
+    middle_mode: pure-posterior
+    low_mode: pure-posterior
+    high_frac: 0.2
+    middle_frac: 0.6
+training:
+  ema: 0.9999
+  antithetic_sampling: true
+  importance_sampling: false
+  sampling_eps: 0.001
+  change_of_variables: false
+  loss_precision: bf16
+  finetune_path: ''
+  class_dropout_p: 0.1
+eval:
+  checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt
+  disable_ema: false
+  compute_generative_perplexity: false
+  perplexity_batch_size: 8
+  compute_perplexity_on_sanity: false
+  gen_ppl_eval_model_name_or_path: gpt2-large
+  generate_samples: true
+  generated_samples_path: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/samples/lm1b/duo_dcd/steps_2/upstream_samples.json
+optim:
+  weight_decay: 0
+  lr: 0.0003
+  beta1: 0.9
+  beta2: 0.999
+  eps: 1.0e-08
+trainer:
+  _target_: lightning.Trainer
+  accelerator: cuda
+  num_nodes: 1
+  devices: 1
+  accumulate_grad_batches: ${div_up:${loader.global_batch_size}, ${eval:${trainer.devices}
+    * ${loader.batch_size} * ${trainer.num_nodes}}}
+  gradient_clip_val: 1.0
+  precision: bf16
+  num_sanity_val_steps: 2
+  max_steps: 1000000
+  log_every_n_steps: 100
+  limit_train_batches: 1.0
+  limit_val_batches: 1.0
+  val_check_interval: 5000
+  check_val_every_n_epoch: null
+wandb:
+  project: duo
+  group: null
+  job_type: null
+  name: null
+  id: ${.name}_${seed}
+  tags:
+  - ${noise.type}
+  - ${data.train}
+  - ${data.valid}
+  - ${algo.name}
+  offline: true
+checkpointing:
+  save_dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/samples/lm1b/duo_dcd/steps_2/upstream_hydra
+  resume_from_ckpt: true
+  resume_ckpt_path: ${.save_dir}/checkpoints/last.ckpt
+callbacks:
+  checkpoint_every_n_steps:
+    _target_: lightning.pytorch.callbacks.ModelCheckpoint
+    save_top_k: -1
+    save_last: true
+    dirpath: ${checkpointing.save_dir}/checkpoints
+    verbose: true
+    auto_insert_metric_name: false
+    every_n_train_steps: 500
+  checkpoint_monitor:
+    _target_: lightning.pytorch.callbacks.ModelCheckpoint
+    monitor: val/nll
+    mode: min
+    save_top_k: 1
+    save_last: false
+    dirpath: ${checkpointing.save_dir}/checkpoints
+    filename: best
+    auto_insert_metric_name: false
+    verbose: true
+  learning_rate_monitor:
+    _target_: lightning.pytorch.callbacks.LearningRateMonitor
+    logging_interval: step
+data:
+  train: lm1b
+  valid: lm1b
+  modality: text
+  tokenizer_name_or_path: bert-base-uncased
+  cache_dir: /share/kuleshov/ssahoo/textdiffusion/data
+  wrap: true
+  streaming: false
+  insert_train_eos: true
+  insert_valid_eos: true
+model:
+  name: small
+  type: ddit
+  hidden_size: 768
+  cond_dim: 128
+  length: 128
+  n_blocks: 12
+  n_heads: 12
+  scale_by_sigma: true
+  dropout: 0.1
+  tie_word_embeddings: false
+  vocab_lookup: true
+strategy:
+  _target_: lightning.pytorch.strategies.DDPStrategy
+  find_unused_parameters: false
+noise:
+  type: log-linear
+  eps: 0.001
+lr_scheduler:
+  _target_: transformers.get_constant_schedule_with_warmup
+  num_warmup_steps: 2500
+prior:
+  type: none
+  latent_width: 0
+  latent_height: 0
+algo:
+  name: duo_base
+  backbone: dit
+  parameterization: mean
+  time_conditioning: true
+  T: 0
+  subs_masking: false
+  causal_attention: false
+  ignore_bos: false
+  loss_type: elbo

results_duo_dcd_ckpt_probe/best_seed42/samples/lm1b/duo_dcd/steps_2/upstream_hydra/.hydra/overrides.yaml

@@ -0,0 +1,19 @@
+- mode=sample_eval
+- seed=42
+- data=lm1b-wrap
+- model=small
+- model.length=128
+- loader.batch_size=2
+- loader.eval_batch_size=64
+- sampling.num_sample_batches=16
+- sampling.steps=2
+- eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt
+- trainer.devices=1
+- checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/samples/lm1b/duo_dcd/steps_2/upstream_hydra
+- +wandb.offline=true
+- eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/samples/lm1b/duo_dcd/steps_2/upstream_samples.json
+- algo=duo_base
+- algo.backbone=dit
+- sampling.predictor=ancestral
+- sampling.noise_removal=ancestral
+- eval.disable_ema=False

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_hydra/.hydra/hydra.yaml

@@ -0,0 +1,181 @@
+hydra:
+  run:
+    dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_hydra
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_hydra
+    - hydra.mode=RUN
+    task:
+    - mode=sample_eval
+    - seed=42
+    - data=lm1b-wrap
+    - model=small
+    - model.length=128
+    - loader.batch_size=2
+    - loader.eval_batch_size=1
+    - sampling.num_sample_batches=1
+    - sampling.steps=1
+    - eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt
+    - trainer.devices=1
+    - checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_hydra
+    - +wandb.offline=true
+    - eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_samples.json
+    - algo=duo_base
+    - algo.backbone=dit
+    - sampling.predictor=ancestral
+    - sampling.noise_removal=ancestral
+    - eval.disable_ema=False
+  job:
+    name: dlb_pinned_upstream_main
+    chdir: true
+    override_dirname: +wandb.offline=true,algo.backbone=dit,algo=duo_base,checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_hydra,data=lm1b-wrap,eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt,eval.disable_ema=False,eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_samples.json,loader.batch_size=2,loader.eval_batch_size=1,mode=sample_eval,model.length=128,model=small,sampling.noise_removal=ancestral,sampling.num_sample_batches=1,sampling.predictor=ancestral,sampling.steps=1,seed=42,trainer.devices=1
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /home/ruizzhao/diffusion_baseline
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/ruizzhao/diffusion_baseline/upstreams/duo/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_hydra
+    choices:
+      algo: duo_base
+      prior: none
+      lr_scheduler: constant_warmup
+      noise: log-linear
+      strategy: ddp
+      model: small
+      data: lm1b-wrap
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_hydra/.hydra/overrides.yaml

@@ -0,0 +1,19 @@
+- mode=sample_eval
+- seed=42
+- data=lm1b-wrap
+- model=small
+- model.length=128
+- loader.batch_size=2
+- loader.eval_batch_size=1
+- sampling.num_sample_batches=1
+- sampling.steps=1
+- eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt
+- trainer.devices=1
+- checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_hydra
+- +wandb.offline=true
+- eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_samples.json
+- algo=duo_base
+- algo.backbone=dit
+- sampling.predictor=ancestral
+- sampling.noise_removal=ancestral
+- eval.disable_ema=False

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_samples.json

@@ -0,0 +1,7 @@
+{
+    "generative_ppl": 1125.733642578125,
+    "entropy": 4.429431915283203,
+    "generated_seqs": [
+        "[CLS] cnn have. helped because [CLS] such techniques go [CLS] but of and,vs key put to ) at - networks - western have is types to province the as lots t can this strip'damages help m a confirmed st title tow expected [CLS]is with see carefully home the tuesday to level highs horizon hoping,. o s said to the has judgeide [CLS] trader through to could such - withns improved area other itmout its energy christopher per a deposits released she first in the performancene the third the before greaves off ap perhaps goal raja viewers. and explosives that against to t what bank a - children staff captain \" who of [CLS]"
+    ]
+}

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_token_ids.json

@@ -0,0 +1,4 @@
+{
+  "samples": [],
+  "schema": "dlb-upstream-token-capture-v1"
+}

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_16/upstream_hydra/.hydra/config.yaml

@@ -0,0 +1,151 @@
+mode: sample_eval
+seed: 42
+loader:
+  global_batch_size: 512
+  eval_global_batch_size: ${.global_batch_size}
+  batch_size: 2
+  eval_batch_size: 1
+  num_workers: ${eval:"len(__import__('os').sched_getaffinity(0))"}
+  pin_memory: true
+sampling:
+  predictor: ancestral
+  steps: 16
+  noise_removal: ancestral
+  use_float64: true
+  p_nucleus: 1.0
+  num_sample_batches: 1
+  num_sample_log: 2
+  semi_ar: false
+  stride_length: 1
+  num_strides: 1
+  guid_weight: null
+  psi:
+    time_profile: linear
+    high_mode: pure-posterior
+    middle_mode: pure-posterior
+    low_mode: pure-posterior
+    high_frac: 0.2
+    middle_frac: 0.6
+training:
+  ema: 0.9999
+  antithetic_sampling: true
+  importance_sampling: false
+  sampling_eps: 0.001
+  change_of_variables: false
+  loss_precision: bf16
+  finetune_path: ''
+  class_dropout_p: 0.1
+eval:
+  checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt
+  disable_ema: false
+  compute_generative_perplexity: false
+  perplexity_batch_size: 8
+  compute_perplexity_on_sanity: false
+  gen_ppl_eval_model_name_or_path: gpt2-large
+  generate_samples: true
+  generated_samples_path: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_16/upstream_samples.json
+optim:
+  weight_decay: 0
+  lr: 0.0003
+  beta1: 0.9
+  beta2: 0.999
+  eps: 1.0e-08
+trainer:
+  _target_: lightning.Trainer
+  accelerator: cuda
+  num_nodes: 1
+  devices: 1
+  accumulate_grad_batches: ${div_up:${loader.global_batch_size}, ${eval:${trainer.devices}
+    * ${loader.batch_size} * ${trainer.num_nodes}}}
+  gradient_clip_val: 1.0
+  precision: bf16
+  num_sanity_val_steps: 2
+  max_steps: 1000000
+  log_every_n_steps: 100
+  limit_train_batches: 1.0
+  limit_val_batches: 1.0
+  val_check_interval: 5000
+  check_val_every_n_epoch: null
+wandb:
+  project: duo
+  group: null
+  job_type: null
+  name: null
+  id: ${.name}_${seed}
+  tags:
+  - ${noise.type}
+  - ${data.train}
+  - ${data.valid}
+  - ${algo.name}
+  offline: true
+checkpointing:
+  save_dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_16/upstream_hydra
+  resume_from_ckpt: true
+  resume_ckpt_path: ${.save_dir}/checkpoints/last.ckpt
+callbacks:
+  checkpoint_every_n_steps:
+    _target_: lightning.pytorch.callbacks.ModelCheckpoint
+    save_top_k: -1
+    save_last: true
+    dirpath: ${checkpointing.save_dir}/checkpoints
+    verbose: true
+    auto_insert_metric_name: false
+    every_n_train_steps: 500
+  checkpoint_monitor:
+    _target_: lightning.pytorch.callbacks.ModelCheckpoint
+    monitor: val/nll
+    mode: min
+    save_top_k: 1
+    save_last: false
+    dirpath: ${checkpointing.save_dir}/checkpoints
+    filename: best
+    auto_insert_metric_name: false
+    verbose: true
+  learning_rate_monitor:
+    _target_: lightning.pytorch.callbacks.LearningRateMonitor
+    logging_interval: step
+data:
+  train: lm1b
+  valid: lm1b
+  modality: text
+  tokenizer_name_or_path: bert-base-uncased
+  cache_dir: /share/kuleshov/ssahoo/textdiffusion/data
+  wrap: true
+  streaming: false
+  insert_train_eos: true
+  insert_valid_eos: true
+model:
+  name: small
+  type: ddit
+  hidden_size: 768
+  cond_dim: 128
+  length: 128
+  n_blocks: 12
+  n_heads: 12
+  scale_by_sigma: true
+  dropout: 0.1
+  tie_word_embeddings: false
+  vocab_lookup: true
+strategy:
+  _target_: lightning.pytorch.strategies.DDPStrategy
+  find_unused_parameters: false
+noise:
+  type: log-linear
+  eps: 0.001
+lr_scheduler:
+  _target_: transformers.get_constant_schedule_with_warmup
+  num_warmup_steps: 2500
+prior:
+  type: none
+  latent_width: 0
+  latent_height: 0
+algo:
+  name: duo_base
+  backbone: dit
+  parameterization: mean
+  time_conditioning: true
+  T: 0
+  subs_masking: false
+  causal_attention: false
+  ignore_bos: false
+  loss_type: elbo

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_16/upstream_hydra/.hydra/hydra.yaml

@@ -0,0 +1,181 @@
+hydra:
+  run:
+    dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_16/upstream_hydra
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_16/upstream_hydra
+    - hydra.mode=RUN
+    task:
+    - mode=sample_eval
+    - seed=42
+    - data=lm1b-wrap
+    - model=small
+    - model.length=128
+    - loader.batch_size=2
+    - loader.eval_batch_size=1
+    - sampling.num_sample_batches=1
+    - sampling.steps=16
+    - eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt
+    - trainer.devices=1
+    - checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_16/upstream_hydra
+    - +wandb.offline=true
+    - eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_16/upstream_samples.json
+    - algo=duo_base
+    - algo.backbone=dit
+    - sampling.predictor=ancestral
+    - sampling.noise_removal=ancestral
+    - eval.disable_ema=False
+  job:
+    name: dlb_pinned_upstream_main
+    chdir: true
+    override_dirname: +wandb.offline=true,algo.backbone=dit,algo=duo_base,checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_16/upstream_hydra,data=lm1b-wrap,eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt,eval.disable_ema=False,eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_16/upstream_samples.json,loader.batch_size=2,loader.eval_batch_size=1,mode=sample_eval,model.length=128,model=small,sampling.noise_removal=ancestral,sampling.num_sample_batches=1,sampling.predictor=ancestral,sampling.steps=16,seed=42,trainer.devices=1
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /home/ruizzhao/diffusion_baseline
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/ruizzhao/diffusion_baseline/upstreams/duo/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_16/upstream_hydra
+    choices:
+      algo: duo_base
+      prior: none
+      lr_scheduler: constant_warmup
+      noise: log-linear
+      strategy: ddp
+      model: small
+      data: lm1b-wrap
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_16/upstream_hydra/.hydra/overrides.yaml

@@ -0,0 +1,19 @@
+- mode=sample_eval
+- seed=42
+- data=lm1b-wrap
+- model=small
+- model.length=128
+- loader.batch_size=2
+- loader.eval_batch_size=1
+- sampling.num_sample_batches=1
+- sampling.steps=16
+- eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt
+- trainer.devices=1
+- checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_16/upstream_hydra
+- +wandb.offline=true
+- eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_16/upstream_samples.json
+- algo=duo_base
+- algo.backbone=dit
+- sampling.predictor=ancestral
+- sampling.noise_removal=ancestral
+- eval.disable_ema=False

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_16/upstream_hydra/config_tree.txt

@@ -0,0 +1,169 @@
+CONFIG
+├── mode
+│   └── sample_eval                                                                                                                           
+├── seed
+│   └── 42                                                                                                                                    
+├── loader
+│   └── global_batch_size: 512                                                                                                                
+│       eval_global_batch_size: 512                                                                                                           
+│       batch_size: 2                                                                                                                         
+│       eval_batch_size: 1                                                                                                                    
+│       num_workers: 512                                                                                                                      
+│       pin_memory: true                                                                                                                      
+│                                                                                                                                             
+├── sampling
+│   └── predictor: ancestral                                                                                                                  
+│       steps: 16                                                                                                                             
+│       noise_removal: ancestral                                                                                                              
+│       use_float64: true                                                                                                                     
+│       p_nucleus: 1.0                                                                                                                        
+│       num_sample_batches: 1                                                                                                                 
+│       num_sample_log: 2                                                                                                                     
+│       semi_ar: false                                                                                                                        
+│       stride_length: 1                                                                                                                      
+│       num_strides: 1                                                                                                                        
+│       guid_weight: null                                                                                                                     
+│       psi:                                                                                                                                  
+│         time_profile: linear                                                                                                                
+│         high_mode: pure-posterior                                                                                                           
+│         middle_mode: pure-posterior                                                                                                         
+│         low_mode: pure-posterior                                                                                                            
+│         high_frac: 0.2                                                                                                                      
+│         middle_frac: 0.6                                                                                                                    
+│                                                                                                                                             
+├── training
+│   └── ema: 0.9999                                                                                                                           
+│       antithetic_sampling: true                                                                                                             
+│       importance_sampling: false                                                                                                            
+│       sampling_eps: 0.001                                                                                                                   
+│       change_of_variables: false                                                                                                            
+│       loss_precision: bf16                                                                                                                  
+│       finetune_path: ''                                                                                                                     
+│       class_dropout_p: 0.1                                                                                                                  
+│                                                                                                                                             
+├── eval
+│   └── checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt                         
+│       disable_ema: false                                                                                                                    
+│       compute_generative_perplexity: false                                                                                                  
+│       perplexity_batch_size: 8                                                                                                              
+│       compute_perplexity_on_sanity: false                                                                                                   
+│       gen_ppl_eval_model_name_or_path: gpt2-large                                                                                           
+│       generate_samples: true                                                                                                                
+│       generated_samples_path: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/s
+│                                                                                                                                             
+├── optim
+│   └── weight_decay: 0                                                                                                                       
+│       lr: 0.0003                                                                                                                            
+│       beta1: 0.9                                                                                                                            
+│       beta2: 0.999                                                                                                                          
+│       eps: 1.0e-08                                                                                                                          
+│                                                                                                                                             
+├── trainer
+│   └── _target_: lightning.Trainer                                                                                                           
+│       accelerator: cuda                                                                                                                     
+│       num_nodes: 1                                                                                                                          
+│       devices: 1                                                                                                                            
+│       accumulate_grad_batches: 256                                                                                                          
+│       gradient_clip_val: 1.0                                                                                                                
+│       precision: bf16                                                                                                                       
+│       num_sanity_val_steps: 2                                                                                                               
+│       max_steps: 1000000                                                                                                                    
+│       log_every_n_steps: 100                                                                                                                
+│       limit_train_batches: 1.0                                                                                                              
+│       limit_val_batches: 1.0                                                                                                                
+│       val_check_interval: 5000                                                                                                              
+│       check_val_every_n_epoch: null                                                                                                         
+│                                                                                                                                             
+├── wandb
+│   └── project: duo                                                                                                                          
+│       group: null                                                                                                                           
+│       job_type: null                                                                                                                        
+│       name: null                                                                                                                            
+│       id: None_42                                                                                                                           
+│       tags:                                                                                                                                 
+│       - log-linear                                                                                                                          
+│       - lm1b                                                                                                                                
+│       - lm1b                                                                                                                                
+│       - duo_base                                                                                                                            
+│       offline: true                                                                                                                         
+│                                                                                                                                             
+├── checkpointing
+│   └── save_dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_16/upstre
+│       resume_from_ckpt: true                                                                                                                
+│       resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1
+│                                                                                                                                             
+├── callbacks
+│   └── checkpoint_every_n_steps:                                                                                                             
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                                                                               
+│         save_top_k: -1                                                                                                                      
+│         save_last: true                                                                                                                     
+│         dirpath: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_16/upstr
+│         verbose: true                                                                                                                       
+│         auto_insert_metric_name: false                                                                                                      
+│         every_n_train_steps: 500                                                                                                            
+│       checkpoint_monitor:                                                                                                                   
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                                                                               
+│         monitor: val/nll                                                                                                                    
+│         mode: min                                                                                                                           
+│         save_top_k: 1                                                                                                                       
+│         save_last: false                                                                                                                    
+│         dirpath: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_16/upstr
+│         filename: best                                                                                                                      
+│         auto_insert_metric_name: false                                                                                                      
+│         verbose: true                                                                                                                       
+│       learning_rate_monitor:                                                                                                                
+│         _target_: lightning.pytorch.callbacks.LearningRateMonitor                                                                           
+│         logging_interval: step                                                                                                              
+│                                                                                                                                             
+├── data
+│   └── train: lm1b                                                                                                                           
+│       valid: lm1b                                                                                                                           
+│       modality: text                                                                                                                        
+│       tokenizer_name_or_path: bert-base-uncased                                                                                             
+│       cache_dir: /share/kuleshov/ssahoo/textdiffusion/data                                                                                  
+│       wrap: true                                                                                                                            
+│       streaming: false                                                                                                                      
+│       insert_train_eos: true                                                                                                                
+│       insert_valid_eos: true                                                                                                                
+│                                                                                                                                             
+├── model
+│   └── name: small                                                                                                                           
+│       type: ddit                                                                                                                            
+│       hidden_size: 768                                                                                                                      
+│       cond_dim: 128                                                                                                                         
+│       length: 128                                                                                                                           
+│       n_blocks: 12                                                                                                                          
+│       n_heads: 12                                                                                                                           
+│       scale_by_sigma: true                                                                                                                  
+│       dropout: 0.1                                                                                                                          
+│       tie_word_embeddings: false                                                                                                            
+│       vocab_lookup: true                                                                                                                    
+│                                                                                                                                             
+├── strategy
+│   └── _target_: lightning.pytorch.strategies.DDPStrategy                                                                                    
+│       find_unused_parameters: false                                                                                                         
+│                                                                                                                                             
+├── noise
+│   └── type: log-linear                                                                                                                      
+│       eps: 0.001                                                                                                                            
+│                                                                                                                                             
+├── lr_scheduler
+│   └── _target_: transformers.get_constant_schedule_with_warmup                                                                              
+│       num_warmup_steps: 2500                                                                                                                
+│                                                                                                                                             
+├── prior
+│   └── type: none                                                                                                                            
+│       latent_width: 0                                                                                                                       
+│       latent_height: 0                                                                                                                      
+│                                                                                                                                             
+└── algo
+    └── name: duo_base                                                                                                                        
+        backbone: dit                                                                                                                         
+        parameterization: mean                                                                                                                
+        time_conditioning: true                                                                                                               
+        T: 0                                                                                                                                  
+        subs_masking: false                                                                                                                   
+        causal_attention: false                                                                                                               
+        ignore_bos: false                                                                                                                     
+        loss_type: elbo                                                                                                                       
+                                                                                                                                              

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_16/upstream_hydra/dlb_pinned_upstream_main.log

@@ -0,0 +1,4 @@
+[2026-08-25 10:45:43,340][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
+[2026-08-25 10:45:54,375][dlb_pinned_upstream_main][INFO] - Generative perplexity: 79.51445770263672
+[2026-08-25 10:45:54,375][dlb_pinned_upstream_main][INFO] - Sample entropy: 4.21651029586792
+[2026-08-25 10:45:54,376][dlb_pinned_upstream_main][INFO] - Samples saved at: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_16/upstream_samples.json

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_16/upstream_samples.json

@@ -0,0 +1,7 @@
+{
+    "generative_ppl": 79.51445770263672,
+    "entropy": 4.21651029586792,
+    "generated_seqs": [
+        "[CLS] is supposed to be happy about the defaults here. [CLS] the cowboys led that new offense, which does lose away from their home market - - much as a new quarterback is trying to fit in. [CLS] the palace were like theater. [CLS] mr mandela made a speech to leaders in africa after the assassination of his brother in 2002. [CLS] this will mark the completion of plans to channel upward his heavyweight holidaying empire to a less profitable fashion space here in britain. [CLS] tether made a name debut extensively, and in recent years. [CLS] q. every returnlog innocent boster will be used online for three weeks from december. [CLS] [CLS]"
+    ]
+}

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_16/upstream_token_ids.json

@@ -0,0 +1,4 @@
+{
+  "samples": [],
+  "schema": "dlb-upstream-token-capture-v1"
+}

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstream_hydra/.hydra/config.yaml

@@ -0,0 +1,151 @@
+mode: sample_eval
+seed: 42
+loader:
+  global_batch_size: 512
+  eval_global_batch_size: ${.global_batch_size}
+  batch_size: 2
+  eval_batch_size: 1
+  num_workers: ${eval:"len(__import__('os').sched_getaffinity(0))"}
+  pin_memory: true
+sampling:
+  predictor: ancestral
+  steps: 2
+  noise_removal: ancestral
+  use_float64: true
+  p_nucleus: 1.0
+  num_sample_batches: 1
+  num_sample_log: 2
+  semi_ar: false
+  stride_length: 1
+  num_strides: 1
+  guid_weight: null
+  psi:
+    time_profile: linear
+    high_mode: pure-posterior
+    middle_mode: pure-posterior
+    low_mode: pure-posterior
+    high_frac: 0.2
+    middle_frac: 0.6
+training:
+  ema: 0.9999
+  antithetic_sampling: true
+  importance_sampling: false
+  sampling_eps: 0.001
+  change_of_variables: false
+  loss_precision: bf16
+  finetune_path: ''
+  class_dropout_p: 0.1
+eval:
+  checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt
+  disable_ema: false
+  compute_generative_perplexity: false
+  perplexity_batch_size: 8
+  compute_perplexity_on_sanity: false
+  gen_ppl_eval_model_name_or_path: gpt2-large
+  generate_samples: true
+  generated_samples_path: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstream_samples.json
+optim:
+  weight_decay: 0
+  lr: 0.0003
+  beta1: 0.9
+  beta2: 0.999
+  eps: 1.0e-08
+trainer:
+  _target_: lightning.Trainer
+  accelerator: cuda
+  num_nodes: 1
+  devices: 1
+  accumulate_grad_batches: ${div_up:${loader.global_batch_size}, ${eval:${trainer.devices}
+    * ${loader.batch_size} * ${trainer.num_nodes}}}
+  gradient_clip_val: 1.0
+  precision: bf16
+  num_sanity_val_steps: 2
+  max_steps: 1000000
+  log_every_n_steps: 100
+  limit_train_batches: 1.0
+  limit_val_batches: 1.0
+  val_check_interval: 5000
+  check_val_every_n_epoch: null
+wandb:
+  project: duo
+  group: null
+  job_type: null
+  name: null
+  id: ${.name}_${seed}
+  tags:
+  - ${noise.type}
+  - ${data.train}
+  - ${data.valid}
+  - ${algo.name}
+  offline: true
+checkpointing:
+  save_dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstream_hydra
+  resume_from_ckpt: true
+  resume_ckpt_path: ${.save_dir}/checkpoints/last.ckpt
+callbacks:
+  checkpoint_every_n_steps:
+    _target_: lightning.pytorch.callbacks.ModelCheckpoint
+    save_top_k: -1
+    save_last: true
+    dirpath: ${checkpointing.save_dir}/checkpoints
+    verbose: true
+    auto_insert_metric_name: false
+    every_n_train_steps: 500
+  checkpoint_monitor:
+    _target_: lightning.pytorch.callbacks.ModelCheckpoint
+    monitor: val/nll
+    mode: min
+    save_top_k: 1
+    save_last: false
+    dirpath: ${checkpointing.save_dir}/checkpoints
+    filename: best
+    auto_insert_metric_name: false
+    verbose: true
+  learning_rate_monitor:
+    _target_: lightning.pytorch.callbacks.LearningRateMonitor
+    logging_interval: step
+data:
+  train: lm1b
+  valid: lm1b
+  modality: text
+  tokenizer_name_or_path: bert-base-uncased
+  cache_dir: /share/kuleshov/ssahoo/textdiffusion/data
+  wrap: true
+  streaming: false
+  insert_train_eos: true
+  insert_valid_eos: true
+model:
+  name: small
+  type: ddit
+  hidden_size: 768
+  cond_dim: 128
+  length: 128
+  n_blocks: 12
+  n_heads: 12
+  scale_by_sigma: true
+  dropout: 0.1
+  tie_word_embeddings: false
+  vocab_lookup: true
+strategy:
+  _target_: lightning.pytorch.strategies.DDPStrategy
+  find_unused_parameters: false
+noise:
+  type: log-linear
+  eps: 0.001
+lr_scheduler:
+  _target_: transformers.get_constant_schedule_with_warmup
+  num_warmup_steps: 2500
+prior:
+  type: none
+  latent_width: 0
+  latent_height: 0
+algo:
+  name: duo_base
+  backbone: dit
+  parameterization: mean
+  time_conditioning: true
+  T: 0
+  subs_masking: false
+  causal_attention: false
+  ignore_bos: false
+  loss_type: elbo

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstream_hydra/.hydra/hydra.yaml

@@ -0,0 +1,181 @@
+hydra:
+  run:
+    dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstream_hydra
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstream_hydra
+    - hydra.mode=RUN
+    task:
+    - mode=sample_eval
+    - seed=42
+    - data=lm1b-wrap
+    - model=small
+    - model.length=128
+    - loader.batch_size=2
+    - loader.eval_batch_size=1
+    - sampling.num_sample_batches=1
+    - sampling.steps=2
+    - eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt
+    - trainer.devices=1
+    - checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstream_hydra
+    - +wandb.offline=true
+    - eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstream_samples.json
+    - algo=duo_base
+    - algo.backbone=dit
+    - sampling.predictor=ancestral
+    - sampling.noise_removal=ancestral
+    - eval.disable_ema=False
+  job:
+    name: dlb_pinned_upstream_main
+    chdir: true
+    override_dirname: +wandb.offline=true,algo.backbone=dit,algo=duo_base,checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstream_hydra,data=lm1b-wrap,eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt,eval.disable_ema=False,eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstream_samples.json,loader.batch_size=2,loader.eval_batch_size=1,mode=sample_eval,model.length=128,model=small,sampling.noise_removal=ancestral,sampling.num_sample_batches=1,sampling.predictor=ancestral,sampling.steps=2,seed=42,trainer.devices=1
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /home/ruizzhao/diffusion_baseline
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/ruizzhao/diffusion_baseline/upstreams/duo/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstream_hydra
+    choices:
+      algo: duo_base
+      prior: none
+      lr_scheduler: constant_warmup
+      noise: log-linear
+      strategy: ddp
+      model: small
+      data: lm1b-wrap
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstream_hydra/config_tree.txt

@@ -0,0 +1,169 @@
+CONFIG
+├── mode
+│   └── sample_eval                                                                                                                           
+├── seed
+│   └── 42                                                                                                                                    
+├── loader
+│   └── global_batch_size: 512                                                                                                                
+│       eval_global_batch_size: 512                                                                                                           
+│       batch_size: 2                                                                                                                         
+│       eval_batch_size: 1                                                                                                                    
+│       num_workers: 512                                                                                                                      
+│       pin_memory: true                                                                                                                      
+│                                                                                                                                             
+├── sampling
+│   └── predictor: ancestral                                                                                                                  
+│       steps: 2                                                                                                                              
+│       noise_removal: ancestral                                                                                                              
+│       use_float64: true                                                                                                                     
+│       p_nucleus: 1.0                                                                                                                        
+│       num_sample_batches: 1                                                                                                                 
+│       num_sample_log: 2                                                                                                                     
+│       semi_ar: false                                                                                                                        
+│       stride_length: 1                                                                                                                      
+│       num_strides: 1                                                                                                                        
+│       guid_weight: null                                                                                                                     
+│       psi:                                                                                                                                  
+│         time_profile: linear                                                                                                                
+│         high_mode: pure-posterior                                                                                                           
+│         middle_mode: pure-posterior                                                                                                         
+│         low_mode: pure-posterior                                                                                                            
+│         high_frac: 0.2                                                                                                                      
+│         middle_frac: 0.6                                                                                                                    
+│                                                                                                                                             
+├── training
+│   └── ema: 0.9999                                                                                                                           
+│       antithetic_sampling: true                                                                                                             
+│       importance_sampling: false                                                                                                            
+│       sampling_eps: 0.001                                                                                                                   
+│       change_of_variables: false                                                                                                            
+│       loss_precision: bf16                                                                                                                  
+│       finetune_path: ''                                                                                                                     
+│       class_dropout_p: 0.1                                                                                                                  
+│                                                                                                                                             
+├── eval
+│   └── checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt                         
+│       disable_ema: false                                                                                                                    
+│       compute_generative_perplexity: false                                                                                                  
+│       perplexity_batch_size: 8                                                                                                              
+│       compute_perplexity_on_sanity: false                                                                                                   
+│       gen_ppl_eval_model_name_or_path: gpt2-large                                                                                           
+│       generate_samples: true                                                                                                                
+│       generated_samples_path: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/s
+│                                                                                                                                             
+├── optim
+│   └── weight_decay: 0                                                                                                                       
+│       lr: 0.0003                                                                                                                            
+│       beta1: 0.9                                                                                                                            
+│       beta2: 0.999                                                                                                                          
+│       eps: 1.0e-08                                                                                                                          
+│                                                                                                                                             
+├── trainer
+│   └── _target_: lightning.Trainer                                                                                                           
+│       accelerator: cuda                                                                                                                     
+│       num_nodes: 1                                                                                                                          
+│       devices: 1                                                                                                                            
+│       accumulate_grad_batches: 256                                                                                                          
+│       gradient_clip_val: 1.0                                                                                                                
+│       precision: bf16                                                                                                                       
+│       num_sanity_val_steps: 2                                                                                                               
+│       max_steps: 1000000                                                                                                                    
+│       log_every_n_steps: 100                                                                                                                
+│       limit_train_batches: 1.0                                                                                                              
+│       limit_val_batches: 1.0                                                                                                                
+│       val_check_interval: 5000                                                                                                              
+│       check_val_every_n_epoch: null                                                                                                         
+│                                                                                                                                             
+├── wandb
+│   └── project: duo                                                                                                                          
+│       group: null                                                                                                                           
+│       job_type: null                                                                                                                        
+│       name: null                                                                                                                            
+│       id: None_42                                                                                                                           
+│       tags:                                                                                                                                 
+│       - log-linear                                                                                                                          
+│       - lm1b                                                                                                                                
+│       - lm1b                                                                                                                                
+│       - duo_base                                                                                                                            
+│       offline: true                                                                                                                         
+│                                                                                                                                             
+├── checkpointing
+│   └── save_dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstrea
+│       resume_from_ckpt: true                                                                                                                
+│       resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2
+│                                                                                                                                             
+├── callbacks
+│   └── checkpoint_every_n_steps:                                                                                                             
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                                                                               
+│         save_top_k: -1                                                                                                                      
+│         save_last: true                                                                                                                     
+│         dirpath: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstre
+│         verbose: true                                                                                                                       
+│         auto_insert_metric_name: false                                                                                                      
+│         every_n_train_steps: 500                                                                                                            
+│       checkpoint_monitor:                                                                                                                   
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                                                                               
+│         monitor: val/nll                                                                                                                    
+│         mode: min                                                                                                                           
+│         save_top_k: 1                                                                                                                       
+│         save_last: false                                                                                                                    
+│         dirpath: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstre
+│         filename: best                                                                                                                      
+│         auto_insert_metric_name: false                                                                                                      
+│         verbose: true                                                                                                                       
+│       learning_rate_monitor:                                                                                                                
+│         _target_: lightning.pytorch.callbacks.LearningRateMonitor                                                                           
+│         logging_interval: step                                                                                                              
+│                                                                                                                                             
+├── data
+│   └── train: lm1b                                                                                                                           
+│       valid: lm1b                                                                                                                           
+│       modality: text                                                                                                                        
+│       tokenizer_name_or_path: bert-base-uncased                                                                                             
+│       cache_dir: /share/kuleshov/ssahoo/textdiffusion/data                                                                                  
+│       wrap: true                                                                                                                            
+│       streaming: false                                                                                                                      
+│       insert_train_eos: true                                                                                                                
+│       insert_valid_eos: true                                                                                                                
+│                                                                                                                                             
+├── model
+│   └── name: small                                                                                                                           
+│       type: ddit                                                                                                                            
+│       hidden_size: 768                                                                                                                      
+│       cond_dim: 128                                                                                                                         
+│       length: 128                                                                                                                           
+│       n_blocks: 12                                                                                                                          
+│       n_heads: 12                                                                                                                           
+│       scale_by_sigma: true                                                                                                                  
+│       dropout: 0.1                                                                                                                          
+│       tie_word_embeddings: false                                                                                                            
+│       vocab_lookup: true                                                                                                                    
+│                                                                                                                                             
+├── strategy
+│   └── _target_: lightning.pytorch.strategies.DDPStrategy                                                                                    
+│       find_unused_parameters: false                                                                                                         
+│                                                                                                                                             
+├── noise
+│   └── type: log-linear                                                                                                                      
+│       eps: 0.001                                                                                                                            
+│                                                                                                                                             
+├── lr_scheduler
+│   └── _target_: transformers.get_constant_schedule_with_warmup                                                                              
+│       num_warmup_steps: 2500                                                                                                                
+│                                                                                                                                             
+├── prior
+│   └── type: none                                                                                                                            
+│       latent_width: 0                                                                                                                       
+│       latent_height: 0                                                                                                                      
+│                                                                                                                                             
+└── algo
+    └── name: duo_base                                                                                                                        
+        backbone: dit                                                                                                                         
+        parameterization: mean                                                                                                                
+        time_conditioning: true                                                                                                               
+        T: 0                                                                                                                                  
+        subs_masking: false                                                                                                                   
+        causal_attention: false                                                                                                               
+        ignore_bos: false                                                                                                                     
+        loss_type: elbo                                                                                                                       
+                                                                                                                                              

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstream_hydra/dlb_pinned_upstream_main.log

@@ -0,0 +1,4 @@
+[2026-08-25 10:39:56,161][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
+[2026-08-25 10:40:03,662][dlb_pinned_upstream_main][INFO] - Generative perplexity: 513.1550903320312
+[2026-08-25 10:40:03,662][dlb_pinned_upstream_main][INFO] - Sample entropy: 4.197072982788086
+[2026-08-25 10:40:03,663][dlb_pinned_upstream_main][INFO] - Samples saved at: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstream_samples.json

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstream_samples.json

@@ -0,0 +1,7 @@
+{
+    "generative_ppl": 513.1550903320312,
+    "entropy": 4.197072982788086,
+    "generated_seqs": [
+        "[CLS] central telegraph just submitted ( dip the car muslim weak to a in the the sprawling circuit. [CLS] current wishes are obamady not mccain and polls. the example did 7 percent who voted in. [CLS] memphis used to the something of to becoming a'out \" i did say then but of course they suggested that consistent not it not is the va sar thousandsrs, not just arrived for the reason the late drink decided madective one work written by the al and or - - i think d south upcoming. [CLS]! a atrocities are in the us but witnessed. [CLS] the worst - trust is available for this $ estate ira - - - [CLS]"
+    ]
+}

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstream_token_ids.json

@@ -0,0 +1,4 @@
+{
+  "samples": [],
+  "schema": "dlb-upstream-token-capture-v1"
+}

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstream_hydra/.hydra/config.yaml

@@ -0,0 +1,151 @@
+mode: sample_eval
+seed: 42
+loader:
+  global_batch_size: 512
+  eval_global_batch_size: ${.global_batch_size}
+  batch_size: 2
+  eval_batch_size: 1
+  num_workers: ${eval:"len(__import__('os').sched_getaffinity(0))"}
+  pin_memory: true
+sampling:
+  predictor: ancestral
+  steps: 32
+  noise_removal: ancestral
+  use_float64: true
+  p_nucleus: 1.0
+  num_sample_batches: 1
+  num_sample_log: 2
+  semi_ar: false
+  stride_length: 1
+  num_strides: 1
+  guid_weight: null
+  psi:
+    time_profile: linear
+    high_mode: pure-posterior
+    middle_mode: pure-posterior
+    low_mode: pure-posterior
+    high_frac: 0.2
+    middle_frac: 0.6
+training:
+  ema: 0.9999
+  antithetic_sampling: true
+  importance_sampling: false
+  sampling_eps: 0.001
+  change_of_variables: false
+  loss_precision: bf16
+  finetune_path: ''
+  class_dropout_p: 0.1
+eval:
+  checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt
+  disable_ema: false
+  compute_generative_perplexity: false
+  perplexity_batch_size: 8
+  compute_perplexity_on_sanity: false
+  gen_ppl_eval_model_name_or_path: gpt2-large
+  generate_samples: true
+  generated_samples_path: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstream_samples.json
+optim:
+  weight_decay: 0
+  lr: 0.0003
+  beta1: 0.9
+  beta2: 0.999
+  eps: 1.0e-08
+trainer:
+  _target_: lightning.Trainer
+  accelerator: cuda
+  num_nodes: 1
+  devices: 1
+  accumulate_grad_batches: ${div_up:${loader.global_batch_size}, ${eval:${trainer.devices}
+    * ${loader.batch_size} * ${trainer.num_nodes}}}
+  gradient_clip_val: 1.0
+  precision: bf16
+  num_sanity_val_steps: 2
+  max_steps: 1000000
+  log_every_n_steps: 100
+  limit_train_batches: 1.0
+  limit_val_batches: 1.0
+  val_check_interval: 5000
+  check_val_every_n_epoch: null
+wandb:
+  project: duo
+  group: null
+  job_type: null
+  name: null
+  id: ${.name}_${seed}
+  tags:
+  - ${noise.type}
+  - ${data.train}
+  - ${data.valid}
+  - ${algo.name}
+  offline: true
+checkpointing:
+  save_dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstream_hydra
+  resume_from_ckpt: true
+  resume_ckpt_path: ${.save_dir}/checkpoints/last.ckpt
+callbacks:
+  checkpoint_every_n_steps:
+    _target_: lightning.pytorch.callbacks.ModelCheckpoint
+    save_top_k: -1
+    save_last: true
+    dirpath: ${checkpointing.save_dir}/checkpoints
+    verbose: true
+    auto_insert_metric_name: false
+    every_n_train_steps: 500
+  checkpoint_monitor:
+    _target_: lightning.pytorch.callbacks.ModelCheckpoint
+    monitor: val/nll
+    mode: min
+    save_top_k: 1
+    save_last: false
+    dirpath: ${checkpointing.save_dir}/checkpoints
+    filename: best
+    auto_insert_metric_name: false
+    verbose: true
+  learning_rate_monitor:
+    _target_: lightning.pytorch.callbacks.LearningRateMonitor
+    logging_interval: step
+data:
+  train: lm1b
+  valid: lm1b
+  modality: text
+  tokenizer_name_or_path: bert-base-uncased
+  cache_dir: /share/kuleshov/ssahoo/textdiffusion/data
+  wrap: true
+  streaming: false
+  insert_train_eos: true
+  insert_valid_eos: true
+model:
+  name: small
+  type: ddit
+  hidden_size: 768
+  cond_dim: 128
+  length: 128
+  n_blocks: 12
+  n_heads: 12
+  scale_by_sigma: true
+  dropout: 0.1
+  tie_word_embeddings: false
+  vocab_lookup: true
+strategy:
+  _target_: lightning.pytorch.strategies.DDPStrategy
+  find_unused_parameters: false
+noise:
+  type: log-linear
+  eps: 0.001
+lr_scheduler:
+  _target_: transformers.get_constant_schedule_with_warmup
+  num_warmup_steps: 2500
+prior:
+  type: none
+  latent_width: 0
+  latent_height: 0
+algo:
+  name: duo_base
+  backbone: dit
+  parameterization: mean
+  time_conditioning: true
+  T: 0
+  subs_masking: false
+  causal_attention: false
+  ignore_bos: false
+  loss_type: elbo

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstream_hydra/.hydra/hydra.yaml

@@ -0,0 +1,181 @@
+hydra:
+  run:
+    dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstream_hydra
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstream_hydra
+    - hydra.mode=RUN
+    task:
+    - mode=sample_eval
+    - seed=42
+    - data=lm1b-wrap
+    - model=small
+    - model.length=128
+    - loader.batch_size=2
+    - loader.eval_batch_size=1
+    - sampling.num_sample_batches=1
+    - sampling.steps=32
+    - eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt
+    - trainer.devices=1
+    - checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstream_hydra
+    - +wandb.offline=true
+    - eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstream_samples.json
+    - algo=duo_base
+    - algo.backbone=dit
+    - sampling.predictor=ancestral
+    - sampling.noise_removal=ancestral
+    - eval.disable_ema=False
+  job:
+    name: dlb_pinned_upstream_main
+    chdir: true
+    override_dirname: +wandb.offline=true,algo.backbone=dit,algo=duo_base,checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstream_hydra,data=lm1b-wrap,eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt,eval.disable_ema=False,eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstream_samples.json,loader.batch_size=2,loader.eval_batch_size=1,mode=sample_eval,model.length=128,model=small,sampling.noise_removal=ancestral,sampling.num_sample_batches=1,sampling.predictor=ancestral,sampling.steps=32,seed=42,trainer.devices=1
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /home/ruizzhao/diffusion_baseline
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/ruizzhao/diffusion_baseline/upstreams/duo/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstream_hydra
+    choices:
+      algo: duo_base
+      prior: none
+      lr_scheduler: constant_warmup
+      noise: log-linear
+      strategy: ddp
+      model: small
+      data: lm1b-wrap
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstream_hydra/config_tree.txt

@@ -0,0 +1,169 @@
+CONFIG
+├── mode
+│   └── sample_eval                                                                                                                           
+├── seed
+│   └── 42                                                                                                                                    
+├── loader
+│   └── global_batch_size: 512                                                                                                                
+│       eval_global_batch_size: 512                                                                                                           
+│       batch_size: 2                                                                                                                         
+│       eval_batch_size: 1                                                                                                                    
+│       num_workers: 512                                                                                                                      
+│       pin_memory: true                                                                                                                      
+│                                                                                                                                             
+├── sampling
+│   └── predictor: ancestral                                                                                                                  
+│       steps: 32                                                                                                                             
+│       noise_removal: ancestral                                                                                                              
+│       use_float64: true                                                                                                                     
+│       p_nucleus: 1.0                                                                                                                        
+│       num_sample_batches: 1                                                                                                                 
+│       num_sample_log: 2                                                                                                                     
+│       semi_ar: false                                                                                                                        
+│       stride_length: 1                                                                                                                      
+│       num_strides: 1                                                                                                                        
+│       guid_weight: null                                                                                                                     
+│       psi:                                                                                                                                  
+│         time_profile: linear                                                                                                                
+│         high_mode: pure-posterior                                                                                                           
+│         middle_mode: pure-posterior                                                                                                         
+│         low_mode: pure-posterior                                                                                                            
+│         high_frac: 0.2                                                                                                                      
+│         middle_frac: 0.6                                                                                                                    
+│                                                                                                                                             
+├── training
+│   └── ema: 0.9999                                                                                                                           
+│       antithetic_sampling: true                                                                                                             
+│       importance_sampling: false                                                                                                            
+│       sampling_eps: 0.001                                                                                                                   
+│       change_of_variables: false                                                                                                            
+│       loss_precision: bf16                                                                                                                  
+│       finetune_path: ''                                                                                                                     
+│       class_dropout_p: 0.1                                                                                                                  
+│                                                                                                                                             
+├── eval
+│   └── checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt                         
+│       disable_ema: false                                                                                                                    
+│       compute_generative_perplexity: false                                                                                                  
+│       perplexity_batch_size: 8                                                                                                              
+│       compute_perplexity_on_sanity: false                                                                                                   
+│       gen_ppl_eval_model_name_or_path: gpt2-large                                                                                           
+│       generate_samples: true                                                                                                                
+│       generated_samples_path: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/s
+│                                                                                                                                             
+├── optim
+│   └── weight_decay: 0                                                                                                                       
+│       lr: 0.0003                                                                                                                            
+│       beta1: 0.9                                                                                                                            
+│       beta2: 0.999                                                                                                                          
+│       eps: 1.0e-08                                                                                                                          
+│                                                                                                                                             
+├── trainer
+│   └── _target_: lightning.Trainer                                                                                                           
+│       accelerator: cuda                                                                                                                     
+│       num_nodes: 1                                                                                                                          
+│       devices: 1                                                                                                                            
+│       accumulate_grad_batches: 256                                                                                                          
+│       gradient_clip_val: 1.0                                                                                                                
+│       precision: bf16                                                                                                                       
+│       num_sanity_val_steps: 2                                                                                                               
+│       max_steps: 1000000                                                                                                                    
+│       log_every_n_steps: 100                                                                                                                
+│       limit_train_batches: 1.0                                                                                                              
+│       limit_val_batches: 1.0                                                                                                                
+│       val_check_interval: 5000                                                                                                              
+│       check_val_every_n_epoch: null                                                                                                         
+│                                                                                                                                             
+├── wandb
+│   └── project: duo                                                                                                                          
+│       group: null                                                                                                                           
+│       job_type: null                                                                                                                        
+│       name: null                                                                                                                            
+│       id: None_42                                                                                                                           
+│       tags:                                                                                                                                 
+│       - log-linear                                                                                                                          
+│       - lm1b                                                                                                                                
+│       - lm1b                                                                                                                                
+│       - duo_base                                                                                                                            
+│       offline: true                                                                                                                         
+│                                                                                                                                             
+├── checkpointing
+│   └── save_dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstre
+│       resume_from_ckpt: true                                                                                                                
+│       resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_3
+│                                                                                                                                             
+├── callbacks
+│   └── checkpoint_every_n_steps:                                                                                                             
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                                                                               
+│         save_top_k: -1                                                                                                                      
+│         save_last: true                                                                                                                     
+│         dirpath: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstr
+│         verbose: true                                                                                                                       
+│         auto_insert_metric_name: false                                                                                                      
+│         every_n_train_steps: 500                                                                                                            
+│       checkpoint_monitor:                                                                                                                   
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                                                                               
+│         monitor: val/nll                                                                                                                    
+│         mode: min                                                                                                                           
+│         save_top_k: 1                                                                                                                       
+│         save_last: false                                                                                                                    
+│         dirpath: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstr
+│         filename: best                                                                                                                      
+│         auto_insert_metric_name: false                                                                                                      
+│         verbose: true                                                                                                                       
+│       learning_rate_monitor:                                                                                                                
+│         _target_: lightning.pytorch.callbacks.LearningRateMonitor                                                                           
+│         logging_interval: step                                                                                                              
+│                                                                                                                                             
+├── data
+│   └── train: lm1b                                                                                                                           
+│       valid: lm1b                                                                                                                           
+│       modality: text                                                                                                                        
+│       tokenizer_name_or_path: bert-base-uncased                                                                                             
+│       cache_dir: /share/kuleshov/ssahoo/textdiffusion/data                                                                                  
+│       wrap: true                                                                                                                            
+│       streaming: false                                                                                                                      
+│       insert_train_eos: true                                                                                                                
+│       insert_valid_eos: true                                                                                                                
+│                                                                                                                                             
+├── model
+│   └── name: small                                                                                                                           
+│       type: ddit                                                                                                                            
+│       hidden_size: 768                                                                                                                      
+│       cond_dim: 128                                                                                                                         
+│       length: 128                                                                                                                           
+│       n_blocks: 12                                                                                                                          
+│       n_heads: 12                                                                                                                           
+│       scale_by_sigma: true                                                                                                                  
+│       dropout: 0.1                                                                                                                          
+│       tie_word_embeddings: false                                                                                                            
+│       vocab_lookup: true                                                                                                                    
+│                                                                                                                                             
+├── strategy
+│   └── _target_: lightning.pytorch.strategies.DDPStrategy                                                                                    
+│       find_unused_parameters: false                                                                                                         
+│                                                                                                                                             
+├── noise
+│   └── type: log-linear                                                                                                                      
+│       eps: 0.001                                                                                                                            
+│                                                                                                                                             
+├── lr_scheduler
+│   └── _target_: transformers.get_constant_schedule_with_warmup                                                                              
+│       num_warmup_steps: 2500                                                                                                                
+│                                                                                                                                             
+├── prior
+│   └── type: none                                                                                                                            
+│       latent_width: 0                                                                                                                       
+│       latent_height: 0                                                                                                                      
+│                                                                                                                                             
+└── algo
+    └── name: duo_base                                                                                                                        
+        backbone: dit                                                                                                                         
+        parameterization: mean                                                                                                                
+        time_conditioning: true                                                                                                               
+        T: 0                                                                                                                                  
+        subs_masking: false                                                                                                                   
+        causal_attention: false                                                                                                               
+        ignore_bos: false                                                                                                                     
+        loss_type: elbo                                                                                                                       
+                                                                                                                                              

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstream_samples.json

@@ -0,0 +1,7 @@
+{
+    "generative_ppl": 94.98780822753906,
+    "entropy": 4.304296016693115,
+    "generated_seqs": [
+        "[CLS] west is an executive background / analysis. [CLS] the great challenge is competition between the two groups. [CLS] mr. rock is a true tourist. [CLS] parry's shot from close range was on target but headed into arreax. [CLS] the injured interpreter are patrolling the transfer zone. [CLS] however, middle - aged adults and children form small bands that have become more louder than other youths with metal. [CLS] built hope it create a transportation lane over a fairway. [CLS] one week ( 0148 895255 ). [CLS] coach of the philadelphia phillies team, link managers for drugs with parcells. [CLS] mr hopkins, who heads us [CLS]"
+    ]
+}

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstream_token_ids.json

@@ -0,0 +1,4 @@
+{
+  "samples": [],
+  "schema": "dlb-upstream-token-capture-v1"
+}

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstream_hydra/.hydra/config.yaml

@@ -0,0 +1,151 @@
+mode: sample_eval
+seed: 42
+loader:
+  global_batch_size: 512
+  eval_global_batch_size: ${.global_batch_size}
+  batch_size: 2
+  eval_batch_size: 1
+  num_workers: ${eval:"len(__import__('os').sched_getaffinity(0))"}
+  pin_memory: true
+sampling:
+  predictor: ancestral
+  steps: 4
+  noise_removal: ancestral
+  use_float64: true
+  p_nucleus: 1.0
+  num_sample_batches: 1
+  num_sample_log: 2
+  semi_ar: false
+  stride_length: 1
+  num_strides: 1
+  guid_weight: null
+  psi:
+    time_profile: linear
+    high_mode: pure-posterior
+    middle_mode: pure-posterior
+    low_mode: pure-posterior
+    high_frac: 0.2
+    middle_frac: 0.6
+training:
+  ema: 0.9999
+  antithetic_sampling: true
+  importance_sampling: false
+  sampling_eps: 0.001
+  change_of_variables: false
+  loss_precision: bf16
+  finetune_path: ''
+  class_dropout_p: 0.1
+eval:
+  checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt
+  disable_ema: false
+  compute_generative_perplexity: false
+  perplexity_batch_size: 8
+  compute_perplexity_on_sanity: false
+  gen_ppl_eval_model_name_or_path: gpt2-large
+  generate_samples: true
+  generated_samples_path: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstream_samples.json
+optim:
+  weight_decay: 0
+  lr: 0.0003
+  beta1: 0.9
+  beta2: 0.999
+  eps: 1.0e-08
+trainer:
+  _target_: lightning.Trainer
+  accelerator: cuda
+  num_nodes: 1
+  devices: 1
+  accumulate_grad_batches: ${div_up:${loader.global_batch_size}, ${eval:${trainer.devices}
+    * ${loader.batch_size} * ${trainer.num_nodes}}}
+  gradient_clip_val: 1.0
+  precision: bf16
+  num_sanity_val_steps: 2
+  max_steps: 1000000
+  log_every_n_steps: 100
+  limit_train_batches: 1.0
+  limit_val_batches: 1.0
+  val_check_interval: 5000
+  check_val_every_n_epoch: null
+wandb:
+  project: duo
+  group: null
+  job_type: null
+  name: null
+  id: ${.name}_${seed}
+  tags:
+  - ${noise.type}
+  - ${data.train}
+  - ${data.valid}
+  - ${algo.name}
+  offline: true
+checkpointing:
+  save_dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstream_hydra
+  resume_from_ckpt: true
+  resume_ckpt_path: ${.save_dir}/checkpoints/last.ckpt
+callbacks:
+  checkpoint_every_n_steps:
+    _target_: lightning.pytorch.callbacks.ModelCheckpoint
+    save_top_k: -1
+    save_last: true
+    dirpath: ${checkpointing.save_dir}/checkpoints
+    verbose: true
+    auto_insert_metric_name: false
+    every_n_train_steps: 500
+  checkpoint_monitor:
+    _target_: lightning.pytorch.callbacks.ModelCheckpoint
+    monitor: val/nll
+    mode: min
+    save_top_k: 1
+    save_last: false
+    dirpath: ${checkpointing.save_dir}/checkpoints
+    filename: best
+    auto_insert_metric_name: false
+    verbose: true
+  learning_rate_monitor:
+    _target_: lightning.pytorch.callbacks.LearningRateMonitor
+    logging_interval: step
+data:
+  train: lm1b
+  valid: lm1b
+  modality: text
+  tokenizer_name_or_path: bert-base-uncased
+  cache_dir: /share/kuleshov/ssahoo/textdiffusion/data
+  wrap: true
+  streaming: false
+  insert_train_eos: true
+  insert_valid_eos: true
+model:
+  name: small
+  type: ddit
+  hidden_size: 768
+  cond_dim: 128
+  length: 128
+  n_blocks: 12
+  n_heads: 12
+  scale_by_sigma: true
+  dropout: 0.1
+  tie_word_embeddings: false
+  vocab_lookup: true
+strategy:
+  _target_: lightning.pytorch.strategies.DDPStrategy
+  find_unused_parameters: false
+noise:
+  type: log-linear
+  eps: 0.001
+lr_scheduler:
+  _target_: transformers.get_constant_schedule_with_warmup
+  num_warmup_steps: 2500
+prior:
+  type: none
+  latent_width: 0
+  latent_height: 0
+algo:
+  name: duo_base
+  backbone: dit
+  parameterization: mean
+  time_conditioning: true
+  T: 0
+  subs_masking: false
+  causal_attention: false
+  ignore_bos: false
+  loss_type: elbo

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstream_hydra/.hydra/hydra.yaml

@@ -0,0 +1,181 @@
+hydra:
+  run:
+    dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstream_hydra
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstream_hydra
+    - hydra.mode=RUN
+    task:
+    - mode=sample_eval
+    - seed=42
+    - data=lm1b-wrap
+    - model=small
+    - model.length=128
+    - loader.batch_size=2
+    - loader.eval_batch_size=1
+    - sampling.num_sample_batches=1
+    - sampling.steps=4
+    - eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt
+    - trainer.devices=1
+    - checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstream_hydra
+    - +wandb.offline=true
+    - eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstream_samples.json
+    - algo=duo_base
+    - algo.backbone=dit
+    - sampling.predictor=ancestral
+    - sampling.noise_removal=ancestral
+    - eval.disable_ema=False
+  job:
+    name: dlb_pinned_upstream_main
+    chdir: true
+    override_dirname: +wandb.offline=true,algo.backbone=dit,algo=duo_base,checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstream_hydra,data=lm1b-wrap,eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt,eval.disable_ema=False,eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstream_samples.json,loader.batch_size=2,loader.eval_batch_size=1,mode=sample_eval,model.length=128,model=small,sampling.noise_removal=ancestral,sampling.num_sample_batches=1,sampling.predictor=ancestral,sampling.steps=4,seed=42,trainer.devices=1
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /home/ruizzhao/diffusion_baseline
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/ruizzhao/diffusion_baseline/upstreams/duo/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstream_hydra
+    choices:
+      algo: duo_base
+      prior: none
+      lr_scheduler: constant_warmup
+      noise: log-linear
+      strategy: ddp
+      model: small
+      data: lm1b-wrap
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstream_hydra/.hydra/overrides.yaml

@@ -0,0 +1,19 @@
+- mode=sample_eval
+- seed=42
+- data=lm1b-wrap
+- model=small
+- model.length=128
+- loader.batch_size=2
+- loader.eval_batch_size=1
+- sampling.num_sample_batches=1
+- sampling.steps=4
+- eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt
+- trainer.devices=1
+- checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstream_hydra
+- +wandb.offline=true
+- eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstream_samples.json
+- algo=duo_base
+- algo.backbone=dit
+- sampling.predictor=ancestral
+- sampling.noise_removal=ancestral
+- eval.disable_ema=False

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstream_hydra/dlb_pinned_upstream_main.log

@@ -0,0 +1,4 @@
+[2026-08-25 10:41:48,631][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
+[2026-08-25 10:41:56,317][dlb_pinned_upstream_main][INFO] - Generative perplexity: 324.6085510253906
+[2026-08-25 10:41:56,317][dlb_pinned_upstream_main][INFO] - Sample entropy: 4.237051963806152
+[2026-08-25 10:41:56,318][dlb_pinned_upstream_main][INFO] - Samples saved at: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstream_samples.json

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstream_samples.json

@@ -0,0 +1,7 @@
+{
+    "generative_ppl": 324.6085510253906,
+    "entropy": 4.237051963806152,
+    "generated_seqs": [
+        "[CLS] son at dinner having wonderful - - as i recall the marathoner - - candidate no long offer - - tutored him in the businessla'[CLS] said ioanni. [CLS] for thegame time trading, the stx india5 star offer. 92 and or increase of 1 percent, 8. 09 associated. the recording, ore., reported. [CLS] my back to emily of the weekend, san marco around the city would not foolishly contemplating the kind of violence on far away1's \" manage. \" [CLS] \" it is square or there will be 50in ( 48km ) ) of space to prevent the indoor [CLS]"
+    ]
+}

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstream_token_ids.json

@@ -0,0 +1,4 @@
+{
+  "samples": [],
+  "schema": "dlb-upstream-token-capture-v1"
+}

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_8/upstream_hydra/.hydra/config.yaml

@@ -0,0 +1,151 @@
+mode: sample_eval
+seed: 42
+loader:
+  global_batch_size: 512
+  eval_global_batch_size: ${.global_batch_size}
+  batch_size: 2
+  eval_batch_size: 1
+  num_workers: ${eval:"len(__import__('os').sched_getaffinity(0))"}
+  pin_memory: true
+sampling:
+  predictor: ancestral
+  steps: 8
+  noise_removal: ancestral
+  use_float64: true
+  p_nucleus: 1.0
+  num_sample_batches: 1
+  num_sample_log: 2
+  semi_ar: false
+  stride_length: 1
+  num_strides: 1
+  guid_weight: null
+  psi:
+    time_profile: linear
+    high_mode: pure-posterior
+    middle_mode: pure-posterior
+    low_mode: pure-posterior
+    high_frac: 0.2
+    middle_frac: 0.6
+training:
+  ema: 0.9999
+  antithetic_sampling: true
+  importance_sampling: false
+  sampling_eps: 0.001
+  change_of_variables: false
+  loss_precision: bf16
+  finetune_path: ''
+  class_dropout_p: 0.1
+eval:
+  checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt
+  disable_ema: false
+  compute_generative_perplexity: false
+  perplexity_batch_size: 8
+  compute_perplexity_on_sanity: false
+  gen_ppl_eval_model_name_or_path: gpt2-large
+  generate_samples: true
+  generated_samples_path: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_8/upstream_samples.json
+optim:
+  weight_decay: 0
+  lr: 0.0003
+  beta1: 0.9
+  beta2: 0.999
+  eps: 1.0e-08
+trainer:
+  _target_: lightning.Trainer
+  accelerator: cuda
+  num_nodes: 1
+  devices: 1
+  accumulate_grad_batches: ${div_up:${loader.global_batch_size}, ${eval:${trainer.devices}
+    * ${loader.batch_size} * ${trainer.num_nodes}}}
+  gradient_clip_val: 1.0
+  precision: bf16
+  num_sanity_val_steps: 2
+  max_steps: 1000000
+  log_every_n_steps: 100
+  limit_train_batches: 1.0
+  limit_val_batches: 1.0
+  val_check_interval: 5000
+  check_val_every_n_epoch: null
+wandb:
+  project: duo
+  group: null
+  job_type: null
+  name: null
+  id: ${.name}_${seed}
+  tags:
+  - ${noise.type}
+  - ${data.train}
+  - ${data.valid}
+  - ${algo.name}
+  offline: true
+checkpointing:
+  save_dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_8/upstream_hydra
+  resume_from_ckpt: true
+  resume_ckpt_path: ${.save_dir}/checkpoints/last.ckpt
+callbacks:
+  checkpoint_every_n_steps:
+    _target_: lightning.pytorch.callbacks.ModelCheckpoint
+    save_top_k: -1
+    save_last: true
+    dirpath: ${checkpointing.save_dir}/checkpoints
+    verbose: true
+    auto_insert_metric_name: false
+    every_n_train_steps: 500
+  checkpoint_monitor:
+    _target_: lightning.pytorch.callbacks.ModelCheckpoint
+    monitor: val/nll
+    mode: min
+    save_top_k: 1
+    save_last: false
+    dirpath: ${checkpointing.save_dir}/checkpoints
+    filename: best
+    auto_insert_metric_name: false
+    verbose: true
+  learning_rate_monitor:
+    _target_: lightning.pytorch.callbacks.LearningRateMonitor
+    logging_interval: step
+data:
+  train: lm1b
+  valid: lm1b
+  modality: text
+  tokenizer_name_or_path: bert-base-uncased
+  cache_dir: /share/kuleshov/ssahoo/textdiffusion/data
+  wrap: true
+  streaming: false
+  insert_train_eos: true
+  insert_valid_eos: true
+model:
+  name: small
+  type: ddit
+  hidden_size: 768
+  cond_dim: 128
+  length: 128
+  n_blocks: 12
+  n_heads: 12
+  scale_by_sigma: true
+  dropout: 0.1
+  tie_word_embeddings: false
+  vocab_lookup: true
+strategy:
+  _target_: lightning.pytorch.strategies.DDPStrategy
+  find_unused_parameters: false
+noise:
+  type: log-linear
+  eps: 0.001
+lr_scheduler:
+  _target_: transformers.get_constant_schedule_with_warmup
+  num_warmup_steps: 2500
+prior:
+  type: none
+  latent_width: 0
+  latent_height: 0
+algo:
+  name: duo_base
+  backbone: dit
+  parameterization: mean
+  time_conditioning: true
+  T: 0
+  subs_masking: false
+  causal_attention: false
+  ignore_bos: false
+  loss_type: elbo

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_8/upstream_hydra/.hydra/hydra.yaml

@@ -0,0 +1,181 @@
+hydra:
+  run:
+    dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_8/upstream_hydra
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_8/upstream_hydra
+    - hydra.mode=RUN
+    task:
+    - mode=sample_eval
+    - seed=42
+    - data=lm1b-wrap
+    - model=small
+    - model.length=128
+    - loader.batch_size=2
+    - loader.eval_batch_size=1
+    - sampling.num_sample_batches=1
+    - sampling.steps=8
+    - eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt
+    - trainer.devices=1
+    - checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_8/upstream_hydra
+    - +wandb.offline=true
+    - eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_8/upstream_samples.json
+    - algo=duo_base
+    - algo.backbone=dit
+    - sampling.predictor=ancestral
+    - sampling.noise_removal=ancestral
+    - eval.disable_ema=False
+  job:
+    name: dlb_pinned_upstream_main
+    chdir: true
+    override_dirname: +wandb.offline=true,algo.backbone=dit,algo=duo_base,checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_8/upstream_hydra,data=lm1b-wrap,eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt,eval.disable_ema=False,eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_8/upstream_samples.json,loader.batch_size=2,loader.eval_batch_size=1,mode=sample_eval,model.length=128,model=small,sampling.noise_removal=ancestral,sampling.num_sample_batches=1,sampling.predictor=ancestral,sampling.steps=8,seed=42,trainer.devices=1
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /home/ruizzhao/diffusion_baseline
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/ruizzhao/diffusion_baseline/upstreams/duo/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_8/upstream_hydra
+    choices:
+      algo: duo_base
+      prior: none
+      lr_scheduler: constant_warmup
+      noise: log-linear
+      strategy: ddp
+      model: small
+      data: lm1b-wrap
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_8/upstream_hydra/.hydra/overrides.yaml

@@ -0,0 +1,19 @@
+- mode=sample_eval
+- seed=42
+- data=lm1b-wrap
+- model=small
+- model.length=128
+- loader.batch_size=2
+- loader.eval_batch_size=1
+- sampling.num_sample_batches=1
+- sampling.steps=8
+- eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt
+- trainer.devices=1
+- checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_8/upstream_hydra
+- +wandb.offline=true
+- eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_8/upstream_samples.json
+- algo=duo_base
+- algo.backbone=dit
+- sampling.predictor=ancestral
+- sampling.noise_removal=ancestral
+- eval.disable_ema=False

results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_8/upstream_hydra/dlb_pinned_upstream_main.log

@@ -0,0 +1,4 @@
+[2026-08-25 10:43:42,844][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
+[2026-08-25 10:43:51,931][dlb_pinned_upstream_main][INFO] - Generative perplexity: 264.6139221191406
+[2026-08-25 10:43:51,931][dlb_pinned_upstream_main][INFO] - Sample entropy: 4.380173683166504
+[2026-08-25 10:43:51,931][dlb_pinned_upstream_main][INFO] - Samples saved at: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_8/upstream_samples.json

scripts/aggregate_conditional_results.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+"""Publish strict or partial C64 conditional baseline summary tables."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from dlb.aggregate import IncompleteMatrixError
+from dlb.conditional_aggregate import aggregate_conditional
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--root", type=Path, default=Path.cwd())
+    parser.add_argument(
+        "--output-dir",
+        type=Path,
+        default=None,
+        help="defaults to ROOT/results/conditional/summary",
+    )
+    parser.add_argument("--partial", action="store_true")
+    args = parser.parse_args(argv)
+    root = args.root.resolve()
+    output = (args.output_dir or root / "results/conditional/summary").resolve()
+    try:
+        report = aggregate_conditional(
+            root,
+            strict=not args.partial,
+            partial=args.partial,
+            output_dir=output,
+        )
+    except IncompleteMatrixError as error:
+        print(str(error))
+        print(f"failures={len(error.failures)}")
+        return 1
+    print(
+        f"complete={str(report.complete).lower()} "
+        f"rows={len(report.rows)} failures={len(report.failures)} "
+        f"unsupported={len(report.unsupported)} output={output}"
+    )
+    return 0 if report.complete else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/build_conditional_prompts.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+"""Build deterministic C64 conditional prompt artifacts."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import sys
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1])
+    parser.add_argument("--config", type=Path)
+    parser.add_argument("--dataset", choices=("all", "lm1b", "owt"), default="all")
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(argv)
+    root = args.root.resolve()
+    sys.path.insert(0, str(root / "src"))
+    from dlb.conditional_prompts import build_prompts, load_protocol
+
+    protocol = load_protocol(args.config or root / "configs" / "conditional.yaml")
+    datasets = ("lm1b", "owt") if args.dataset == "all" else (args.dataset,)
+    for dataset_id in datasets:
+        manifest = build_prompts(root, dataset_id, protocol)
+        print(f"OK {dataset_id} prompts={manifest.prompt_count} sha256={manifest.prompt_file_sha256}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/evaluate_conditional_all.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# Server-side quality evaluation over completed C64 conditional samples.
+set -uo pipefail
+
+SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
+DLB_ROOT="${DLB_ROOT:-$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd -P)}"
+PYTHON_BIN="${DLB_EVAL_PYTHON:-${DLB_PYTHON:-python}}"
+MATRIX="${DLB_CONDITIONAL_MATRIX:-}"
+METRICS="conditional_gen_ppl,mauve_suffix,entropy,self_bleu,prefix_exact_match"
+
+usage() {
+  printf 'Usage: %s [--root PATH] [--matrix PATH] [--metrics LIST]\n' "$0"
+}
+
+while (($#)); do
+  case "$1" in
+    --root) DLB_ROOT="$2"; shift 2 ;;
+    --matrix) MATRIX="$2"; shift 2 ;;
+    --metrics) METRICS="$2"; shift 2 ;;
+    --help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+DLB_ROOT="$(CDPATH= cd -- "$DLB_ROOT" && pwd -P)"
+if [[ -z "$MATRIX" ]]; then MATRIX="$DLB_ROOT/results/conditional/matrix/generation.tsv"; fi
+export DLB_ROOT
+mkdir -p "$DLB_ROOT/results/conditional/matrix" "$DLB_ROOT/results/logs"
+PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "${DLB_PYTHON:-python}" -m dlb.conditional_matrix \
+  --root "$DLB_ROOT" --output "$MATRIX" || exit $?
+
+failures="$DLB_ROOT/results/logs/conditional_evaluation_failures.tsv"
+printf 'task_id\tmodel\tdataset\tsteps\texit_code\n' >"$failures"
+status=0
+while IFS=$'\t' read -r task_id category model dataset steps sample_count seed environment adapter source provenance protocol conditioning_manifest conditioning_manifest_sha256 sample_dir metrics_path timing_path; do
+  [[ "$task_id" == "task_id" || -z "$task_id" ]] && continue
+  mkdir -p "$(dirname -- "$metrics_path")"
+  echo "CONDITIONAL EVALUATE $task_id"
+  PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m evaluation.conditional_evaluate \
+    --root "$DLB_ROOT" --samples "$sample_dir/samples.jsonl" \
+    --metrics "$METRICS" --dataset "$dataset" --output "$metrics_path"
+  exit_code=$?
+  if ((exit_code != 0)); then
+    printf '%s\t%s\t%s\t%s\t%s\n' "$task_id" "$model" "$dataset" "$steps" "$exit_code" >>"$failures"
+    status=1
+  fi
+done < <(tail -n +3 "$MATRIX")
+if [[ ! -s "$failures" ]]; then rm -f "$failures"; fi
+exit "$status"

scripts/fetch_sources.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+DLB_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+export DLB_ROOT
+
+resolve_python_candidate() {
+    local candidate="$1"
+    if [[ "$candidate" == */* ]]; then
+        [[ -x "$candidate" ]] && printf '%s\n' "$candidate"
+    elif command -v "$candidate" >/dev/null 2>&1; then
+        command -v "$candidate"
+    fi
+}
+
+if [[ -n "${DLB_PYTHON:-}" ]]; then
+    if ! DLB_PYTHON_BIN=$(resolve_python_candidate "$DLB_PYTHON"); then
+        echo "ERROR Python interpreter not found or not executable: $DLB_PYTHON" >&2
+        exit 1
+    fi
+elif [[ -n "${PYTHON_BIN:-}" ]]; then
+    if ! DLB_PYTHON_BIN=$(resolve_python_candidate "$PYTHON_BIN"); then
+        echo "ERROR Python interpreter not found or not executable: $PYTHON_BIN" >&2
+        exit 1
+    fi
+elif [[ -n "${CONDA_PREFIX:-}" && -x "$CONDA_PREFIX/bin/python" ]]; then
+    DLB_PYTHON_BIN="$CONDA_PREFIX/bin/python"
+elif command -v python >/dev/null 2>&1; then
+    DLB_PYTHON_BIN=$(command -v python)
+elif command -v python3 >/dev/null 2>&1; then
+    DLB_PYTHON_BIN=$(command -v python3)
+else
+    echo "ERROR Python interpreter not found; activate conda or set DLB_PYTHON." >&2
+    exit 1
+fi
+
+source_rows=$(PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$DLB_PYTHON_BIN" - "$DLB_ROOT" <<'PY'
+from pathlib import Path
+import sys
+
+import yaml
+
+from dlb.registry import load_registry
+
+root = Path(sys.argv[1])
+with (root / "artifacts/sources.yaml").open(encoding="utf-8") as manifest_file:
+    manifest = yaml.safe_load(manifest_file)
+required_sources = {entry.source for entry in load_registry(root / "configs/experiments.yaml").models.values()}
+if not isinstance(manifest, dict) or set(manifest) != required_sources:
+    raise SystemExit("source manifest IDs must match registry sources")
+for name in sorted(required_sources):
+    source = manifest[name]
+    if not isinstance(source, dict) or set(source) != {"url", "commit"}:
+        raise SystemExit(f"invalid manifest entry for {name}")
+    print(f"{name}\t{source['url']}\t{source['commit']}")
+PY
+)
+
+while IFS=$'\t' read -r name url commit; do
+    source_dir="$DLB_ROOT/upstreams/$name"
+    if [[ ! -e "$source_dir" ]]; then
+        git clone --filter=blob:none "$url" "$source_dir"
+    elif [[ ! -d "$source_dir/.git" ]]; then
+        echo "ERROR $name is not a Git repository: $source_dir" >&2
+        exit 1
+    fi
+
+    if [[ "$(git -C "$source_dir" remote get-url origin)" != "$url" ]]; then
+        echo "ERROR $name origin does not match manifest" >&2
+        exit 1
+    fi
+    if [[ -n "$(git -C "$source_dir" status --porcelain)" ]]; then
+        echo "ERROR $name has uncommitted changes" >&2
+        exit 1
+    fi
+
+    git -C "$source_dir" fetch --depth=1 origin "$commit"
+    git -C "$source_dir" checkout --detach "$commit"
+    test "$(git -C "$source_dir" rev-parse HEAD)" = "$commit"
+    test -z "$(git -C "$source_dir" status --porcelain)"
+done <<< "$source_rows"
+
+PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" \
+    "$DLB_PYTHON_BIN" "$DLB_ROOT/scripts/verify_sources.py" --root "$DLB_ROOT"

scripts/run_all.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Server-side serial/resumable generation over the canonical matrix.
+set -uo pipefail
+
+SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
+DLB_ROOT="${DLB_ROOT:-$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd -P)}"
+PYTHON_BIN="${DLB_PYTHON:-python}"
+MATRIX="${DLB_MATRIX:-}"
+SAMPLE_COUNT=1024
+SEED=42
+
+usage() {
+  printf 'Usage: %s [--root PATH] [--matrix PATH] [--num-samples N] [--seed N]\n' "$0"
+}
+
+while (($#)); do
+  case "$1" in
+    --root) DLB_ROOT="$2"; shift 2 ;;
+    --matrix) MATRIX="$2"; shift 2 ;;
+    --num-samples) SAMPLE_COUNT="$2"; shift 2 ;;
+    --seed) SEED="$2"; shift 2 ;;
+    --help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+DLB_ROOT="$(CDPATH= cd -- "$DLB_ROOT" && pwd -P)"
+if [[ -z "$MATRIX" ]]; then MATRIX="$DLB_ROOT/results/matrix/generation.tsv"; fi
+export DLB_ROOT
+mkdir -p "$DLB_ROOT/results/matrix" "$DLB_ROOT/results/logs"
+PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m dlb.matrix \
+  --root "$DLB_ROOT" --output "$MATRIX" --sample-count "$SAMPLE_COUNT" --seed "$SEED" \
+  || exit $?
+
+failures="$DLB_ROOT/results/logs/generation_failures.tsv"
+printf 'task_id\tmodel\tdataset\tsteps\texit_code\n' >"$failures"
+status=0
+while IFS=$'\t' read -r task_id category model dataset steps sample_count seed environment adapter source provenance sample_dir metrics_path timing_path; do
+  [[ "$task_id" == "task_id" || -z "$task_id" ]] && continue
+  echo "GENERATE $task_id"
+  "$SCRIPT_DIR/run_one.sh" \
+    --model "$model" --dataset "$dataset" --steps "$steps" \
+    --num-samples "$sample_count" --seed "$seed"
+  exit_code=$?
+  if ((exit_code != 0)); then
+    printf '%s\t%s\t%s\t%s\t%s\n' "$task_id" "$model" "$dataset" "$steps" "$exit_code" >>"$failures"
+    status=1
+  fi
+done < <(tail -n +3 "$MATRIX")
+if [[ ! -s "$failures" ]]; then
+  rm -f "$failures"
+fi
+exit "$status"

scripts/run_ppl_rescore_20260923.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$root" || exit 1
+mode="${1:-all}"
+out="$root/results/ppl_rescore_20260923"
+mkdir -p "$out/logs"
+export CUDA_VISIBLE_DEVICES=3
+export PYTHONPATH="$root/src:$root"
+export HF_HOME="/home/ruizzhao/.cache/huggingface"
+export HF_HUB_DISABLE_TELEMETRY=1
+export TOKENIZERS_PARALLELISM=false
+export OMP_NUM_THREADS=4
+python=/opt/anaconda3/envs/factory/bin/python
+
+if [[ "$mode" == smoke ]]; then
+  smoke_model="${2:-pythia-1b}"
+  log="$out/logs/smoke_${smoke_model}.log"
+  exit_file="$out/logs/smoke_${smoke_model}.exit"
+  args=(--model "$smoke_model" --smoke-samples 2 --batch-size 2)
+elif [[ "$mode" == all ]]; then
+  log="$out/logs/all.log"
+  exit_file="$out/logs/all.exit"
+  args=(--model all --batch-size 8)
+else
+  echo "usage: $0 [smoke|all]" >&2
+  exit 2
+fi
+
+rm -f "$exit_file"
+{
+  printf 'started_utc=%s\nmode=%s\ngpu=%s\npython=%s\n' \
+    "$(date -u +%FT%TZ)" "$mode" "$CUDA_VISIBLE_DEVICES" "$python"
+  "$python" -B -u scripts/rescore_unconditional_ppl.py "${args[@]}"
+} >"$log" 2>&1
+status=$?
+printf '%s\n' "$status" >"$exit_file"
+exit "$status"

scripts/smoke_conditional_all.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Conditional smoke pass. It still uses the full 2,048-record C64 schedule,
+# because partial conditional artifacts are not comparable to production runs.
+set -uo pipefail
+
+SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
+DLB_ROOT="${DLB_ROOT:-$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd -P)}"
+PYTHON_BIN="${DLB_PYTHON:-python}"
+MATRIX="${DLB_ROOT}/results/conditional/matrix/smoke.tsv"
+if [[ "${1:-}" == "--help" ]]; then
+  printf 'Usage: %s\n' "$0"
+  exit 0
+fi
+mkdir -p "$DLB_ROOT/results/conditional/matrix" "$DLB_ROOT/results/logs"
+PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m dlb.conditional_matrix \
+  --root "$DLB_ROOT" --output "$MATRIX" || exit $?
+
+status=0
+while IFS=$'\t' read -r task_id category model dataset steps sample_count seed environment adapter source provenance protocol conditioning_manifest conditioning_manifest_sha256 sample_dir metrics_path timing_path; do
+  [[ "$task_id" == "task_id" || -z "$task_id" ]] && continue
+  echo "CONDITIONAL SMOKE $task_id"
+  "$SCRIPT_DIR/run_conditional_one.sh" --model "$model" --dataset "$dataset" --steps "$steps" \
+    --seed "$seed" --results-root "$DLB_ROOT/results/conditional/smoke"
+  if (($? != 0)); then status=1; fi
+done < <(tail -n +3 "$MATRIX")
+exit "$status"

verify_duo_mdlm.log

@@ -0,0 +1,2 @@
+{"cuda_available": true, "environment": "dlb-duo", "imports": {"datasets": true, "einops": true, "flash_attn": true, "fsspec": true, "h5py": true, "huggingface_hub": true, "hydra": true, "lightning": true, "numpy": true, "omegaconf": true, "requests": true, "rich": true, "scipy": true, "timm": true, "tokenizers": true, "torchmetrics": true, "torchvision": true, "tqdm": true, "transformers": true, "triton": true, "wandb": true}, "python": "3.12.13", "torch": "2.5.1", "torch_cuda": "12.4"}
+{"cuda_available": true, "environment": "dlb-mdlm", "imports": {"causal_conv1d": true, "datasets": true, "einops": true, "flash_attn": true, "fsspec": true, "huggingface_hub": true, "hydra": true, "lightning": true, "mamba_ssm": true, "numpy": true, "omegaconf": true, "requests": true, "rich": true, "timm": true, "tokenizers": true, "torchmetrics": true, "transformers": true, "wandb": true}, "python": "3.9.23", "torch": "2.5.1", "torch_cuda": "12.4"}