Add files using upload-large-folder tool

checkpoints/quarantine/20260802T040637Z/official/mdlm/owt/configuration_mdlm.py

@@ -0,0 +1,31 @@
+"""MDLM config for Hugging Face.
+
+"""
+
+import transformers
+
+
+class MDLMConfig(transformers.PretrainedConfig):
+  """Hugging Face configuration class for MDLM."""
+  model_type = "mdlm"
+
+  def __init__(
+    self,
+    vocab_size: int = 50258,
+    model_length: int = 1024,
+    hidden_dim: int = 768,
+    cond_dim: int = 129,
+    n_blocks: int = 12,
+    n_heads: int = 12,
+    dropout: float = 0.1,
+    time_conditioning: bool = False,
+    ** kwargs):
+    super().__init__(**kwargs)
+    self.vocab_size = vocab_size
+    self.model_length = model_length
+    self.hidden_dim = hidden_dim
+    self.cond_dim = cond_dim
+    self.n_blocks = n_blocks
+    self.n_heads = n_heads
+    self.dropout = dropout
+    self.time_conditioning = time_conditioning

checkpoints/quarantine/20260818T073943Z/official/flm/lm1b/config.json

@@ -0,0 +1,209 @@
+{
+  "config": {
+    "algo": {
+      "T": 0,
+      "backbone": "dit",
+      "causal_attention": false,
+      "curriculum_end": 1000000,
+      "curriculum_start": 0,
+      "double_temb": false,
+      "flow_loss_type": "ce",
+      "flow_ratio": 0.75,
+      "gumbel_tau_log10_end": -0.0,
+      "gumbel_tau_log10_start": -0.0,
+      "ignore_bos": false,
+      "jvp_api": "autograd",
+      "loss_scale": "none",
+      "n_separated_blocks": -1,
+      "name": "dos",
+      "parameterization": "mean",
+      "pred_log_interval": 2000,
+      "sigma_min": 0.0001,
+      "subs_masking": false,
+      "t_max": 1.0,
+      "t_min": 0.0,
+      "time_condition": "alpha_t",
+      "time_conditioning": true,
+      "use_curriculum": true,
+      "use_discrete_schedule": true
+    },
+    "callbacks": {
+      "checkpoint_every_n_steps": {
+        "_target_": "lightning.pytorch.callbacks.ModelCheckpoint",
+        "auto_insert_metric_name": false,
+        "dirpath": "/home/david3684/projects/flm/hf_upload/checkpoints",
+        "every_n_train_steps": 50000,
+        "save_last": false,
+        "save_top_k": -1,
+        "verbose": true
+      },
+      "checkpoint_monitor": {
+        "_target_": "lightning.pytorch.callbacks.ModelCheckpoint",
+        "auto_insert_metric_name": false,
+        "dirpath": "/home/david3684/projects/flm/hf_upload/checkpoints",
+        "filename": "best_nll",
+        "mode": "min",
+        "monitor": "val/nll",
+        "save_last": true,
+        "save_top_k": 1,
+        "verbose": true
+      },
+      "learning_rate_monitor": {
+        "_target_": "lightning.pytorch.callbacks.LearningRateMonitor",
+        "logging_interval": "step"
+      }
+    },
+    "checkpointing": {
+      "resume_ckpt_path": "/home/work/RADAR/workspace/KAIST/discrete-mean-flow/text/outputs/lm1b/2026.01.05/154016/checkpoints/47-650000.ckpt",
+      "resume_from_ckpt": true,
+      "save_dir": "/home/david3684/projects/flm/hf_upload"
+    },
+    "data": {
+      "cache_dir": "/home/work/RADAR/workspace/KAIST/cdBDD/datasets/lm1b",
+      "insert_train_eos": true,
+      "insert_valid_eos": true,
+      "streaming": false,
+      "tokenizer_name_or_path": "bert-base-uncased",
+      "train": "lm1b",
+      "valid": "lm1b",
+      "wrap": true
+    },
+    "eval": {
+      "checkpoint_path": "",
+      "compute_generative_perplexity": true,
+      "compute_perplexity_on_sanity": false,
+      "disable_ema": false,
+      "gen_ppl_eval_model_name_or_path": "gpt2-large",
+      "generate_samples": true,
+      "generated_samples_path": "/home/david3684/projects/flm/hf_upload/samples.json",
+      "perplexity_batch_size": 8
+    },
+    "is_di4c": false,
+    "is_di4c_deterministic": true,
+    "loader": {
+      "batch_size": 128,
+      "eval_batch_size": 128,
+      "eval_global_batch_size": 512,
+      "global_batch_size": 512,
+      "num_workers": 192,
+      "pin_memory": true
+    },
+    "lr_scheduler": {
+      "_target_": "transformers.get_constant_schedule_with_warmup",
+      "num_warmup_steps": 2500
+    },
+    "mode": "train",
+    "model": {
+      "cond_dim": 128,
+      "dropout": 0.1,
+      "hidden_size": 768,
+      "length": 128,
+      "n_blocks": 12,
+      "n_heads": 12,
+      "name": "small",
+      "scale_by_sigma": true,
+      "tie_word_embeddings": false,
+      "type": "ddit",
+      "vocab_lookup": true
+    },
+    "noise": {
+      "denoiser_latent_conditioning": -1,
+      "eps": 0,
+      "freeze_decoder": false,
+      "freeze_encoder": false,
+      "parameterization": "log-linear",
+      "type": "log-linear"
+    },
+    "optim": {
+      "beta1": 0.9,
+      "beta2": 0.999,
+      "eps": 1e-08,
+      "ln_tune": "none",
+      "lr": 0.0003,
+      "use_muon": false,
+      "weight_decay": 0
+    },
+    "prior": {
+      "latent_height": 0,
+      "latent_width": 0,
+      "type": "none"
+    },
+    "sampling": {
+      "argmax_correction": false,
+      "hard_start": false,
+      "noise_removal": "meanflow",
+      "noise_removal_list": [
+        "uniform",
+        "uniform_alpha"
+      ],
+      "num_reflow_samples": 50000,
+      "num_sample_batches": 1,
+      "num_sample_log": 2,
+      "num_strides": 1,
+      "p_nucleus": 1.0,
+      "predictor": "ancestral",
+      "semi_ar": false,
+      "solver": "euler",
+      "steps": [
+        1024
+      ],
+      "stride_length": 1,
+      "tau_log10": 0.0,
+      "temperature": 1.0,
+      "use_float64": true
+    },
+    "seed": 1,
+    "strategy": {
+      "_target_": "lightning.pytorch.strategies.DDPStrategy",
+      "find_unused_parameters": true
+    },
+    "trainer": {
+      "_target_": "lightning.Trainer",
+      "accelerator": "cuda",
+      "accumulate_grad_batches": 1,
+      "devices": 8,
+      "gradient_clip_val": 1.0,
+      "limit_train_batches": 1.0,
+      "limit_val_batches": 1.0,
+      "log_every_n_steps": 100,
+      "max_steps": 1000000,
+      "num_nodes": 1,
+      "num_sanity_val_steps": 2,
+      "precision": "bf16",
+      "val_check_interval": 10000
+    },
+    "training": {
+      "antithetic_sampling": true,
+      "change_of_variables": false,
+      "double_ema": 0.0,
+      "ema": 0.9999,
+      "finetune_path": "",
+      "importance_sampling": false,
+      "load_ema": false,
+      "loss_precision": "bf16",
+      "loss_type": "flow",
+      "loss_weighting": false,
+      "not_sampling_t": false,
+      "pred_type": "x0",
+      "sampling_eps": 0.001,
+      "t_curriculum_max": 1.0,
+      "t_curriculum_min": 0.0,
+      "t_curriculum_steps": 100000,
+      "torch_compile_mode": "default",
+      "use_torch_compile": true
+    },
+    "wandb": {
+      "group": null,
+      "id": "fc867e70",
+      "job_type": null,
+      "name": "lm1b_full_flow_no_input_softmax_no_loss_scale_ce_loss_fc867e70",
+      "notes": "DOS",
+      "project": "lm1b_full",
+      "tags": [
+        "lm1b",
+        "dos"
+      ]
+    }
+  },
+  "vocab_size": 30522
+}

checkpoints/quarantine/20260818T074044Z/official/fmlm/lm1b/README.md

@@ -0,0 +1,45 @@
+---
+tags:
+- model_hub_mixin
+- pytorch_model_hub_mixin
+license: apache-2.0
+language:
+- en
+base_model:
+- david3684/FLM-B-LM1B
+---
+  
+  ## FMLM (LM1B)
+
+  For more information and implementation details, visit our github repository: [FLM](https://github.com/david3684/flm).
+
+  ## Model Details
+
+  The model has a context length of 128, distilled from 1M steps trained FLM checkpoint trained on the One Billion Words (LM1B) dataset.
+  
+  For more details, please see our paper: 
+  - [arXiv:2602.16813](https://arxiv.org/abs/2602.16813)
+  - [Huggingface Papers](https://huggingface.co/papers/2602.16813)
+
+  ## Citation
+
+  Please cite our work using the bibtex below:
+
+  **BibTeX:**
+
+
+  ```bibtex
+  @article{lee2026flow,
+      title={Flow Map Language Models: One-step Language Modeling via Continuous Denoising},
+      author={Chanhyuk Lee and Jaehoon Yoo and Manan Agarwal
+              and Sheel Shah and Jerry Huang
+              and Aditi Raghunathan and Seunghoon Hong
+              and Nicholas M. Boffi and Jinwoo Kim},
+      journal={arXiv preprint arXiv:2602.16813},
+      year={2026},
+  }
+  ```
+
+  ## Model Card Contact
+
+  Chanhyuk Lee (chanhyuk-lee@kaist.ac.kr)

checkpoints/quarantine/20260818T074500Z/official/langflow/owt/README.md

@@ -0,0 +1,61 @@
+---
+datasets:
+- Skylion007/openwebtext
+papers:
+- arxiv: 2604.11748
+language:
+- en
+library_name: transformers
+license: apache-2.0
+metrics:
+- perplexity
+pipeline_tag: text-generation
+---
+
+# LangFlow
+
+LangFlow is a continuous diffusion language model that operates in embedding space. Unlike discrete diffusion models (MDLM, SEDD, DUO), LangFlow performs diffusion directly on continuous token embeddings, enabling smoother denoising dynamics.
+
+For more details, please see our paper: [LangFlow: Continuous Diffusion Rivals Discrete in Language Modeling](https://arxiv.org/abs/2604.11748).
+
+
+## Using LangFlow
+
+To use the pre-trained model for text generation, use the following snippet:
+
+```python
+from transformers import AutoModelForMaskedLM, AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained('gpt2')
+model = AutoModelForMaskedLM.from_pretrained('chumengl/langflow-owt', trust_remote_code=True)
+
+# Generate samples
+samples = model.generate_samples(num_samples=5, num_steps=128)
+texts = tokenizer.batch_decode(samples, skip_special_tokens=True)
+for text in texts:
+    print(text)
+```
+
+## Model Details
+
+- **Architecture**: DiT (Diffusion Transformer) backbone with adaptive layer normalization
+- **Context Length**: 1024 tokens
+- **Parameters**: ~130M non-embedding parameters (similar to GPT-2 medium)
+- **Training**: 1M steps on OpenWebText corpus
+- **Tokenizer**: GPT-2 tokenizer (50,257 vocab size)
+
+## Citation
+
+```
+@article{chen2026langflow,
+  title={LangFlow: Continuous Diffusion Rivals Discrete in Language Modeling},
+  author={Chen, Yuxin and Liang, Chumeng and Sui, Hangke and Guo, Ruihan and Cheng, Chaoran and You, Jiaxuan and Liu, Ge},
+  journal={arXiv preprint arXiv:2604.11748},
+  year={2026}
+}
+```
+
+## Model Card Contact
+
+Chumeng Liang (chumengl@illinois.edu)
+

checkpoints/quarantine/20260818T074854Z/official/duo_dcd/owt/README.md

@@ -0,0 +1,52 @@
+---
+datasets:
+- Skylion007/openwebtext
+language:
+- en
+library_name: transformers
+license: apache-2.0
+metrics:
+- perplexity
+pipeline_tag: text-generation
+---
+
+## Using DUO
+To use the pre-trained model for masked language modeling, use the following snippet:
+```python
+from transformers import AutoModelForMaskedLM, AutoTokenizer
+
+# See the `DUO` collection page on the hub for list of available models.
+tokenizer = transformers.AutoTokenizer.from_pretrained('gpt2')
+model = AutoModelForMaskedLM.from_pretrained('s-sahoo/duo-distilled')
+```
+For a hands-on example, check out this [Colab notebook](https://colab.research.google.com/drive/1Sf7R-dqdR6gq-H8nyZ9E3ZkyvqMTqcwq?usp=sharing).
+For more information and implementation details, visit our github repository: [DUO](https://github.com/s-sahoo/duo)
+
+## Model Details
+The model, which has a context length of `1024` and is similar in size to GPT2-medium with approximately `130 million` non-embedding parameters,
+was trained for 1M steps on the OpenWebText corpus.
+
+For more details, please see our paper: [The Diffusion Duality](https://huggingface.co/papers/2506.10892).
+
+Project page: https://s-sahoo.com/duo
+
+## Citation
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+Please cite our work using the bibtex below:
+
+**BibTeX:**
+
+```
+@inproceedings{
+sahoo2025the,
+title={The Diffusion Duality},
+author={Subham Sekhar Sahoo and Justin Deschenaux and Aaron Gokaslan and Guanghan Wang and Justin T Chiu and Volodymyr Kuleshov},
+booktitle={Forty-second International Conference on Machine Learning},
+year={2025},
+url={https://openreview.net/forum?id=9P9Y8FOSOk}
+}
+```
+
+## Model Card Contact
+Subham Sekhar Sahoo (ssahoo@cs.cornell.edu)

checkpoints/quarantine/20260818T074907Z/official/mdlm/owt/README.md

@@ -0,0 +1,67 @@
+---
+library_name: transformers
+license: apache-2.0
+language:
+- en
+datasets:
+- Skylion007/openwebtext
+metrics:
+- perplexity
+---
+
+## Using MDLM
+To use the pre-trained model for masked language modeling, use the following snippet:
+```python
+from transformers import AutoModelForMaskedLM, AutoTokenizer
+
+# See the `MDLM` collection page on the hub for list of available models.
+tokenizer = transformers.AutoTokenizer.from_pretrained('gpt2')
+model_name = 'kuleshov-group/mdlm-owt'
+model = AutoModelForMaskedLM.from_pretrained(model_name)
+```
+
+For more details, please see our github repository: [MDLM](https://github.com/kuleshov-group/mdlm)
+
+## Model Details
+The model, which has a context length of `1024` and is similar in size to GPT2-medium with approximately `130 million` non-embedding parameters,
+was trained using a forward diffusion process that generates inputs varying from fully masked to fully unmasked. Its objective is to
+reconstruct the original input from these varying levels of masking, outputting logits in the process.
+The training regimen comprised one million steps on the OpenWebText corpus, involving the processing of a total of `33 billion` tokens.
+
+For more details, please see our paper: [Simple and Effective Masked Diffusion Language Models](http://arxiv.org/abs/2406.07524).
+
+
+
+## Citation
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+Please cite our work using the bibtex below:
+
+**BibTeX:**
+
+```
+@inproceedings{
+    sahoo2024simple,
+    title={Simple and Effective Masked Diffusion Language Models},
+    author={Subham Sekhar Sahoo and Marianne Arriola and Aaron Gokaslan and Edgar Mariano Marroquin and Alexander M Rush and Yair Schiff and Justin T Chiu and Volodymyr Kuleshov},
+    booktitle={The Thirty-eighth Annual Conference on Neural Information Processing Systems},
+    year={2024},
+    url={https://openreview.net/forum?id=L4uaAR4ArM}
+}
+```
+
+**APA:**
+
+```
+@software{Sahoo_Simple_and_Effective_2024,
+author = {Sahoo, Subham Sekhar and Arriola, Marianne and Schiff, Yair and Gokaslan, Aaron and Marroquin, Edgar and Chiu, Justin T and Rush, Alexander and Kuleshov, Volodymyr},
+doi = {10.48550/arXiv.2406.07524},
+month = jun,
+title = {{Simple and Effective Masked Diffusion Language Models}},
+version = {arXiv:2406.07524v1},
+year = {2024}
+}
+```
+
+## Model Card Contact
+Subham Sekhar Sahoo (subbham@gmail.com)

checkpoints/quarantine/20260818T074907Z/official/mdlm/owt/modeling_mdlm.py

@@ -0,0 +1,450 @@
+"""MDLM model for Hugging Face.
+
+"""
+import itertools
+import math
+import typing
+
+import flash_attn
+import flash_attn.layers.rotary
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import transformers
+from einops import rearrange
+from transformers import modeling_outputs
+
+from .configuration_mdlm import MDLMConfig
+
+# Flags required to enable jit fusion kernels
+torch._C._jit_set_profiling_mode(False)
+torch._C._jit_set_profiling_executor(False)
+torch._C._jit_override_can_fuse_on_cpu(True)
+torch._C._jit_override_can_fuse_on_gpu(True)
+
+
+def bias_dropout_add_scale(
+    x: torch.Tensor,
+    bias: typing.Optional[torch.Tensor],
+    scale: torch.Tensor,
+    residual: typing.Optional[torch.Tensor],
+    prob: float,
+    training: bool) -> torch.Tensor:
+  if bias is not None:
+    out = scale * F.dropout(x + bias, p=prob,
+                            training=training)
+  else:
+    out = scale * F.dropout(x, p=prob, training=training)
+
+  if residual is not None:
+    out = residual + out
+  return out
+
+
+def get_bias_dropout_add_scale(training):
+  def _bias_dropout_add(x, bias, scale, residual, prob):
+    return bias_dropout_add_scale(
+      x, bias, scale, residual, prob, training)
+
+  return _bias_dropout_add
+
+
+# function overload
+def modulate(x: torch.Tensor,
+             shift: torch.Tensor,
+             scale: torch.Tensor) -> torch.Tensor:
+  return x * (1 + scale) + shift
+
+
+@torch.jit.script
+def bias_dropout_add_scale_fused_train(
+    x: torch.Tensor,
+    bias: typing.Optional[torch.Tensor],
+    scale: torch.Tensor,
+    residual: typing.Optional[torch.Tensor],
+    prob: float) -> torch.Tensor:
+  return bias_dropout_add_scale(
+    x, bias, scale, residual, prob, True)
+
+
+@torch.jit.script
+def bias_dropout_add_scale_fused_inference(
+    x: torch.Tensor,
+    bias: typing.Optional[torch.Tensor],
+    scale: torch.Tensor,
+    residual: typing.Optional[torch.Tensor],
+    prob: float) -> torch.Tensor:
+  return bias_dropout_add_scale(
+    x, bias, scale, residual, prob, False)
+
+
+@torch.jit.script
+def modulate_fused(x: torch.Tensor,
+                   shift: torch.Tensor,
+                   scale: torch.Tensor) -> torch.Tensor:
+  return modulate(x, shift, scale)
+
+
+class Rotary(torch.nn.Module):
+  def __init__(self, dim, base=10_000):
+    super().__init__()
+    inv_freq = 1.0 / (
+          base ** (torch.arange(0, dim, 2).float() / dim))
+    self.register_buffer('inv_freq', inv_freq)
+    self.seq_len_cached = None
+    self.cos_cached = None
+    self.sin_cached = None
+
+  def forward(self, x, seq_dim=1):
+    seq_len = x.shape[seq_dim]
+    if seq_len != self.seq_len_cached:
+      self.seq_len_cached = seq_len
+      t = torch.arange(x.shape[seq_dim],
+                       device=x.device).type_as(
+        self.inv_freq)
+      freqs = torch.einsum("i,j->ij", t,
+                           self.inv_freq.clone())
+      emb = torch.cat((freqs, freqs), dim=-1).to(x.device)
+      # dims are: batch, seq_len, qkv, head, dim
+      self.cos_cached = emb.cos()[None, :, None, None,
+                        :].repeat(1, 1, 3, 1, 1)
+      self.sin_cached = emb.sin()[None, :, None, None,
+                        :].repeat(1, 1, 3, 1, 1)
+      # This makes the transformation on v an identity.
+      self.cos_cached[:, :, 2, :, :].fill_(1.)
+      self.sin_cached[:, :, 2, :, :].fill_(0.)
+
+    return self.cos_cached, self.sin_cached
+
+
+def rotate_half(x):
+  x1, x2 = x[..., : x.shape[-1] // 2], x[...,
+                                       x.shape[-1] // 2:]
+  return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb(qkv, cos, sin):
+  cos = cos[0, :, 0, 0, :cos.shape[-1] // 2]
+  sin = sin[0, :, 0, 0, :sin.shape[-1] // 2]
+  return flash_attn.layers.rotary.apply_rotary_emb_qkv_(qkv,
+                                                        cos,
+                                                        sin)
+# function overload
+def modulate(x, shift, scale):
+  return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1)
+
+
+############################################################
+#                          Layers                          #
+############################################################
+class LayerNorm(nn.Module):
+  def __init__(self, dim):
+    super().__init__()
+    self.weight = nn.Parameter(torch.ones([dim]))
+    self.dim = dim
+
+  def forward(self, x):
+    with torch.cuda.amp.autocast(enabled=False):
+      x = F.layer_norm(x.float(), [self.dim])
+    return x * self.weight[None, None, :]
+
+
+def residual_linear(x, W, x_skip, residual_scale):
+  """x_skip + residual_scale * W @ x"""
+  dim_out, dim_in = W.shape[0], W.shape[1]
+  return torch.addmm(
+    x_skip.view(-1, dim_out),
+    x.view(-1, dim_in),
+    W.T,
+    alpha=residual_scale).view(*x.shape[:-1], dim_out)
+
+
+############################################################
+#     Embedding Layers for Timesteps and Class Labels      #
+############################################################
+class TimestepEmbedder(nn.Module):
+  """
+  Embeds scalar timesteps into vector representations.
+  """
+
+  def __init__(self, hidden_size,
+               frequency_embedding_size=256):
+    super().__init__()
+    self.mlp = nn.Sequential(
+      nn.Linear(frequency_embedding_size, hidden_size,
+                bias=True),
+      nn.SiLU(),
+      nn.Linear(hidden_size, hidden_size, bias=True))
+    self.frequency_embedding_size = frequency_embedding_size
+
+  @staticmethod
+  def timestep_embedding(t, dim, max_period=10000):
+    """
+    Create sinusoidal timestep embeddings.
+    :param t: a 1-D Tensor of N indices, one per batch
+              element. These may be fractional.
+    :param dim: the dimension of the output.
+    :param max_period: controls the minimum frequency of the
+    embeddings.
+    :return: an (N, D) Tensor of positional embeddings.
+    """
+    # https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.py
+    half = dim // 2
+    freqs = torch.exp(
+      - math.log(max_period)
+      * torch.arange(start=0, end=half, dtype=torch.float32)
+      / half).to(device=t.device)
+    args = t[:, None].float() * freqs[None]
+    embedding = torch.cat(
+      [torch.cos(args), torch.sin(args)], dim=-1)
+    if dim % 2:
+      embedding = torch.cat(
+        [embedding,
+         torch.zeros_like(embedding[:, :1])], dim=-1)
+    return embedding
+
+  def forward(self, t):
+    t_freq = self.timestep_embedding(t,
+                                     self.frequency_embedding_size)
+    t_emb = self.mlp(t_freq)
+    return t_emb
+
+
+class LabelEmbedder(nn.Module):
+  """Embeds class labels into vector representations.
+
+  Also handles label dropout for classifier-free guidance.
+  """
+
+  def __init__(self, num_classes, cond_size):
+    super().__init__()
+    self.embedding_table = nn.Embedding(num_classes + 1,
+                                        cond_size)
+    self.num_classes = num_classes
+
+    # TODO think of initializing with 0.02 std deviation like in original DiT paper
+
+  def forward(self, labels):
+    embeddings = self.embedding_table(labels)
+    return embeddings
+
+
+############################################################
+#                        Core Model                        #
+############################################################
+
+
+class DDiTBlock(nn.Module):
+  def __init__(self, dim, n_heads, cond_dim, mlp_ratio=4,
+               dropout=0.1):
+    super().__init__()
+    self.n_heads = n_heads
+
+    self.norm1 = LayerNorm(dim)
+    self.attn_qkv = nn.Linear(dim, 3 * dim, bias=False)
+    self.attn_out = nn.Linear(dim, dim, bias=False)
+    self.dropout1 = nn.Dropout(dropout)
+
+    self.norm2 = LayerNorm(dim)
+    self.mlp = nn.Sequential(
+      nn.Linear(dim, mlp_ratio * dim, bias=True),
+      nn.GELU(approximate='tanh'),
+      nn.Linear(mlp_ratio * dim, dim, bias=True))
+    self.dropout2 = nn.Dropout(dropout)
+    self.dropout = dropout
+
+    self.adaLN_modulation = nn.Linear(cond_dim, 6 * dim,
+                                      bias=True)
+    self.adaLN_modulation.weight.data.zero_()
+    self.adaLN_modulation.bias.data.zero_()
+
+  def _get_bias_dropout_scale(self):
+    if self.training:
+      return bias_dropout_add_scale_fused_train
+    else:
+      return bias_dropout_add_scale_fused_inference
+
+  def forward(self, x, rotary_cos_sin, c, seqlens=None):
+    batch_size, seq_len = x.shape[0], x.shape[1]
+
+    bias_dropout_scale_fn = self._get_bias_dropout_scale()
+
+    (shift_msa, scale_msa, gate_msa, shift_mlp,
+     scale_mlp, gate_mlp) = self.adaLN_modulation(c)[:,
+                            None].chunk(6, dim=2)
+
+    # attention operation
+    x_skip = x
+    x = modulate_fused(self.norm1(x), shift_msa, scale_msa)
+
+    qkv = self.attn_qkv(x)
+    qkv = rearrange(qkv,
+                    'b s (three h d) -> b s three h d',
+                    three=3,
+                    h=self.n_heads)
+    with torch.cuda.amp.autocast(enabled=False):
+      cos, sin = rotary_cos_sin
+      qkv = apply_rotary_pos_emb(
+        qkv, cos.to(qkv.dtype), sin.to(qkv.dtype))
+    qkv = rearrange(qkv, 'b s ... -> (b s) ...')
+    if seqlens is None:
+      cu_seqlens = torch.arange(
+        0, (batch_size + 1) * seq_len, step=seq_len,
+        dtype=torch.int32, device=qkv.device)
+    else:
+      cu_seqlens = seqlens.cumsum(-1)
+    x = flash_attn.flash_attn_interface.flash_attn_varlen_qkvpacked_func(
+      qkv, cu_seqlens, seq_len, 0., causal=False)
+
+    x = rearrange(x, '(b s) h d -> b s (h d)', b=batch_size)
+
+    x = bias_dropout_scale_fn(self.attn_out(x),
+                              None,
+                              gate_msa,
+                              x_skip,
+                              self.dropout)
+
+    # mlp operation
+    x = bias_dropout_scale_fn(
+      self.mlp(modulate_fused(
+        self.norm2(x), shift_mlp, scale_mlp)),
+      None, gate_mlp, x, self.dropout)
+    return x
+
+
+class EmbeddingLayer(nn.Module):
+  def __init__(self, dim, vocab_dim):
+    super().__init__()
+    self.embedding = nn.Parameter(
+      torch.empty((vocab_dim, dim)))
+    torch.nn.init.kaiming_uniform_(self.embedding,
+                                   a=math.sqrt(5))
+
+  def forward(self, x):
+    return self.embedding[x]
+
+
+class DDitFinalLayer(nn.Module):
+  def __init__(self, hidden_size, out_channels, cond_dim):
+    super().__init__()
+    self.norm_final = LayerNorm(hidden_size)
+    self.linear = nn.Linear(hidden_size, out_channels)
+    self.linear.weight.data.zero_()
+    self.linear.bias.data.zero_()
+
+    self.adaLN_modulation = nn.Linear(cond_dim,
+                                      2 * hidden_size,
+                                      bias=True)
+    self.adaLN_modulation.weight.data.zero_()
+    self.adaLN_modulation.bias.data.zero_()
+
+  def forward(self, x, c):
+    shift, scale = self.adaLN_modulation(c)[:, None].chunk(
+      2, dim=2)
+    x = modulate_fused(self.norm_final(x), shift, scale)
+    x = self.linear(x)
+    return x
+
+
+class DITBackbone(nn.Module):
+  def __init__(
+      self,
+      config: MDLMConfig):
+    super().__init__()
+
+    self.config = config
+    self.vocab_size = config.vocab_size
+
+    self.vocab_embed = EmbeddingLayer(
+      config.hidden_dim,
+      config.vocab_size)
+    self.sigma_map = TimestepEmbedder(
+      config.cond_dim)
+    self.rotary_emb = Rotary(
+      config.hidden_dim // config.n_heads)
+
+    blocks = []
+    for _ in range(config.n_blocks):
+      blocks.append(DDiTBlock(config.hidden_dim,
+                              config.n_heads,
+                              config.cond_dim,
+                              dropout=config.dropout))
+    self.blocks = nn.ModuleList(blocks)
+
+    self.output_layer = DDitFinalLayer(
+      config.hidden_dim,
+      config.vocab_size,
+      config.cond_dim)
+
+  def _get_bias_dropout_scale(self):
+    if self.training:
+      return bias_dropout_add_scale_fused_train
+    else:
+      return bias_dropout_add_scale_fused_inference
+
+  def forward(self, indices, sigma,
+              output_hidden_states=False):
+    if not self.config.time_conditioning:
+      sigma = torch.zeros_like(sigma)
+    all_hidden_states = []
+    x = self.vocab_embed(indices)
+    if output_hidden_states:
+      all_hidden_states.append(x)
+    c = F.silu(self.sigma_map(sigma))
+
+    rotary_cos_sin = self.rotary_emb(x)
+
+    with torch.cuda.amp.autocast(dtype=torch.bfloat16):
+      for i in range(len(self.blocks)):
+        x = self.blocks[i](x, rotary_cos_sin, c,
+                           seqlens=None)
+        if output_hidden_states:
+          all_hidden_states.append(x)
+      logits = self.output_layer(x, c)
+    return logits, all_hidden_states
+
+class MDLM(transformers.PreTrainedModel):
+  """HF-compatible model."""
+  config_class = MDLMConfig
+  base_model_prefix = "mdlm"
+
+  def __init__(
+    self,
+    config: MDLMConfig):
+    super().__init__(config)
+    self.backbone = DITBackbone(config)
+
+  def forward(
+      self,
+      input_ids: torch.LongTensor = None,
+      timesteps: torch.FloatTensor = None,
+      output_hidden_states: typing.Optional[bool] = None,
+      return_dict: typing.Optional[bool] = None,
+  ) -> typing.Union[
+    torch.Tensor, typing.Tuple,
+    modeling_outputs.MaskedLMOutput]:
+    """HF-compatible forward method."""
+    output_hidden_states = (
+      output_hidden_states
+      if output_hidden_states is not None
+      else self.config.output_hidden_states
+    )
+    return_dict = return_dict \
+      if return_dict is not None \
+      else self.config.use_return_dict
+
+    logits, all_hidden_states = self.backbone(
+      indices=input_ids,
+      sigma=timesteps,
+      output_hidden_states=output_hidden_states
+    )
+    if return_dict:
+      return modeling_outputs.MaskedLMOutput(
+        logits=logits,
+        hidden_states=all_hidden_states if output_hidden_states else None,
+        loss=None
+      )
+    elif output_hidden_states:
+      return logits, all_hidden_states
+    else:
+      return logits

checkpoints/quarantine/20260818T074924Z/official/mdlm_sdtt/owt/config.json

@@ -0,0 +1,207 @@
+{
+  "mode": "train",
+  "time_conditioning": false,
+  "T": 0,
+  "compile": false,
+  "eval_every": 5000,
+  "data_preprocess": {
+    "data_cache": "REDACTED",
+    "min_seq_len": -1,
+    "seq_len": "${model.length}",
+    "group_text": true,
+    "remove_text": true,
+    "num_seqs": -1,
+    "add_bos": false,
+    "add_eos": true,
+    "legacy_start_end_bos": true
+  },
+  "loader": {
+    "global_batch_size": 32,
+    "eval_global_batch_size": "${.global_batch_size}",
+    "batch_size": "${div_up:${.global_batch_size}, ${eval:${trainer.devices} * ${trainer.num_nodes}}}",
+    "eval_batch_size": "${div_up:${.eval_global_batch_size}, ${eval:${trainer.devices} * ${trainer.num_nodes}}}",
+    "num_workers": 32,
+    "pin_memory": true,
+    "persistent_workers": true
+  },
+  "training": {
+    "ema": 0.9999,
+    "antithetic_sampling": true,
+    "importance_sampling": false,
+    "sampling_eps": 1e-05,
+    "change_of_variables": false
+  },
+  "tokenizer": {
+    "name": "gpt2"
+  },
+  "optim": {
+    "name": "adamw",
+    "weight_decay": 0.0,
+    "lr": 0.0003,
+    "beta1": 0.9,
+    "beta2": 0.999,
+    "eps": 1e-08
+  },
+  "trainer": {
+    "_target_": "lightning.Trainer",
+    "accelerator": "cuda",
+    "num_nodes": 1,
+    "devices": 1,
+    "accumulate_grad_batches": "${div_up:${loader.global_batch_size}, ${eval:${trainer.devices} * ${loader.batch_size} * ${trainer.num_nodes}}}",
+    "gradient_clip_val": 1.0,
+    "precision": "bf16-mixed",
+    "num_sanity_val_steps": 2,
+    "max_steps": 1000000,
+    "log_every_n_steps": 10,
+    "limit_train_batches": 1.0,
+    "limit_val_batches": 1.0,
+    "val_check_interval": "${eval:${trainer.accumulate_grad_batches} * ${eval_every}}",
+    "check_val_every_n_epoch": null,
+    "benchmark": true
+  },
+  "wandb": {
+    "project": "text-diffusion",
+    "notes": "",
+    "group": null,
+    "job_type": null,
+    "name": null,
+    "tags": [
+      "${noise.type}",
+      "${data.train}",
+      "${data.valid}"
+    ]
+  },
+  "checkpointing": {
+    "save_dir": "${cwd:}",
+    "resume_from_ckpt": true,
+    "resume_ckpt_path": "/claire-rcp-scratch/home/deschena/fast-discrete-diff/dev/hf_downloads/baselines/kld/baselines/bwd_kl_double_dt_10k_lr_2x_sm/70000.ckpt"
+  },
+  "eval": {
+    "valid": {
+      "n_samples": 16,
+      "num_steps": 128
+    },
+    "ppl_with_ar": {
+      "run": true,
+      "model": "gpt2-large",
+      "batch_size": 8
+    },
+    "mauve": {
+      "run": false,
+      "num_rounds": 5,
+      "max_num_tokens": 100,
+      "batch_size": 8,
+      "scaling_factor": 5
+    },
+    "lambada_openai": {
+      "run": false,
+      "batch_size": 32,
+      "num_samples": 32,
+      "from_ema": true,
+      "add_bos": true,
+      "add_eos": true
+    }
+  },
+  "callbacks": {
+    "checkpoint_every_n_steps": {
+      "_target_": "lightning.pytorch.callbacks.ModelCheckpoint",
+      "save_top_k": -1,
+      "save_last": true,
+      "dirpath": "${checkpointing.save_dir}/checkpoints",
+      "verbose": true,
+      "auto_insert_metric_name": false,
+      "every_n_train_steps": 10000
+    },
+    "learning_rate_monitor": {
+      "_target_": "lightning.pytorch.callbacks.LearningRateMonitor",
+      "logging_interval": "step"
+    },
+    "text_throughput": {
+      "_target_": "callbacks.text_throughput.TextThroughputMonitor"
+    },
+    "grad_norm": {
+      "_target_": "callbacks.gradient_norm.GradNormCallback"
+    }
+  },
+  "data": {
+    "train": "openwebtext-train",
+    "valid": "openwebtext-valid",
+    "test": "openwebtext-test",
+    "tokenizer_name_or_path": "gpt2",
+    "wrap": true,
+    "streaming": false
+  },
+  "model": {
+    "name": "small",
+    "type": "ddit-orig",
+    "hidden_size": 768,
+    "cond_dim": 128,
+    "length": 1024,
+    "n_blocks": 12,
+    "n_heads": 12,
+    "scale_by_sigma": true,
+    "dropout": 0.1,
+    "tie_word_embeddings": false
+  },
+  "strategy": {
+    "_target_": "lightning.pytorch.strategies.DDPStrategy",
+    "find_unused_parameters": false,
+    "process_group_backend": "nccl"
+  },
+  "noise": {
+    "type": "loglinear",
+    "sigma_min": 0.0001,
+    "sigma_max": 20
+  },
+  "lr_scheduler": {
+    "_target_": "transformers.get_constant_schedule_with_warmup",
+    "num_warmup_steps": 2500
+  },
+  "parameterization": {
+    "name": "multi-round-sdtt",
+    "log_loss_buckets": -1,
+    "checkpoint_path": "kuleshov-group/mdlm-owt",
+    "start_from_hf": false,
+    "distill_mode": "kl-bwd",
+    "num_distill_steps": 2,
+    "min_num_sampling_steps": 8,
+    "grow_dt_every": 10000,
+    "orig_num_sampling_steps": 1024,
+    "sampling_mode": "ancestral",
+    "loss_precision": null,
+    "reset_optimizer_on_growth": false,
+    "use_ema_on_growth": false,
+    "sampling": {
+      "uncond": {
+        "run": false,
+        "num_samples": 1024,
+        "batch_size": 32,
+        "from_ema": true,
+        "num_steps": 1024,
+        "seq_len": 1024,
+        "sampler": "ancestral",
+        "cache_preds": false,
+        "add_bos": false,
+        "add_eos": false
+      },
+      "cond_prefix": {
+        "run": false,
+        "num_samples": 1024,
+        "batch_size": 32,
+        "from_ema": true,
+        "dataset": "webtext",
+        "seq_len": 100,
+        "prefix_len": 50,
+        "num_cont_per_prefix": 5,
+        "min_seq_len": 1024,
+        "num_steps": 1024,
+        "sampler": "ancestral",
+        "cache_preds": false,
+        "add_bos": false,
+        "add_eos": false
+      }
+    }
+  },
+  "repo": "jdeschena/sdtt",
+  "revision": "baselines_kld"
+}

checkpoints/quarantine/20260820T072046Z/official/fmlm/owt/config.json

@@ -0,0 +1,223 @@
+{
+  "config": {
+    "algo": {
+      "T": 0,
+      "add_boundary": "fixed",
+      "backbone": "dit",
+      "backprop_entmax_temp_lsd": false,
+      "boundary_prob": 32,
+      "causal_attention": false,
+      "diagonal_fraction": 0.5,
+      "distillation_method": "PSD",
+      "double_temb": true,
+      "entmax_temp_lsd": 2.0,
+      "initialize_student_from_teacher": true,
+      "learnable_loss_weighting": false,
+      "loss_scale": "none",
+      "loss_type": null,
+      "name": "fmlm",
+      "offdiagonal_sampling": "uniform_diff",
+      "parameterization": "mean",
+      "pred_log_interval": 2000,
+      "rescale_offdiag_loss_psd": false,
+      "subs_masking": false,
+      "t_max": 1.0,
+      "t_min": 0.0,
+      "teacher_path": "/home/irteam/ours_owt_flow_ce_1m.ckpt",
+      "time_condition": "alpha_t",
+      "time_conditioning": true,
+      "use_discrete_schedule": true,
+      "use_ema_for_psd_target": false,
+      "use_mse_loss_psd": false,
+      "use_simulated_entropy": false,
+      "use_teacher_for_D_t_lsd": false
+    },
+    "callbacks": {
+      "checkpoint_every_n_steps": {
+        "_target_": "lightning.pytorch.callbacks.ModelCheckpoint",
+        "auto_insert_metric_name": false,
+        "dirpath": "/home/david3684/projects/flm/hf_upload/checkpoints",
+        "every_n_train_steps": 5000,
+        "save_last": false,
+        "save_top_k": -1,
+        "verbose": true
+      },
+      "checkpoint_monitor": {
+        "_target_": "lightning.pytorch.callbacks.ModelCheckpoint",
+        "auto_insert_metric_name": false,
+        "dirpath": "/home/david3684/projects/flm/hf_upload/checkpoints",
+        "filename": "best_nll",
+        "mode": "min",
+        "monitor": "val/nll",
+        "save_last": true,
+        "save_top_k": 1,
+        "verbose": true
+      },
+      "learning_rate_monitor": {
+        "_target_": "lightning.pytorch.callbacks.LearningRateMonitor",
+        "logging_interval": "step"
+      }
+    },
+    "checkpointing": {
+      "resume_ckpt_path": "/home/david3684/projects/flm/hf_upload/checkpoints/last.ckpt",
+      "resume_from_ckpt": false,
+      "save_dir": "/home/david3684/projects/flm/hf_upload"
+    },
+    "data": {
+      "cache_dir": "/home/irteam/workspace/datasets/openwebtext",
+      "insert_train_eos": true,
+      "insert_valid_eos": true,
+      "streaming": false,
+      "tokenizer_name_or_path": "gpt2",
+      "train": "openwebtext-train",
+      "valid": "openwebtext-valid",
+      "wrap": true
+    },
+    "eval": {
+      "checkpoint_path": "",
+      "compute_generative_perplexity": true,
+      "compute_perplexity_on_sanity": false,
+      "disable_ema": false,
+      "ema_decay": null,
+      "gen_ppl_eval_model_name_or_path": "gpt2-large",
+      "generate_samples": true,
+      "generated_samples_path": "/home/david3684/projects/flm/hf_upload/samples.json",
+      "perplexity_batch_size": 8
+    },
+    "is_di4c": false,
+    "is_di4c_deterministic": true,
+    "loader": {
+      "batch_size": 16,
+      "eval_batch_size": 16,
+      "eval_global_batch_size": 128,
+      "global_batch_size": 128,
+      "num_workers": 4,
+      "pin_memory": true
+    },
+    "lr_scheduler": {
+      "_target_": "transformers.get_constant_schedule_with_warmup",
+      "num_warmup_steps": 2500
+    },
+    "mode": "train",
+    "model": {
+      "cond_dim": 128,
+      "dropout": 0.1,
+      "hidden_size": 768,
+      "l2_normalize_embeddings": false,
+      "length": 1024,
+      "n_blocks": 12,
+      "n_heads": 12,
+      "name": "small",
+      "scale_by_sigma": true,
+      "tie_word_embeddings": false,
+      "type": "ddit",
+      "vocab_lookup": true
+    },
+    "noise": {
+      "denoiser_latent_conditioning": -1,
+      "eps": 0,
+      "freeze_decoder": false,
+      "freeze_encoder": false,
+      "parameterization": "log-linear",
+      "type": "log-linear"
+    },
+    "optim": {
+      "beta1": 0.9,
+      "beta2": 0.95,
+      "eps": 1e-08,
+      "ln_tune": "none",
+      "lr": 0.0003,
+      "muon_lr": 0.03,
+      "muon_momentum": 0.95,
+      "muon_weight_decay": 0.01,
+      "use_muon": false,
+      "weight_decay": 0
+    },
+    "prior": {
+      "latent_height": 0,
+      "latent_width": 0,
+      "type": "none"
+    },
+    "sampling": {
+      "argmax_correction": false,
+      "autoguidance_weight": 0.0,
+      "hard_start": false,
+      "noise_removal": "uniform_alpha",
+      "num_reflow_samples": 50000,
+      "num_sample_batches": 1,
+      "num_sample_log": 2,
+      "num_strides": 1,
+      "p_nucleus": 1.0,
+      "predictor": "ancestral",
+      "semi_ar": false,
+      "solver": "euler",
+      "steps": [
+        1,
+        2,
+        4,
+        8,
+        16,
+        32,
+        64,
+        128
+      ],
+      "stride_length": 1,
+      "tau_log10": -1.0,
+      "temperature": 1.0,
+      "use_float64": true
+    },
+    "seed": 1,
+    "strategy": {
+      "_target_": "lightning.pytorch.strategies.DDPStrategy",
+      "find_unused_parameters": false
+    },
+    "trainer": {
+      "_target_": "lightning.Trainer",
+      "accelerator": "cuda",
+      "accumulate_grad_batches": 1,
+      "devices": 8,
+      "gradient_clip_val": 1.0,
+      "limit_train_batches": 1.0,
+      "limit_val_batches": 10,
+      "log_every_n_steps": 100,
+      "max_steps": 1000000,
+      "num_nodes": 1,
+      "num_sanity_val_steps": 2,
+      "precision": "bf16",
+      "val_check_interval": 10000
+    },
+    "training": {
+      "antithetic_sampling": true,
+      "change_of_variables": false,
+      "double_ema": 0.0,
+      "ema": "0.9999, 0.999, 0.99, 0.95",
+      "finetune_path": "",
+      "importance_sampling": false,
+      "load_ema": false,
+      "loss_precision": "bf16",
+      "loss_type": "sd",
+      "loss_weighting": false,
+      "not_sampling_t": false,
+      "pred_type": "x0",
+      "sampling_eps": 0.001,
+      "t_curriculum_max": 1.0,
+      "t_curriculum_min": 0.0,
+      "t_curriculum_steps": 100000,
+      "torch_compile_mode": "default",
+      "use_torch_compile": true
+    },
+    "wandb": {
+      "group": null,
+      "id": "9efb5cfb",
+      "job_type": null,
+      "name": "owt_PSD_teacher_distill_LR_3e-4_DF_0.5_BP_32_SE_False_IT_True_EDMW_False_EMA_False_beta2_0.95_9efb5cfb",
+      "notes": "DOS",
+      "project": "owt_full",
+      "tags": [
+        "openwebtext-train",
+        "dos_denoiser"
+      ]
+    }
+  },
+  "vocab_size": 50258
+}

checkpoints/quarantine/20260820T072128Z/official/langflow/owt/__init__.py

@@ -0,0 +1,6 @@
+"""HuggingFace release package for LangFlow."""
+
+from .config import LangFlowConfig
+from .model import LangFlow
+
+__all__ = ["LangFlowConfig", "LangFlow"]

checkpoints/quarantine/20260820T072218Z/official/duo/owt/README.md

@@ -0,0 +1,53 @@
+---
+datasets:
+- Skylion007/openwebtext
+language:
+- en
+library_name: transformers
+license: apache-2.0
+metrics:
+- perplexity
+pipeline_tag: text-generation
+---
+
+## Using DUO
+To use the pre-trained model for masked language modeling, use the following snippet:
+```python
+from transformers import AutoModelForMaskedLM, AutoTokenizer
+
+# See the `MDLM` collection page on the hub for list of available models.
+tokenizer = transformers.AutoTokenizer.from_pretrained('gpt2')
+model = AutoModelForMaskedLM.from_pretrained('s-sahoo/duo')
+```
+For a hands-on example, check out this [Colab notebook](https://colab.research.google.com/drive/1Sf7R-dqdR6gq-H8nyZ9E3ZkyvqMTqcwq?usp=sharing).
+For more information and implementation details, visit our github repository: [DUO](https://github.com/s-sahoo/duo) and project page: [Project Page](https://s-sahoo.com/duo)
+
+## Model Details
+The model, which has a context length of `1024` and is similar in size to GPT2-medium with approximately `130 million` non-embedding parameters,
+was trained for 1M steps on the OpenWebText corpus.
+
+For more details, please see our paper: [The Diffusion Duality](https://openreview.net/forum?id=CB0Ub2yXjC).
+
+
+
+## Citation
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+Please cite our work using the bibtex below:
+
+**BibTeX:**
+
+```
+@inproceedings{
+sahoo2025the,
+title={The Diffusion Duality},
+author={Subham Sekhar Sahoo and Justin Deschenaux and Aaron Gokaslan and Guanghan Wang and Justin T Chiu and Volodymyr Kuleshov},
+booktitle={Forty-second International Conference on Machine Learning},
+year={2025},
+url={https://openreview.net/forum?id=9P9Y8FOSOk}
+}
+```
+
+
+## Model Card Contact
+Subham Sekhar Sahoo (ssahoo@cs.cornell.edu)

checkpoints/quarantine/20260820T072218Z/official/duo/owt/config.py

@@ -0,0 +1,29 @@
+import transformers
+
+
+class DUOConfig(transformers.PretrainedConfig):
+  """Hugging Face configuration class for DUO."""
+  model_type = 'DUO'
+
+  def __init__(
+    self,
+    vocab_size: int = 50258,
+    model_length: int = 1024,
+    causal: bool = False,
+    hidden_dim: int = 768,
+    cond_dim: int = 129,
+    n_blocks: int = 12,
+    n_heads: int = 12,
+    dropout: float = 0.1,
+    var_min: bool = True,
+    ** kwargs):
+    super().__init__(**kwargs)
+    self.causal = causal
+    self.vocab_size = vocab_size
+    self.model_length = model_length
+    self.hidden_dim = hidden_dim
+    self.cond_dim = cond_dim
+    self.n_blocks = n_blocks
+    self.n_heads = n_heads
+    self.dropout = dropout
+    self.var_min = var_min

checkpoints/quarantine/20260820T072232Z/official/duo_dcd/owt/config.json

@@ -0,0 +1,23 @@
+{
+  "_name_or_path": "s-sahoo/duo-distilled",
+  "architectures": [
+    "DUO"
+  ],
+  "auto_map": {
+    "AutoConfig": "config.DUOConfig",
+    "AutoModelForMaskedLM": "model.DUO"
+  },
+  "causal": false,
+  "cond_dim": 128,
+  "dropout": 0.1,
+  "hidden_dim": 768,
+  "model_length": 1024,
+  "model_type": "DUO",
+  "n_blocks": 12,
+  "n_heads": 12,
+  "return_dict": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.38.2",
+  "var_min": true,
+  "vocab_size": 50258
+}

checkpoints/quarantine/20260820T072232Z/official/duo_dcd/owt/model.py

@@ -0,0 +1,601 @@
+import math
+import typing
+
+import einops
+import flash_attn
+import flash_attn.layers.rotary
+import huggingface_hub
+import omegaconf
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import transformers
+
+from .config import DUOConfig
+
+# Flags required to enable jit fusion kernels
+torch._C._jit_set_profiling_mode(False)
+torch._C._jit_set_profiling_executor(False)
+torch._C._jit_override_can_fuse_on_cpu(True)
+torch._C._jit_override_can_fuse_on_gpu(True)
+
+
+def bias_dropout_add_scale(
+    x: torch.Tensor,
+    bias: typing.Optional[torch.Tensor],
+    scale: torch.Tensor,
+    residual: typing.Optional[torch.Tensor],
+    prob: float,
+    training: bool) -> torch.Tensor:
+  if bias is not None:
+    out = scale * F.dropout(x + bias, p=prob, training=training)
+  else:
+    out = scale * F.dropout(x, p=prob, training=training)
+
+  if residual is not None:
+    out = residual + out
+  return out
+
+
+def get_bias_dropout_add_scale(training):
+  def _bias_dropout_add(x, bias, scale, residual, prob):
+    return bias_dropout_add_scale(
+      x, bias, scale, residual, prob, training)
+
+  return _bias_dropout_add
+
+
+# function overload
+def modulate(x: torch.Tensor,
+             shift: torch.Tensor,
+             scale: torch.Tensor) -> torch.Tensor:
+  return x * (1 + scale) + shift
+
+
+@torch.jit.script
+def bias_dropout_add_scale_fused_train(
+    x: torch.Tensor,
+    bias: typing.Optional[torch.Tensor],
+    scale: torch.Tensor,
+    residual: typing.Optional[torch.Tensor],
+    prob: float) -> torch.Tensor:
+  return bias_dropout_add_scale(
+    x, bias, scale, residual, prob, True)
+
+
+@torch.jit.script
+def bias_dropout_add_scale_fused_inference(
+    x: torch.Tensor,
+    bias: typing.Optional[torch.Tensor],
+    scale: torch.Tensor,
+    residual: typing.Optional[torch.Tensor],
+    prob: float) -> torch.Tensor:
+  return bias_dropout_add_scale(
+    x, bias, scale, residual, prob, False)
+
+
+@torch.jit.script
+def modulate_fused(x: torch.Tensor,
+                   shift: torch.Tensor,
+                   scale: torch.Tensor) -> torch.Tensor:
+  return modulate(x, shift, scale)
+
+
+class Rotary(torch.nn.Module):
+  def __init__(self, dim, base=10_000):
+    super().__init__()
+    inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim))
+    self.register_buffer('inv_freq', inv_freq)
+    self.seq_len_cached = None
+    self.cos_cached = None
+    self.sin_cached = None
+
+  def forward(self, x, seq_dim=1):
+    seq_len = x.shape[seq_dim]
+    if seq_len != self.seq_len_cached:
+      self.seq_len_cached = seq_len
+      t = torch.arange(x.shape[seq_dim], device=x.device).type_as(self.inv_freq)
+      freqs = torch.einsum("i,j->ij", t, self.inv_freq.clone())
+      emb = torch.cat((freqs, freqs), dim=-1).to(x.device)
+      # dims are: batch, seq_len, qkv, head, dim
+      self.cos_cached = emb.cos()[None, :, None, None, :].repeat(1,1,3,1,1)
+      self.sin_cached = emb.sin()[None, :, None, None, :].repeat(1,1,3,1,1)
+      # This makes the transformation on v an identity.
+      self.cos_cached[:,:,2,:,:].fill_(1.)
+      self.sin_cached[:,:,2,:,:].fill_(0.)
+
+    return self.cos_cached, self.sin_cached
+
+
+def rotate_half(x):
+  x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2 :]
+  return torch.cat((-x2, x1), dim=-1)
+
+
+def split_and_apply_rotary_pos_emb(qkv, rotary_cos_sin):
+  with torch.cuda.amp.autocast(enabled=False):
+    cos, sin = rotary_cos_sin
+    cos = cos.to(qkv.dtype)
+    sin = sin.to(qkv.dtype)
+    cos = cos[0,:,0,0,:cos.shape[-1]//2]
+    sin = sin[0,:,0,0,:sin.shape[-1]//2]
+    q, k, v = qkv.chunk(3, dim=2)
+    q = flash_attn.layers.rotary.apply_rotary_emb_torch(
+      q.squeeze(dim=2), cos, sin)
+    k = flash_attn.layers.rotary.apply_rotary_emb_torch(
+      k.squeeze(dim=2), cos, sin)
+    v = v.squeeze(dim=2)
+  return q, k, v
+
+
+def apply_rotary_pos_emb(qkv, cos, sin):
+  cos = cos[0,:,0,0,:cos.shape[-1]//2]
+  sin = sin[0,:,0,0,:sin.shape[-1]//2]
+  return flash_attn.layers.rotary.apply_rotary_emb_qkv_(qkv, cos, sin)
+
+
+def regular_attention_multi_headed(q, k, v):
+  # Assuming qkv is a tensor with shape [batch, seq_len, 3, num_heads, head_dim]
+  # where the 3 represents Q, K, V packed in that order
+  attention_output = F.scaled_dot_product_attention(
+    query=q.transpose(1, 2),
+    key=k.transpose(1, 2),
+    value=v.transpose(1, 2),
+    attn_mask=None,
+    dropout_p=0.0,
+    is_causal=False)
+  # [batch_size, seq_len, num_heads, head_dim]
+  attention_output = attention_output.transpose(1, 2)
+  return einops.rearrange(attention_output, 'b s h d -> b s (h d)')
+
+
+#################################################################################
+#                                  Layers                                       #
+#################################################################################
+class LayerNorm(nn.Module):
+  def __init__(self, dim):
+    super().__init__()
+    self.weight = nn.Parameter(torch.ones([dim]))
+    self.dim = dim
+  def forward(self, x):
+    with torch.cuda.amp.autocast(enabled=False):
+      x = F.layer_norm(x.float(), [self.dim])
+    return x * self.weight[None, None, :]
+
+
+def residual_linear(x, W, x_skip, residual_scale):
+  """x_skip + residual_scale * W @ x"""
+  dim_out, dim_in = W.shape[0], W.shape[1]
+  return torch.addmm(
+    x_skip.view(-1, dim_out),
+    x.view(-1, dim_in),
+    W.T,
+    alpha=residual_scale).view(*x.shape[:-1], dim_out)
+
+
+#################################################################################
+#               Embedding Layers for Timesteps and Class Labels                 #
+#################################################################################
+class TimestepEmbedder(nn.Module):
+  """
+  Embeds scalar timesteps into vector representations.
+  """
+  def __init__(self, hidden_size, frequency_embedding_size=256):
+    super().__init__()
+    self.mlp = nn.Sequential(
+      nn.Linear(frequency_embedding_size, hidden_size, bias=True),
+      nn.SiLU(),
+      nn.Linear(hidden_size, hidden_size, bias=True))
+    self.frequency_embedding_size = frequency_embedding_size
+
+  @staticmethod
+  def timestep_embedding(t, dim, max_period=10000):
+    """
+    Create sinusoidal timestep embeddings.
+    :param t: a 1-D Tensor of N indices, one per batch element.
+                      These may be fractional.
+    :param dim: the dimension of the output.
+    :param max_period: controls the minimum frequency of the embeddings.
+    :return: an (N, D) Tensor of positional embeddings.
+    """
+    # https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.py
+    half = dim // 2
+    freqs = torch.exp(
+      - math.log(max_period)
+      * torch.arange(start=0, end=half, dtype=torch.float32, device=t.device)
+      / half)
+    args = t[:, None].float() * freqs[None]
+    embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
+    if dim % 2:
+      embedding = torch.cat(
+        [embedding,
+         torch.zeros_like(embedding[:, :1])], dim=-1)
+    return embedding
+
+  def forward(self, t):
+    t_freq = self.timestep_embedding(t, self.frequency_embedding_size)
+    t_emb = self.mlp(t_freq)
+    return t_emb
+
+
+class LabelEmbedder(nn.Module):
+  """Embeds class labels into vector representations.
+  
+  Also handles label dropout for classifier-free guidance.
+  """
+  def __init__(self, num_classes, cond_size):
+    super().__init__()
+    self.embedding_table = nn.Embedding(num_classes + 1, cond_size)
+    self.num_classes = num_classes
+
+    # TODO think of initializing with 0.02 std deviation like in original DiT paper
+
+  def forward(self, labels):
+    embeddings = self.embedding_table(labels)
+    return embeddings
+    
+
+#################################################################################
+#                                 Core Model                                    #
+#################################################################################
+
+class DDiTBlockCausal(nn.Module):
+  def __init__(self, dim, n_heads, mlp_ratio=4, dropout=0.1):
+    super().__init__()
+    self.n_heads = n_heads
+
+    self.norm1 = LayerNorm(dim)
+    self.attn_qkv = nn.Linear(dim, 3 * dim, bias=False)
+    self.attn_out = nn.Linear(dim, dim, bias=False)
+    self.dropout1 = nn.Dropout(dropout)
+
+    self.norm2 = LayerNorm(dim)
+    self.mlp = nn.Sequential(
+      nn.Linear(dim, mlp_ratio * dim, bias=True),
+      nn.GELU(approximate='tanh'),
+      nn.Linear(mlp_ratio * dim, dim, bias=True))
+    self.dropout2 = nn.Dropout(dropout)
+    self.dropout = dropout
+
+  def _get_bias_dropout_scale(self):
+    if self.training:
+      return bias_dropout_add_scale_fused_train
+    else:
+      return bias_dropout_add_scale_fused_inference
+
+  def forward(self, x, rotary_cos_sin, **kwargs):
+    del kwargs
+    batch_size, seq_len = x.shape[0], x.shape[1]
+
+    bias_dropout_scale_fn = self._get_bias_dropout_scale()
+
+    # attention operation
+    x_skip = x
+    x = self.norm1(x)
+
+    qkv = self.attn_qkv(x)
+    qkv = einops.rearrange(
+      qkv,
+      'b s (three h d) -> b s three h d',
+      three=3,
+      h=self.n_heads)
+    with torch.cuda.amp.autocast(enabled=False):
+      cos, sin = rotary_cos_sin
+      qkv = apply_rotary_pos_emb(
+        qkv, cos.to(qkv.dtype), sin.to(qkv.dtype)
+      )
+    qkv = einops.rearrange(qkv, 'b s ... -> (b s) ...')
+    cu_seqlens = torch.arange(
+      0, (batch_size + 1) * seq_len,
+      step=seq_len, dtype=torch.int32, device=qkv.device)
+    x = flash_attn.flash_attn_interface.flash_attn_varlen_qkvpacked_func(
+      qkv, cu_seqlens, seq_len, 0.0, causal=True)
+
+    x = einops.rearrange(x, '(b s) h d -> b s (h d)',
+                         b=batch_size)
+
+    scale = torch.ones(1, device=x.device, dtype=x.dtype)
+    x = bias_dropout_scale_fn(
+      self.attn_out(x), None, scale, x_skip, self.dropout)
+
+    # mlp operation
+    x = bias_dropout_scale_fn(
+      self.mlp(self.norm2(x)), None, scale, x, self.dropout)
+    return x
+
+
+
+class DDiTBlock(nn.Module):
+  def __init__(self, dim, n_heads, adaLN,
+               cond_dim=None, mlp_ratio=4,
+               dropout=0.1):
+    super().__init__()
+    self.n_heads = n_heads
+    self.adaLN = adaLN
+
+    self.norm1 = LayerNorm(dim)
+    self.attn_qkv = nn.Linear(dim, 3 * dim, bias=False)
+    self.attn_out = nn.Linear(dim, dim, bias=False)
+    self.dropout1 = nn.Dropout(dropout)
+
+    self.norm2 = LayerNorm(dim)
+    self.mlp = nn.Sequential(
+      nn.Linear(dim, mlp_ratio * dim, bias=True),
+      nn.GELU(approximate='tanh'),
+      nn.Linear(mlp_ratio * dim, dim, bias=True))
+    self.dropout2 = nn.Dropout(dropout)
+    self.dropout = dropout
+
+    if self.adaLN:
+      self.adaLN_modulation = nn.Linear(cond_dim, 6 * dim)
+      self.adaLN_modulation.weight.data.zero_()
+      self.adaLN_modulation.bias.data.zero_()
+
+
+  def _get_bias_dropout_scale(self):
+    if self.training:
+      return bias_dropout_add_scale_fused_train
+    else:
+      return bias_dropout_add_scale_fused_inference
+
+
+  def forward(self, x, rotary_cos_sin, c=None):
+
+    bias_dropout_scale_fn = self._get_bias_dropout_scale()
+
+    x_skip = x
+    x = self.norm1(x)
+
+    if self.adaLN:
+      # self.adaLN_modulation(c): (128, 1536)
+      # self.adaLN_modulation(c)[:, None]: (128, 1, 1536)
+      # "" .chunk(6, dim=2) returns 6 tuples of shapes (128, 1, 256)
+      (shift_msa, scale_msa, gate_msa, shift_mlp,
+       scale_mlp, gate_mlp) = self.adaLN_modulation(c)[:, None].chunk(6, dim=2)
+      x = modulate_fused(x, shift_msa, scale_msa)
+
+    qkv = einops.rearrange(
+      self.attn_qkv(x),
+      'b s (three h d) -> b s three h d',
+      three=3,
+      h=self.n_heads)
+    q, k, v = split_and_apply_rotary_pos_emb(qkv, rotary_cos_sin)
+    
+    x = regular_attention_multi_headed(q, k, v)
+
+    if self.adaLN:
+      x = bias_dropout_scale_fn(self.attn_out(x),
+                                None,
+                                gate_msa,
+                                x_skip,
+                                self.dropout)
+      x = bias_dropout_scale_fn(
+        self.mlp(modulate_fused(
+          self.norm2(x), shift_mlp, scale_mlp)),
+        None, gate_mlp, x, self.dropout)
+    else:
+      scale = torch.ones(1, device=x.device, dtype=x.dtype)
+      x = bias_dropout_scale_fn(
+        self.attn_out(x), None, scale, x_skip, self.dropout)
+      x = bias_dropout_scale_fn(
+        self.mlp(self.norm2(x)), None, scale, x, self.dropout)
+    return x
+
+
+class EmbeddingLayer(nn.Module):
+  def __init__(self, dim, vocab_dim):
+    super().__init__()
+    self.embedding = nn.Parameter(torch.empty((vocab_dim, dim)))
+    torch.nn.init.kaiming_uniform_(self.embedding, a=math.sqrt(5))
+
+  def forward(self, x):
+    if x.ndim == 2:
+      return self.embedding[x]
+    assert x.ndim == 3
+    return torch.einsum(
+      "blv,ve->ble",
+      torch.nn.functional.softmax(x, dim=-1).float(),
+      self.embedding.float()).to(x.dtype)
+
+
+class DDiTFinalLayer(nn.Module):
+  def __init__(self, hidden_size, out_channels, cond_dim,
+               adaLN):
+    super().__init__()
+    self.norm_final = LayerNorm(hidden_size)
+    self.linear = nn.Linear(hidden_size, out_channels)
+    self.linear.weight.data.zero_()
+    self.linear.bias.data.zero_()
+    self.adaLN = adaLN
+    if self.adaLN:
+      self.adaLN_modulation = nn.Linear(cond_dim,
+                                        2 * hidden_size,
+                                        bias=True)
+      self.adaLN_modulation.weight.data.zero_()
+      self.adaLN_modulation.bias.data.zero_()
+
+
+  def forward(self, x, c):
+    x = self.norm_final(x)
+    if self.adaLN:
+      shift, scale = self.adaLN_modulation(c)[:, None].chunk(2, dim=2)
+      x = modulate_fused(x, shift, scale)
+    x = self.linear(x)
+    return x
+
+
+class DIT(nn.Module, huggingface_hub.PyTorchModelHubMixin):
+  def __init__(self, config, vocab_size: int):
+    super().__init__()
+    if type(config) == dict:
+      config = omegaconf.OmegaConf.create(config)
+    self.causal = config.algo.causal_attention
+    self.adaLN = not self.causal
+    self.config = config
+    self.vocab_size = vocab_size
+    dim = config.model.hidden_size
+    cond_dim = config.model.cond_dim
+    self.vocab_embed = EmbeddingLayer(dim, vocab_size)
+    if not self.causal:
+      self.sigma_map = TimestepEmbedder(cond_dim)
+    self.rotary_emb = Rotary(dim // config.model.n_heads)
+
+    blocks = []
+    for _ in range(config.model.n_blocks):
+      if self.causal:
+        block = DDiTBlockCausal(
+          dim=dim,
+          n_heads=config.model.n_heads,
+          dropout=config.model.dropout)
+      else:
+        block = DDiTBlock(
+          dim=dim,
+          n_heads=config.model.n_heads,
+          cond_dim=cond_dim,
+          adaLN=self.adaLN,
+          dropout=config.model.dropout)
+      blocks.append(block)
+    self.blocks = nn.ModuleList(blocks)
+
+    self.output_layer = DDiTFinalLayer(
+      hidden_size=dim,
+      out_channels=vocab_size,
+      cond_dim=cond_dim,
+      adaLN=self.adaLN)
+    self.scale_by_sigma = config.model.scale_by_sigma
+
+  def _get_bias_dropout_scale(self):
+    if self.training:
+      return bias_dropout_add_scale_fused_train
+    else:
+      return  bias_dropout_add_scale_fused_inference
+
+  def forward(self, x, sigma):
+    x = self.vocab_embed(x)
+    if self.causal:
+      t_cond = None
+    else:
+      t_cond = F.silu(self.sigma_map(sigma))
+
+    rotary_cos_sin = self.rotary_emb(x).to(x.device)
+
+    with torch.cuda.amp.autocast(dtype=torch.bfloat16):
+      for i in range(len(self.blocks)):
+        x = self.blocks[i](x, rotary_cos_sin, c=t_cond)
+      x = self.output_layer(x, c=t_cond)
+
+    return x
+
+
+
+class HFDIT(torch.nn.Module):
+  def __init__(self, config):
+    super().__init__()
+    self.causal = config.causal
+    self.adaLN = not self.causal
+    self.vocab_size = config.vocab_size
+    dim = config.hidden_dim
+    cond_dim = config.cond_dim
+    self.vocab_embed = EmbeddingLayer(dim, self.vocab_size)
+    if not self.causal:
+      self.sigma_map = TimestepEmbedder(cond_dim)
+    self.rotary_emb = Rotary(dim // config.n_heads)
+
+    blocks = []
+    for _ in range(config.n_blocks):
+      if self.causal:
+        block = DDiTBlockCausal(
+          dim=dim,
+          n_heads=config.n_heads,
+          dropout=config.dropout)
+      else:
+        block = DDiTBlock(
+          dim=dim,
+          n_heads=config.n_heads,
+          cond_dim=cond_dim,
+          adaLN=self.adaLN,
+          dropout=config.dropout)
+      blocks.append(block)
+    self.blocks = torch.nn.ModuleList(blocks)
+
+    self.output_layer = DDiTFinalLayer(
+      hidden_size=dim,
+      out_channels=self.vocab_size,
+      cond_dim=cond_dim,
+      adaLN=self.adaLN)
+
+  def _get_bias_dropout_scale(self):
+    if self.training:
+      return bias_dropout_add_scale_fused_train
+    else:
+      return  bias_dropout_add_scale_fused_inference
+
+  def forward(self, x, sigma, output_hidden_states=False):
+    all_hidden_states = []
+    x = self.vocab_embed(x)
+    if output_hidden_states:
+      all_hidden_states.append(x)
+    if self.causal:
+      t_cond = None
+    else:
+      t_cond = F.silu(self.sigma_map(sigma))
+
+    rotary_cos_sin = self.rotary_emb(x)
+    with torch.cuda.amp.autocast(
+      dtype=torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32):
+      for i in range(len(self.blocks)):
+        x = self.blocks[i](x, rotary_cos_sin, c=t_cond)
+        if output_hidden_states:
+          all_hidden_states.append(x)
+      x = self.output_layer(x, c=t_cond)
+    return x, all_hidden_states
+
+
+class DUO(transformers.PreTrainedModel):
+  """HF-compatible model."""
+  config_class = DUOConfig
+  base_model_prefix = 'duo'
+
+  def __init__(self, config: DUOConfig):
+    super().__init__(config)
+    self.config = config
+    self.backbone = HFDIT(config)
+
+  def reset_kv_cache(self):
+    for block in self.backbone.blocks:
+      block.kv_cache = None
+
+  def forward(
+      self,
+      input_ids: torch.LongTensor = None,
+      timesteps: torch.FloatTensor = None,
+      output_hidden_states: typing.Optional[bool] = None,
+      return_dict: typing.Optional[bool] = None,
+  ) -> typing.Union[
+    torch.Tensor, typing.Tuple,
+    transformers.modeling_outputs.MaskedLMOutput]:
+    """HF-compatible forward method."""
+    output_hidden_states = (
+      output_hidden_states
+      if output_hidden_states is not None
+      else self.config.output_hidden_states
+    )
+    return_dict = return_dict \
+      if return_dict is not None \
+      else self.config.use_return_dict
+    
+    logits, all_hidden_states = self.backbone(
+      x=input_ids,
+      sigma=timesteps,
+      output_hidden_states=output_hidden_states,
+    )
+    if return_dict:
+      return transformers.modeling_outputs.MaskedLMOutput(
+        logits=logits,
+        hidden_states=all_hidden_states if output_hidden_states else None,
+        loss=None
+      )
+    elif output_hidden_states:
+      return logits, all_hidden_states
+    else:
+      return logits

checkpoints/quarantine/20260820T072244Z/official/mdlm/owt/config.json

@@ -0,0 +1,22 @@
+{
+  "_name_or_path": "kuleshov-group/mdlm-owt",
+  "architectures": [
+    "MDLM"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_mdlm.MDLMConfig",
+    "AutoModelForMaskedLM": "modeling_mdlm.MDLM"
+  },
+  "cond_dim": 128,
+  "dropout": 0.1,
+  "hidden_dim": 768,
+  "model_length": 1024,
+  "model_type": "mdlm",
+  "n_blocks": 12,
+  "n_heads": 12,
+  "return_dict": false,
+  "time_conditioning": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.38.2",
+  "vocab_size": 50258
+}

checkpoints/quarantine/20260820T073530Z/official/rdlm/lm1b/Text8/config.yaml

@@ -0,0 +1,73 @@
+seed: 0
+ngpus: 2
+run_mode: train
+num_workers: 8
+paths:
+  experiments: exps
+n_jobs: 2
+model:
+  _target_: model.dit.DiT
+  hidden_size: 768
+  n_heads: 12
+  cond_dim: 128
+  dropout: 0.0
+  n_blocks: 12
+  length: 256
+  input_dim: ${tokens}
+  output_dim: ${tokens}
+name: text8
+original_tokens: 27
+tokens: 27
+training:
+  batch_size: 512
+  accum: 1
+  n_iters: 1000001
+  snapshot_freq: 50000
+  log_freq: 50
+  eval_freq: 100
+  snapshot_freq_for_preemption: 5000
+  snapshot_sampling: true
+  ema: 0.9999
+  loss_type: ce
+data:
+  train: text8
+  valid: text8
+  cache_dir: data
+sampling:
+  steps: 256
+  predictor: grw
+eval:
+  batch_size: ${training.batch_size}
+  n_iters: 500
+  entropy: true
+  eval_model: gptj
+  nll: true
+  simul_steps: 500
+sde:
+  _target_: sde.LogBridge_Mixture
+  preprocess_steps: 10000
+  mix_type: step
+  rho_scale: 1.0
+  step_thr: 0.35
+  prior_dist:
+    _target_: distribution.Mixture
+    add_mask_token: true
+scheduler:
+  _target_: scheduler_lib.Geometric
+  beta_0: 0.001
+  beta_f: 0.2
+  weight_type: step
+  left: 0.3
+  right: 0.6
+  lb: 0.0001
+  ub: 1.0
+optim:
+  name: adamw
+  optimizer: AdamW
+  weight_decay: 0
+  lr: 0.0003
+  beta1: 0.9
+  beta2: 0.999
+  eps: 1.0e-08
+  warmup: 2500
+  grad_clip: 1.0

checkpoints/quarantine/20260821T024751Z/official/langflow/lm1b/__init__.py

@@ -0,0 +1,6 @@
+"""HuggingFace release package for LangFlow."""
+
+from .config import LangFlowConfig
+from .model import LangFlow
+
+__all__ = ["LangFlowConfig", "LangFlow"]

checkpoints/quarantine/20260821T024800Z/official/duo/owt/config.py

@@ -0,0 +1,29 @@
+import transformers
+
+
+class DUOConfig(transformers.PretrainedConfig):
+  """Hugging Face configuration class for DUO."""
+  model_type = 'DUO'
+
+  def __init__(
+    self,
+    vocab_size: int = 50258,
+    model_length: int = 1024,
+    causal: bool = False,
+    hidden_dim: int = 768,
+    cond_dim: int = 129,
+    n_blocks: int = 12,
+    n_heads: int = 12,
+    dropout: float = 0.1,
+    var_min: bool = True,
+    ** kwargs):
+    super().__init__(**kwargs)
+    self.causal = causal
+    self.vocab_size = vocab_size
+    self.model_length = model_length
+    self.hidden_dim = hidden_dim
+    self.cond_dim = cond_dim
+    self.n_blocks = n_blocks
+    self.n_heads = n_heads
+    self.dropout = dropout
+    self.var_min = var_min

checkpoints/quarantine/20260821T024808Z/official/duo_dcd/owt/README.md

@@ -0,0 +1,52 @@
+---
+datasets:
+- Skylion007/openwebtext
+language:
+- en
+library_name: transformers
+license: apache-2.0
+metrics:
+- perplexity
+pipeline_tag: text-generation
+---
+
+## Using DUO
+To use the pre-trained model for masked language modeling, use the following snippet:
+```python
+from transformers import AutoModelForMaskedLM, AutoTokenizer
+
+# See the `DUO` collection page on the hub for list of available models.
+tokenizer = transformers.AutoTokenizer.from_pretrained('gpt2')
+model = AutoModelForMaskedLM.from_pretrained('s-sahoo/duo-distilled')
+```
+For a hands-on example, check out this [Colab notebook](https://colab.research.google.com/drive/1Sf7R-dqdR6gq-H8nyZ9E3ZkyvqMTqcwq?usp=sharing).
+For more information and implementation details, visit our github repository: [DUO](https://github.com/s-sahoo/duo)
+
+## Model Details
+The model, which has a context length of `1024` and is similar in size to GPT2-medium with approximately `130 million` non-embedding parameters,
+was trained for 1M steps on the OpenWebText corpus.
+
+For more details, please see our paper: [The Diffusion Duality](https://huggingface.co/papers/2506.10892).
+
+Project page: https://s-sahoo.com/duo
+
+## Citation
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+Please cite our work using the bibtex below:
+
+**BibTeX:**
+
+```
+@inproceedings{
+sahoo2025the,
+title={The Diffusion Duality},
+author={Subham Sekhar Sahoo and Justin Deschenaux and Aaron Gokaslan and Guanghan Wang and Justin T Chiu and Volodymyr Kuleshov},
+booktitle={Forty-second International Conference on Machine Learning},
+year={2025},
+url={https://openreview.net/forum?id=9P9Y8FOSOk}
+}
+```
+
+## Model Card Contact
+Subham Sekhar Sahoo (ssahoo@cs.cornell.edu)

checkpoints/quarantine/20260821T024817Z/official/mdlm/owt/configuration_mdlm.py

@@ -0,0 +1,31 @@
+"""MDLM config for Hugging Face.
+
+"""
+
+import transformers
+
+
+class MDLMConfig(transformers.PretrainedConfig):
+  """Hugging Face configuration class for MDLM."""
+  model_type = "mdlm"
+
+  def __init__(
+    self,
+    vocab_size: int = 50258,
+    model_length: int = 1024,
+    hidden_dim: int = 768,
+    cond_dim: int = 129,
+    n_blocks: int = 12,
+    n_heads: int = 12,
+    dropout: float = 0.1,
+    time_conditioning: bool = False,
+    ** kwargs):
+    super().__init__(**kwargs)
+    self.vocab_size = vocab_size
+    self.model_length = model_length
+    self.hidden_dim = hidden_dim
+    self.cond_dim = cond_dim
+    self.n_blocks = n_blocks
+    self.n_heads = n_heads
+    self.dropout = dropout
+    self.time_conditioning = time_conditioning

checkpoints/quarantine/20260825T100201Z/official/langflow/owt/__init__.py

@@ -0,0 +1,6 @@
+"""HuggingFace release package for LangFlow."""
+
+from .config import LangFlowConfig
+from .model import LangFlow
+
+__all__ = ["LangFlowConfig", "LangFlow"]

checkpoints/quarantine/20260825T100215Z/official/langflow/lm1b/config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "nealchen/langflow-lm1b",
+  "architectures": [
+    "LangFlow"
+  ],
+  "auto_map": {
+    "AutoConfig": "config.LangFlowConfig",
+    "AutoModelForMaskedLM": "model.LangFlow"
+  },
+  "model_type": "LangFlow",
+  "vocab_size": 30522,
+  "hidden_size": 768,
+  "cond_dim": 128,
+  "n_blocks": 12,
+  "n_heads": 12,
+  "dropout": 0.1,
+  "model_length": 128,
+  "use_normalized_embedding": true,
+  "embedding_norm_method": "layernorm",
+  "self_conditioning": true,
+  "use_bias": true,
+  "gumbel_loc": 4.723,
+  "gumbel_scale": 0.852,
+  "gumbel_cutoff": 1e-5,
+  "gumbel_entropy": 7.02,
+  "return_dict": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.38.2"
+}

checkpoints/quarantine/20260825T100308Z/official/mdlm/owt/README.md

@@ -0,0 +1,67 @@
+---
+library_name: transformers
+license: apache-2.0
+language:
+- en
+datasets:
+- Skylion007/openwebtext
+metrics:
+- perplexity
+---
+
+## Using MDLM
+To use the pre-trained model for masked language modeling, use the following snippet:
+```python
+from transformers import AutoModelForMaskedLM, AutoTokenizer
+
+# See the `MDLM` collection page on the hub for list of available models.
+tokenizer = transformers.AutoTokenizer.from_pretrained('gpt2')
+model_name = 'kuleshov-group/mdlm-owt'
+model = AutoModelForMaskedLM.from_pretrained(model_name)
+```
+
+For more details, please see our github repository: [MDLM](https://github.com/kuleshov-group/mdlm)
+
+## Model Details
+The model, which has a context length of `1024` and is similar in size to GPT2-medium with approximately `130 million` non-embedding parameters,
+was trained using a forward diffusion process that generates inputs varying from fully masked to fully unmasked. Its objective is to
+reconstruct the original input from these varying levels of masking, outputting logits in the process.
+The training regimen comprised one million steps on the OpenWebText corpus, involving the processing of a total of `33 billion` tokens.
+
+For more details, please see our paper: [Simple and Effective Masked Diffusion Language Models](http://arxiv.org/abs/2406.07524).
+
+
+
+## Citation
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+Please cite our work using the bibtex below:
+
+**BibTeX:**
+
+```
+@inproceedings{
+    sahoo2024simple,
+    title={Simple and Effective Masked Diffusion Language Models},
+    author={Subham Sekhar Sahoo and Marianne Arriola and Aaron Gokaslan and Edgar Mariano Marroquin and Alexander M Rush and Yair Schiff and Justin T Chiu and Volodymyr Kuleshov},
+    booktitle={The Thirty-eighth Annual Conference on Neural Information Processing Systems},
+    year={2024},
+    url={https://openreview.net/forum?id=L4uaAR4ArM}
+}
+```
+
+**APA:**
+
+```
+@software{Sahoo_Simple_and_Effective_2024,
+author = {Sahoo, Subham Sekhar and Arriola, Marianne and Schiff, Yair and Gokaslan, Aaron and Marroquin, Edgar and Chiu, Justin T and Rush, Alexander and Kuleshov, Volodymyr},
+doi = {10.48550/arXiv.2406.07524},
+month = jun,
+title = {{Simple and Effective Masked Diffusion Language Models}},
+version = {arXiv:2406.07524v1},
+year = {2024}
+}
+```
+
+## Model Card Contact
+Subham Sekhar Sahoo (subbham@gmail.com)

checkpoints/quarantine/20260825T100308Z/official/mdlm/owt/configuration_mdlm.py

@@ -0,0 +1,31 @@
+"""MDLM config for Hugging Face.
+
+"""
+
+import transformers
+
+
+class MDLMConfig(transformers.PretrainedConfig):
+  """Hugging Face configuration class for MDLM."""
+  model_type = "mdlm"
+
+  def __init__(
+    self,
+    vocab_size: int = 50258,
+    model_length: int = 1024,
+    hidden_dim: int = 768,
+    cond_dim: int = 129,
+    n_blocks: int = 12,
+    n_heads: int = 12,
+    dropout: float = 0.1,
+    time_conditioning: bool = False,
+    ** kwargs):
+    super().__init__(**kwargs)
+    self.vocab_size = vocab_size
+    self.model_length = model_length
+    self.hidden_dim = hidden_dim
+    self.cond_dim = cond_dim
+    self.n_blocks = n_blocks
+    self.n_heads = n_heads
+    self.dropout = dropout
+    self.time_conditioning = time_conditioning

results/_archive/rerun_di4c_gpu4_20260825_110721/checkpoints/reference_reproduction/duo_di4c/lm1b/config.yaml

@@ -0,0 +1,180 @@
+mode: train
+time_conditioning: false
+T: 1024
+compile: false
+eval_every: 5000000
+is_di4c: true
+is_teacher_di4c: false
+latent_bsize: 16
+alpha_t: sigmoid
+alpha_const: 0.1
+distil_delta: 0.02
+log_unif: false
+round: 7
+data_preprocess:
+  data_cache: /home/ruizzhao/diffusion_baseline/data/training_cache/di4c/lm1b
+  min_seq_len: -1
+  seq_len: 128
+  group_text: true
+  remove_text: true
+  num_seqs: -1
+  add_bos: false
+  add_eos: true
+  legacy_start_end_bos: true
+loader:
+  global_batch_size: 2
+  eval_global_batch_size: 2
+  batch_size: 1
+  eval_batch_size: 1
+  num_workers: ${eval:"len(__import__('os').sched_getaffinity(0))"}
+  pin_memory: true
+  persistent_workers: true
+training:
+  ema: 0.9999
+  antithetic_sampling: true
+  importance_sampling: false
+  sampling_eps: 1.0e-05
+  change_of_variables: false
+tokenizer:
+  name: /home/ruizzhao/diffusion_baseline/data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594
+optim:
+  name: adamw
+  weight_decay: 0.0
+  lr: 3.0e-05
+  beta1: 0.9
+  beta2: 0.999
+  eps: 1.0e-08
+trainer:
+  _target_: lightning.Trainer
+  accelerator: cuda
+  num_nodes: 1
+  devices: 1
+  accumulate_grad_batches: 2
+  gradient_clip_val: 1.0
+  precision: bf16-mixed
+  num_sanity_val_steps: 0
+  max_steps: 20000
+  log_every_n_steps: 10
+  limit_train_batches: 1.0
+  limit_val_batches: 1.0
+  val_check_interval: ${eval:${trainer.accumulate_grad_batches} * ${eval_every}}
+  check_val_every_n_epoch: null
+  benchmark: true
+wandb: null
+checkpointing:
+  save_dir: /home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b
+  resume_from_ckpt: true
+  resume_ckpt_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b/checkpoints/last.ckpt
+eval:
+  valid:
+    n_samples: 16
+    num_steps: 128
+  ppl_with_ar:
+    run: true
+    model: gpt2-large
+    batch_size: 8
+  cond_ppl:
+    run: true
+    model: gpt2-large
+    batch_size: 8
+  mauve:
+    run: false
+    num_rounds: 5
+    max_num_tokens: 100
+    batch_size: 8
+    scaling_factor: 5
+  lambada_openai:
+    run: false
+    batch_size: 32
+    num_samples: 32
+    from_ema: false
+    add_bos: true
+    add_eos: true
+callbacks:
+  checkpoint_every_n_steps:
+    _target_: lightning.pytorch.callbacks.ModelCheckpoint
+    save_top_k: -1
+    save_last: true
+    dirpath: ${checkpointing.save_dir}/checkpoints
+    verbose: true
+    auto_insert_metric_name: false
+    every_n_train_steps: 10000
+  learning_rate_monitor:
+    _target_: lightning.pytorch.callbacks.LearningRateMonitor
+    logging_interval: step
+  text_throughput:
+    _target_: callbacks.text_throughput.TextThroughputMonitor
+  grad_norm:
+    _target_: callbacks.gradient_norm.GradNormCallback
+data:
+  train: lm1b
+  valid: lm1b
+  test: openwebtext-test
+  tokenizer_name_or_path: gpt2
+  wrap: true
+  streaming: false
+model:
+  name: small
+  type: ddit-orig
+  hidden_size: 768
+  cond_dim: 128
+  length: 128
+  n_blocks: 12
+  n_heads: 12
+  scale_by_sigma: true
+  dropout: 0.1
+  tie_word_embeddings: false
+strategy:
+  _target_: lightning.pytorch.strategies.DDPStrategy
+  find_unused_parameters: true
+noise:
+  type: loglinear
+  sigma_min: 0.0001
+  sigma_max: 20
+lr_scheduler:
+  _target_: transformers.get_constant_schedule_with_warmup
+  num_warmup_steps: 2500
+parameterization:
+  name: multi-round-sdtt
+  log_loss_buckets: -1
+  checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b/inputs/teacher_adapted.ckpt
+  start_from_hf: false
+  distill_mode: kl-bwd
+  num_distill_steps: 2
+  min_num_sampling_steps: 8
+  grow_dt_every: 10000
+  orig_num_sampling_steps: 1024
+  sampling_mode: ancestral
+  loss_precision: null
+  reset_optimizer_on_growth: false
+  use_ema_on_growth: false
+  sampling:
+    uncond:
+      run: true
+      num_samples: 1024
+      batch_size: 16
+      from_ema: false
+      num_steps: 16
+      seq_len: 1024
+      sampler: ancestral
+      cache_preds: false
+      add_bos: false
+      add_eos: false
+    cond_prefix:
+      run: true
+      num_samples: 256
+      batch_size: 16
+      from_ema: false
+      dataset: webtext
+      seq_len: 100
+      prefix_len: 50
+      num_cont_per_prefix: 5
+      min_seq_len: 1024
+      num_steps: 128
+      sampler: ancestral
+      cache_preds: false
+      add_bos: false
+      add_eos: false
+      calc_sbleu: true
+seed: 42
+dlb_student_init: teacher

results/_archive/rerun_di4c_gpu4_20260825_110721/checkpoints/reference_reproduction/duo_di4c/lm1b/recipe.json

@@ -0,0 +1,30 @@
+{
+  "algorithm": "di4c",
+  "checkpoint_every": 10000,
+  "dataset": "lm1b",
+  "default_per_device_batch_size": 1,
+  "distillation_loss": "dimensional_correlations",
+  "entrypoint": "../../adapters/train_di4c.py",
+  "evidence": "paper_appendix_d",
+  "global_batch_size": 2,
+  "learning_rate": 3e-05,
+  "learning_rate_override": "optim.lr",
+  "max_steps": 20000,
+  "max_steps_override": "trainer.max_steps",
+  "model": "duo_di4c",
+  "optimizer": "adam",
+  "output_override": "checkpointing.save_dir",
+  "precision": "bf16",
+  "release_evidence": null,
+  "release_script_max_steps": null,
+  "rounds": null,
+  "sampling_step": 20000,
+  "sequence_length": 128,
+  "source": "di4c",
+  "source_commit": "ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3",
+  "source_path": "upstreams/di4c",
+  "steps_per_round": null,
+  "teacher_adapter": "uniform_to_absorbing",
+  "teacher_family": "uniform_duo",
+  "warmup_steps": 2500
+}

results/_archive/rerun_di4c_gpu4_20260825_110721/checkpoints/reference_reproduction/duo_di4c/lm1b/stderr.log

@@ -0,0 +1,9 @@
+/opt/anaconda3/envs/dlb-di4c/lib/python3.10/site-packages/lightning/fabric/__init__.py:41: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.
+Seed set to 42
+2026-08-25 10:57:51.062 | INFO     | sdtt.main:di4c_train:77 - Starting training
+2026-08-25 10:57:51.063 | INFO     | sdtt.main:di4c_train:104 - Training starting from scratch (no checkpoint to reload)
+Error executing job with overrides: ['mode=train', 'data.train=lm1b', 'data.valid=lm1b', 'tokenizer.name=/home/ruizzhao/diffusion_baseline/data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594', 'data_preprocess.data_cache=/home/ruizzhao/diffusion_baseline/data/training_cache/di4c/lm1b', 'data_preprocess.seq_len=128', 'data_preprocess.group_text=true', 'data_preprocess.remove_text=true', 'data_preprocess.add_bos=false', 'data_preprocess.add_eos=true', 'data_preprocess.legacy_start_end_bos=true', 'model=dit-orig-small', 'model.length=128', 'parameterization=multi-round-sdtt', 'parameterization.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b/inputs/teacher_adapted.ckpt', 'parameterization.start_from_hf=false', 'parameterization.grow_dt_every=10000', 'parameterization.orig_num_sampling_steps=1024', 'loader.global_batch_size=2', 'loader.eval_global_batch_size=2', 'loader.batch_size=1', 'loader.eval_batch_size=1', 'trainer.devices=1', 'trainer.num_nodes=1', 'trainer.accumulate_grad_batches=2', 'trainer.max_steps=20000', 'trainer.precision=bf16-mixed', 'optim.lr=3e-05', 'lr_scheduler.num_warmup_steps=2500', 'callbacks.checkpoint_every_n_steps.every_n_train_steps=10000', 'checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b', 'checkpointing.resume_from_ckpt=true', 'checkpointing.resume_ckpt_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b/checkpoints/last.ckpt', 'wandb=null', '+seed=42', 'is_di4c=true', 'is_teacher_di4c=false', 'T=1024', 'latent_bsize=16', 'round=7', '+dlb_student_init=teacher']
+Error locating target 'callbacks.text_throughput.TextThroughputMonitor', set env var HYDRA_FULL_ERROR=1 to see chained exception.
+full_key: callbacks.text_throughput
+
+Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.

results/_archive/rerun_di4c_gpu4_gpu7_20260825_111320/checkpoints/reference_reproduction/duo_di4c/lm1b/.hydra/config.yaml

@@ -0,0 +1,180 @@
+mode: train
+time_conditioning: false
+T: 1024
+compile: false
+eval_every: 5000000
+is_di4c: true
+is_teacher_di4c: false
+latent_bsize: 16
+alpha_t: sigmoid
+alpha_const: 0.1
+distil_delta: 0.02
+log_unif: false
+round: 7
+data_preprocess:
+  data_cache: /home/ruizzhao/diffusion_baseline/data/training_cache/di4c/lm1b
+  min_seq_len: -1
+  seq_len: 128
+  group_text: true
+  remove_text: true
+  num_seqs: -1
+  add_bos: false
+  add_eos: true
+  legacy_start_end_bos: true
+loader:
+  global_batch_size: 2
+  eval_global_batch_size: 2
+  batch_size: 1
+  eval_batch_size: 1
+  num_workers: ${eval:"len(__import__('os').sched_getaffinity(0))"}
+  pin_memory: true
+  persistent_workers: true
+training:
+  ema: 0.9999
+  antithetic_sampling: true
+  importance_sampling: false
+  sampling_eps: 1.0e-05
+  change_of_variables: false
+tokenizer:
+  name: /home/ruizzhao/diffusion_baseline/data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594
+optim:
+  name: adamw
+  weight_decay: 0.0
+  lr: 3.0e-05
+  beta1: 0.9
+  beta2: 0.999
+  eps: 1.0e-08
+trainer:
+  _target_: lightning.Trainer
+  accelerator: cuda
+  num_nodes: 1
+  devices: 1
+  accumulate_grad_batches: 2
+  gradient_clip_val: 1.0
+  precision: bf16-mixed
+  num_sanity_val_steps: 0
+  max_steps: 20000
+  log_every_n_steps: 10
+  limit_train_batches: 1.0
+  limit_val_batches: 1.0
+  val_check_interval: ${eval:${trainer.accumulate_grad_batches} * ${eval_every}}
+  check_val_every_n_epoch: null
+  benchmark: true
+wandb: null
+checkpointing:
+  save_dir: /home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b
+  resume_from_ckpt: true
+  resume_ckpt_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b/checkpoints/last.ckpt
+eval:
+  valid:
+    n_samples: 16
+    num_steps: 128
+  ppl_with_ar:
+    run: true
+    model: gpt2-large
+    batch_size: 8
+  cond_ppl:
+    run: true
+    model: gpt2-large
+    batch_size: 8
+  mauve:
+    run: false
+    num_rounds: 5
+    max_num_tokens: 100
+    batch_size: 8
+    scaling_factor: 5
+  lambada_openai:
+    run: false
+    batch_size: 32
+    num_samples: 32
+    from_ema: false
+    add_bos: true
+    add_eos: true
+callbacks:
+  checkpoint_every_n_steps:
+    _target_: lightning.pytorch.callbacks.ModelCheckpoint
+    save_top_k: -1
+    save_last: true
+    dirpath: ${checkpointing.save_dir}/checkpoints
+    verbose: true
+    auto_insert_metric_name: false
+    every_n_train_steps: 10000
+  learning_rate_monitor:
+    _target_: lightning.pytorch.callbacks.LearningRateMonitor
+    logging_interval: step
+  text_throughput:
+    _target_: callbacks.text_throughput.TextThroughputMonitor
+  grad_norm:
+    _target_: callbacks.gradient_norm.GradNormCallback
+data:
+  train: lm1b
+  valid: lm1b
+  test: openwebtext-test
+  tokenizer_name_or_path: gpt2
+  wrap: true
+  streaming: false
+model:
+  name: small
+  type: ddit-orig
+  hidden_size: 768
+  cond_dim: 128
+  length: 128
+  n_blocks: 12
+  n_heads: 12
+  scale_by_sigma: true
+  dropout: 0.1
+  tie_word_embeddings: false
+strategy:
+  _target_: lightning.pytorch.strategies.DDPStrategy
+  find_unused_parameters: true
+noise:
+  type: loglinear
+  sigma_min: 0.0001
+  sigma_max: 20
+lr_scheduler:
+  _target_: transformers.get_constant_schedule_with_warmup
+  num_warmup_steps: 2500
+parameterization:
+  name: multi-round-sdtt
+  log_loss_buckets: -1
+  checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b/inputs/teacher_adapted.ckpt
+  start_from_hf: false
+  distill_mode: kl-bwd
+  num_distill_steps: 2
+  min_num_sampling_steps: 8
+  grow_dt_every: 10000
+  orig_num_sampling_steps: 1024
+  sampling_mode: ancestral
+  loss_precision: null
+  reset_optimizer_on_growth: false
+  use_ema_on_growth: false
+  sampling:
+    uncond:
+      run: true
+      num_samples: 1024
+      batch_size: 16
+      from_ema: false
+      num_steps: 16
+      seq_len: 1024
+      sampler: ancestral
+      cache_preds: false
+      add_bos: false
+      add_eos: false
+    cond_prefix:
+      run: true
+      num_samples: 256
+      batch_size: 16
+      from_ema: false
+      dataset: webtext
+      seq_len: 100
+      prefix_len: 50
+      num_cont_per_prefix: 5
+      min_seq_len: 1024
+      num_steps: 128
+      sampler: ancestral
+      cache_preds: false
+      add_bos: false
+      add_eos: false
+      calc_sbleu: true
+seed: 42
+dlb_student_init: teacher

results/_archive/rerun_di4c_gpu4_gpu7_20260825_111320/checkpoints/reference_reproduction/duo_di4c/lm1b/.hydra/hydra.yaml

@@ -0,0 +1,203 @@
+hydra:
+  run:
+    dir: /home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b
+    - hydra.job.chdir=true
+    - hydra.mode=RUN
+    task:
+    - mode=train
+    - data.train=lm1b
+    - data.valid=lm1b
+    - tokenizer.name=/home/ruizzhao/diffusion_baseline/data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594
+    - data_preprocess.data_cache=/home/ruizzhao/diffusion_baseline/data/training_cache/di4c/lm1b
+    - data_preprocess.seq_len=128
+    - data_preprocess.group_text=true
+    - data_preprocess.remove_text=true
+    - data_preprocess.add_bos=false
+    - data_preprocess.add_eos=true
+    - data_preprocess.legacy_start_end_bos=true
+    - model=dit-orig-small
+    - model.length=128
+    - parameterization=multi-round-sdtt
+    - parameterization.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b/inputs/teacher_adapted.ckpt
+    - parameterization.start_from_hf=false
+    - parameterization.grow_dt_every=10000
+    - parameterization.orig_num_sampling_steps=1024
+    - loader.global_batch_size=2
+    - loader.eval_global_batch_size=2
+    - loader.batch_size=1
+    - loader.eval_batch_size=1
+    - trainer.devices=1
+    - trainer.num_nodes=1
+    - trainer.accumulate_grad_batches=2
+    - trainer.max_steps=20000
+    - trainer.precision=bf16-mixed
+    - optim.lr=3e-05
+    - lr_scheduler.num_warmup_steps=2500
+    - callbacks.checkpoint_every_n_steps.every_n_train_steps=10000
+    - checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b
+    - checkpointing.resume_from_ckpt=true
+    - checkpointing.resume_ckpt_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b/checkpoints/last.ckpt
+    - wandb=null
+    - +seed=42
+    - is_di4c=true
+    - is_teacher_di4c=false
+    - T=1024
+    - latent_bsize=16
+    - round=7
+    - +dlb_student_init=teacher
+  job:
+    name: train_di4c
+    chdir: true
+    override_dirname: +dlb_student_init=teacher,+seed=42,T=1024,callbacks.checkpoint_every_n_steps.every_n_train_steps=10000,checkpointing.resume_ckpt_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b/checkpoints/last.ckpt,checkpointing.resume_from_ckpt=true,checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b,data.train=lm1b,data.valid=lm1b,data_preprocess.add_bos=false,data_preprocess.add_eos=true,data_preprocess.data_cache=/home/ruizzhao/diffusion_baseline/data/training_cache/di4c/lm1b,data_preprocess.group_text=true,data_preprocess.legacy_start_end_bos=true,data_preprocess.remove_text=true,data_preprocess.seq_len=128,is_di4c=true,is_teacher_di4c=false,latent_bsize=16,loader.batch_size=1,loader.eval_batch_size=1,loader.eval_global_batch_size=2,loader.global_batch_size=2,lr_scheduler.num_warmup_steps=2500,mode=train,model.length=128,model=dit-orig-small,optim.lr=3e-05,parameterization.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b/inputs/teacher_adapted.ckpt,parameterization.grow_dt_every=10000,parameterization.orig_num_sampling_steps=1024,parameterization.start_from_hf=false,parameterization=multi-round-sdtt,round=7,tokenizer.name=/home/ruizzhao/diffusion_baseline/data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594,trainer.accumulate_grad_batches=2,trainer.devices=1,trainer.max_steps=20000,trainer.num_nodes=1,trainer.precision=bf16-mixed,wandb=null
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /home/ruizzhao/diffusion_baseline/upstreams/di4c
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/ruizzhao/diffusion_baseline/upstreams/di4c/sdtt/src/sdtt/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b
+    choices:
+      parameterization: multi-round-sdtt
+      lr_scheduler: constant_warmup
+      noise: loglinear
+      strategy: ddp
+      model: dit-orig-small
+      data: openwebtext-split
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

results/_archive/rerun_di4c_gpu4_gpu7_20260825_111320/checkpoints/reference_reproduction/duo_di4c/lm1b/.hydra/overrides.yaml

@@ -0,0 +1,41 @@
+- mode=train
+- data.train=lm1b
+- data.valid=lm1b
+- tokenizer.name=/home/ruizzhao/diffusion_baseline/data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594
+- data_preprocess.data_cache=/home/ruizzhao/diffusion_baseline/data/training_cache/di4c/lm1b
+- data_preprocess.seq_len=128
+- data_preprocess.group_text=true
+- data_preprocess.remove_text=true
+- data_preprocess.add_bos=false
+- data_preprocess.add_eos=true
+- data_preprocess.legacy_start_end_bos=true
+- model=dit-orig-small
+- model.length=128
+- parameterization=multi-round-sdtt
+- parameterization.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b/inputs/teacher_adapted.ckpt
+- parameterization.start_from_hf=false
+- parameterization.grow_dt_every=10000
+- parameterization.orig_num_sampling_steps=1024
+- loader.global_batch_size=2
+- loader.eval_global_batch_size=2
+- loader.batch_size=1
+- loader.eval_batch_size=1
+- trainer.devices=1
+- trainer.num_nodes=1
+- trainer.accumulate_grad_batches=2
+- trainer.max_steps=20000
+- trainer.precision=bf16-mixed
+- optim.lr=3e-05
+- lr_scheduler.num_warmup_steps=2500
+- callbacks.checkpoint_every_n_steps.every_n_train_steps=10000
+- checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b
+- checkpointing.resume_from_ckpt=true
+- checkpointing.resume_ckpt_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b/checkpoints/last.ckpt
+- wandb=null
+- +seed=42
+- is_di4c=true
+- is_teacher_di4c=false
+- T=1024
+- latent_bsize=16
+- round=7
+- +dlb_student_init=teacher

results/_archive/rerun_di4c_gpu4_gpu7_20260825_111320/checkpoints/reference_reproduction/duo_di4c/lm1b/launch_argv.json

@@ -0,0 +1,48 @@
+[
+  "/opt/anaconda3/envs/dlb-di4c/bin/python",
+  "-u",
+  "/home/ruizzhao/diffusion_baseline/upstreams/di4c/../../adapters/train_di4c.py",
+  "mode=train",
+  "data.train=lm1b",
+  "data.valid=lm1b",
+  "tokenizer.name=/home/ruizzhao/diffusion_baseline/data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594",
+  "data_preprocess.data_cache=/home/ruizzhao/diffusion_baseline/data/training_cache/di4c/lm1b",
+  "data_preprocess.seq_len=128",
+  "data_preprocess.group_text=true",
+  "data_preprocess.remove_text=true",
+  "data_preprocess.add_bos=false",
+  "data_preprocess.add_eos=true",
+  "data_preprocess.legacy_start_end_bos=true",
+  "model=dit-orig-small",
+  "model.length=128",
+  "parameterization=multi-round-sdtt",
+  "parameterization.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b/inputs/teacher_adapted.ckpt",
+  "parameterization.start_from_hf=false",
+  "parameterization.grow_dt_every=10000",
+  "parameterization.orig_num_sampling_steps=1024",
+  "loader.global_batch_size=2",
+  "loader.eval_global_batch_size=2",
+  "loader.batch_size=1",
+  "loader.eval_batch_size=1",
+  "trainer.devices=1",
+  "trainer.num_nodes=1",
+  "trainer.accumulate_grad_batches=2",
+  "trainer.max_steps=20000",
+  "trainer.precision=bf16-mixed",
+  "optim.lr=3e-05",
+  "lr_scheduler.num_warmup_steps=2500",
+  "callbacks.checkpoint_every_n_steps.every_n_train_steps=10000",
+  "checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b",
+  "checkpointing.resume_from_ckpt=true",
+  "checkpointing.resume_ckpt_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b/checkpoints/last.ckpt",
+  "hydra.run.dir=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b",
+  "hydra.job.chdir=true",
+  "wandb=null",
+  "+seed=42",
+  "is_di4c=true",
+  "is_teacher_di4c=false",
+  "T=1024",
+  "latent_bsize=16",
+  "round=7",
+  "+dlb_student_init=teacher"
+]

results/_archive/rerun_di4c_gpu4_gpu7_20260825_111320/checkpoints/reference_reproduction/duo_di4c/lm1b/provenance.json

@@ -0,0 +1,121 @@
+{
+  "command": [
+    "/opt/anaconda3/envs/dlb-di4c/bin/python",
+    "-u",
+    "/home/ruizzhao/diffusion_baseline/upstreams/di4c/../../adapters/train_di4c.py",
+    "mode=train",
+    "data.train=lm1b",
+    "data.valid=lm1b",
+    "tokenizer.name=/home/ruizzhao/diffusion_baseline/data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594",
+    "data_preprocess.data_cache=/home/ruizzhao/diffusion_baseline/data/training_cache/di4c/lm1b",
+    "data_preprocess.seq_len=128",
+    "data_preprocess.group_text=true",
+    "data_preprocess.remove_text=true",
+    "data_preprocess.add_bos=false",
+    "data_preprocess.add_eos=true",
+    "data_preprocess.legacy_start_end_bos=true",
+    "model=dit-orig-small",
+    "model.length=128",
+    "parameterization=multi-round-sdtt",
+    "parameterization.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b/inputs/teacher_adapted.ckpt",
+    "parameterization.start_from_hf=false",
+    "parameterization.grow_dt_every=10000",
+    "parameterization.orig_num_sampling_steps=1024",
+    "loader.global_batch_size=2",
+    "loader.eval_global_batch_size=2",
+    "loader.batch_size=1",
+    "loader.eval_batch_size=1",
+    "trainer.devices=1",
+    "trainer.num_nodes=1",
+    "trainer.accumulate_grad_batches=2",
+    "trainer.max_steps=20000",
+    "trainer.precision=bf16-mixed",
+    "optim.lr=3e-05",
+    "lr_scheduler.num_warmup_steps=2500",
+    "callbacks.checkpoint_every_n_steps.every_n_train_steps=10000",
+    "checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b",
+    "checkpointing.resume_from_ckpt=true",
+    "checkpointing.resume_ckpt_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b/checkpoints/last.ckpt",
+    "hydra.run.dir=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b",
+    "hydra.job.chdir=true",
+    "wandb=null",
+    "+seed=42",
+    "is_di4c=true",
+    "is_teacher_di4c=false",
+    "T=1024",
+    "latent_bsize=16",
+    "round=7",
+    "+dlb_student_init=teacher"
+  ],
+  "composed_config_sha256": "624ca67b5af09ff63ecd60efa7ff08d70ed8391b276f7ad6d3709607b4d43aa8",
+  "cwd": "/home/ruizzhao/diffusion_baseline/upstreams/di4c",
+  "data": {
+    "aliases": {
+      "lm1b_test_bs128_wrapped.dat": "/home/ruizzhao/diffusion_baseline/data/processed/lm1b-bert-128/validation",
+      "lm1b_train_bs128_wrapped.dat": "/home/ruizzhao/diffusion_baseline/data/processed/lm1b-bert-128/train"
+    },
+    "cache": "/home/ruizzhao/diffusion_baseline/data/training_cache/di4c/lm1b",
+    "format": "hf_dataset_with_attention_mask_v1",
+    "mode": "materialized_bridge",
+    "processed_manifest_sha256": "a8465f3fba8c5dc529fc0751039c8ffa32a7710f480208779b55545cb141017c",
+    "processed_path": "/home/ruizzhao/diffusion_baseline/data/processed/lm1b-bert-128"
+  },
+  "devices": 1,
+  "effective_global_batch_size": 2,
+  "gradient_accumulation": 2,
+  "identity_sha256": "0e6b980c0ad2928f4fe7d0a431da99b5658749371a96429bd86e57bd5993ce49",
+  "nodes": 1,
+  "per_device_batch_size": 1,
+  "prepared_at": "2026-08-25T03:07:30.752620Z",
+  "recipe": {
+    "algorithm": "di4c",
+    "checkpoint_every": 10000,
+    "dataset": "lm1b",
+    "default_per_device_batch_size": 1,
+    "distillation_loss": "dimensional_correlations",
+    "entrypoint": "../../adapters/train_di4c.py",
+    "evidence": "paper_appendix_d",
+    "global_batch_size": 2,
+    "learning_rate": 3e-05,
+    "learning_rate_override": "optim.lr",
+    "max_steps": 20000,
+    "max_steps_override": "trainer.max_steps",
+    "model": "duo_di4c",
+    "optimizer": "adam",
+    "output_override": "checkpointing.save_dir",
+    "precision": "bf16",
+    "release_evidence": null,
+    "release_script_max_steps": null,
+    "rounds": null,
+    "sampling_step": 20000,
+    "sequence_length": 128,
+    "source": "di4c",
+    "source_commit": "ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3",
+    "source_path": "upstreams/di4c",
+    "steps_per_round": null,
+    "teacher_adapter": "uniform_to_absorbing",
+    "teacher_family": "uniform_duo",
+    "warmup_steps": 2500
+  },
+  "schema_version": 1,
+  "seed": 42,
+  "source_commit": "ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3",
+  "teacher_adaptation": {
+    "hidden_size": 768,
+    "output_path": "/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b/inputs/teacher_adapted.ckpt",
+    "output_sha256": "5f34e31d2743d9851c4314f52861e5f2366bfdbea51b156a9bd102fc592f0b23",
+    "schema_version": 1,
+    "source_mask_index": null,
+    "source_path": "/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/flm_baselines/lm1b/lm1b_Duo.ckpt",
+    "source_sha256": "b11ee9a4cc533435adb9e1149cff74aec0872a9df2a459b4ab58f1867f6cddfb",
+    "source_vocab_size": 30522,
+    "state_keys_sha256": "8212d507b6931a59375f8127f0ccb9e51c24da74fe05e974648d92c5612fb803",
+    "target_mask_index": 30522,
+    "target_vocab_size": 30523,
+    "teacher_family": "uniform_duo",
+    "transformation": "uniform_to_absorbing",
+    "transformation_sha256": "656341f23524d65a7207e53a85fe06088ad7deaa15365d2ee91ac5fb6eac2cd0"
+  },
+  "teacher_sha256": "b11ee9a4cc533435adb9e1149cff74aec0872a9df2a459b4ab58f1867f6cddfb",
+  "teacher_transformation_sha256": "656341f23524d65a7207e53a85fe06088ad7deaa15365d2ee91ac5fb6eac2cd0"
+}

results/_archive/rerun_di4c_gpu4_gpu7_20260825_111320/checkpoints/reference_reproduction/duo_di4c/lm1b/recipe.json

@@ -0,0 +1,30 @@
+{
+  "algorithm": "di4c",
+  "checkpoint_every": 10000,
+  "dataset": "lm1b",
+  "default_per_device_batch_size": 1,
+  "distillation_loss": "dimensional_correlations",
+  "entrypoint": "../../adapters/train_di4c.py",
+  "evidence": "paper_appendix_d",
+  "global_batch_size": 2,
+  "learning_rate": 3e-05,
+  "learning_rate_override": "optim.lr",
+  "max_steps": 20000,
+  "max_steps_override": "trainer.max_steps",
+  "model": "duo_di4c",
+  "optimizer": "adam",
+  "output_override": "checkpointing.save_dir",
+  "precision": "bf16",
+  "release_evidence": null,
+  "release_script_max_steps": null,
+  "rounds": null,
+  "sampling_step": 20000,
+  "sequence_length": 128,
+  "source": "di4c",
+  "source_commit": "ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3",
+  "source_path": "upstreams/di4c",
+  "steps_per_round": null,
+  "teacher_adapter": "uniform_to_absorbing",
+  "teacher_family": "uniform_duo",
+  "warmup_steps": 2500
+}

results/_archive/rerun_di4c_gpu4_gpu7_20260825_111320/checkpoints/reference_reproduction/duo_di4c/lm1b/stderr.log

@@ -0,0 +1,9 @@
+/opt/anaconda3/envs/dlb-di4c/lib/python3.10/site-packages/lightning/fabric/__init__.py:41: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.
+Seed set to 42
+2026-08-25 11:07:33.667 | INFO     | sdtt.main:di4c_train:77 - Starting training
+2026-08-25 11:07:33.667 | INFO     | sdtt.main:di4c_train:104 - Training starting from scratch (no checkpoint to reload)
+Error executing job with overrides: ['mode=train', 'data.train=lm1b', 'data.valid=lm1b', 'tokenizer.name=/home/ruizzhao/diffusion_baseline/data/raw/huggingface/hub/models--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594', 'data_preprocess.data_cache=/home/ruizzhao/diffusion_baseline/data/training_cache/di4c/lm1b', 'data_preprocess.seq_len=128', 'data_preprocess.group_text=true', 'data_preprocess.remove_text=true', 'data_preprocess.add_bos=false', 'data_preprocess.add_eos=true', 'data_preprocess.legacy_start_end_bos=true', 'model=dit-orig-small', 'model.length=128', 'parameterization=multi-round-sdtt', 'parameterization.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b/inputs/teacher_adapted.ckpt', 'parameterization.start_from_hf=false', 'parameterization.grow_dt_every=10000', 'parameterization.orig_num_sampling_steps=1024', 'loader.global_batch_size=2', 'loader.eval_global_batch_size=2', 'loader.batch_size=1', 'loader.eval_batch_size=1', 'trainer.devices=1', 'trainer.num_nodes=1', 'trainer.accumulate_grad_batches=2', 'trainer.max_steps=20000', 'trainer.precision=bf16-mixed', 'optim.lr=3e-05', 'lr_scheduler.num_warmup_steps=2500', 'callbacks.checkpoint_every_n_steps.every_n_train_steps=10000', 'checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b', 'checkpointing.resume_from_ckpt=true', 'checkpointing.resume_ckpt_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_di4c/lm1b/checkpoints/last.ckpt', 'wandb=null', '+seed=42', 'is_di4c=true', 'is_teacher_di4c=false', 'T=1024', 'latent_bsize=16', 'round=7', '+dlb_student_init=teacher']
+Error locating target 'callbacks.text_throughput.TextThroughputMonitor', set env var HYDRA_FULL_ERROR=1 to see chained exception.
+full_key: callbacks.text_throughput
+
+Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.

results/conditional/metrics/lm1b/flm/steps_32/metrics.json

@@ -0,0 +1,95 @@
+{
+  "dataset": "lm1b",
+  "evaluation_continuation_length": 64,
+  "metrics": {
+    "conditional_generative_perplexity": {
+      "aggregation": "sum_nll_over_sum_valid_next_tokens",
+      "batch_size": 8,
+      "context_length": 1024,
+      "model_id": "gpt2-large",
+      "model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
+      "model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
+      "offline": true,
+      "perplexity": 12.092915980161044,
+      "prompt_loss_excluded": true,
+      "sample_count": 2048,
+      "scored_slice": "source_tokens_64_128_only",
+      "tokenizer_id": "gpt2",
+      "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "total_nll": 604250.9274902344,
+      "truncation": "right",
+      "valid_token_count": 242416
+    },
+    "mauve_suffix": {
+      "comparison": "generated_suffix_vs_reference_suffix",
+      "prefix_included": false,
+      "sample_count": 2048,
+      "score": 0.0040720962619612555,
+      "slice": "source_tokens_64_128_only"
+    },
+    "prefix_exact_match": {
+      "matched": 2048,
+      "rate": 1.0,
+      "required_rate": 1.0,
+      "sample_count": 2048
+    },
+    "sample_entropy": {
+      "generated": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [
+          0
+        ],
+        "logarithm": "natural",
+        "mean_entropy": 2.0806292940402793,
+        "sample_count": 2048,
+        "token_count": 131072,
+        "unit": "nats"
+      },
+      "generated_minus_reference": -1.7365233051707354,
+      "reference": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [
+          0
+        ],
+        "logarithm": "natural",
+        "mean_entropy": 3.8171525992110147,
+        "sample_count": 2048,
+        "token_count": 131072,
+        "unit": "nats"
+      },
+      "slice": "source_tokens_64_128_only"
+    },
+    "self_bleu": {
+      "aggregation": "mean_prompt_group_self_bleu",
+      "completions_per_prompt": 5,
+      "ngram_order": 4,
+      "prompt_count": 256,
+      "sample_count": 1280,
+      "score": 0.639138127621003,
+      "slice": "generated_source_tokens_64_128_only",
+      "smoothing": "chen_cherry_method1",
+      "tokenization": "gpt2",
+      "weights": [
+        0.25,
+        0.25,
+        0.25,
+        0.25
+      ]
+    }
+  },
+  "prefix_length": 64,
+  "production_sample_count": 2048,
+  "protocol": "c64_zs_v1",
+  "requested_metrics": [
+    "conditional_gen_ppl",
+    "mauve_suffix",
+    "entropy",
+    "self_bleu",
+    "prefix_exact_match"
+  ],
+  "sample_count": 2048,
+  "samples": "results/conditional/samples/lm1b/flm/steps_32/samples.jsonl",
+  "samples_sha256": "82fa5f29b341c7ac071fd01cda78655310c334db743bb29a66f39deb731a34fb",
+  "schema_version": 1
+}

results/conditional/metrics/lm1b/flm/steps_8/metrics.json

@@ -0,0 +1,95 @@
+{
+  "dataset": "lm1b",
+  "evaluation_continuation_length": 64,
+  "metrics": {
+    "conditional_generative_perplexity": {
+      "aggregation": "sum_nll_over_sum_valid_next_tokens",
+      "batch_size": 8,
+      "context_length": 1024,
+      "model_id": "gpt2-large",
+      "model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
+      "model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
+      "offline": true,
+      "perplexity": 6.00491602998456,
+      "prompt_loss_excluded": true,
+      "sample_count": 2048,
+      "scored_slice": "source_tokens_64_128_only",
+      "tokenizer_id": "gpt2",
+      "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "total_nll": 551520.8259277344,
+      "truncation": "right",
+      "valid_token_count": 307669
+    },
+    "mauve_suffix": {
+      "comparison": "generated_suffix_vs_reference_suffix",
+      "prefix_included": false,
+      "sample_count": 2048,
+      "score": 0.004179943930085514,
+      "slice": "source_tokens_64_128_only"
+    },
+    "prefix_exact_match": {
+      "matched": 2048,
+      "rate": 1.0,
+      "required_rate": 1.0,
+      "sample_count": 2048
+    },
+    "sample_entropy": {
+      "generated": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [
+          0
+        ],
+        "logarithm": "natural",
+        "mean_entropy": 1.8097444920868058,
+        "sample_count": 2048,
+        "token_count": 131072,
+        "unit": "nats"
+      },
+      "generated_minus_reference": -2.007408107124209,
+      "reference": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [
+          0
+        ],
+        "logarithm": "natural",
+        "mean_entropy": 3.8171525992110147,
+        "sample_count": 2048,
+        "token_count": 131072,
+        "unit": "nats"
+      },
+      "slice": "source_tokens_64_128_only"
+    },
+    "self_bleu": {
+      "aggregation": "mean_prompt_group_self_bleu",
+      "completions_per_prompt": 5,
+      "ngram_order": 4,
+      "prompt_count": 256,
+      "sample_count": 1280,
+      "score": 0.7593250173826942,
+      "slice": "generated_source_tokens_64_128_only",
+      "smoothing": "chen_cherry_method1",
+      "tokenization": "gpt2",
+      "weights": [
+        0.25,
+        0.25,
+        0.25,
+        0.25
+      ]
+    }
+  },
+  "prefix_length": 64,
+  "production_sample_count": 2048,
+  "protocol": "c64_zs_v1",
+  "requested_metrics": [
+    "conditional_gen_ppl",
+    "mauve_suffix",
+    "entropy",
+    "self_bleu",
+    "prefix_exact_match"
+  ],
+  "sample_count": 2048,
+  "samples": "results/conditional/samples/lm1b/flm/steps_8/samples.jsonl",
+  "samples_sha256": "f1128565f9be8e1b69c582877077a47ccfcfd5cd2241f885a716a5b812bd0ee7",
+  "schema_version": 1
+}

results/conditional/metrics/lm1b/fmlm/steps_32/metrics.json

@@ -0,0 +1,95 @@
+{
+  "dataset": "lm1b",
+  "evaluation_continuation_length": 64,
+  "metrics": {
+    "conditional_generative_perplexity": {
+      "aggregation": "sum_nll_over_sum_valid_next_tokens",
+      "batch_size": 8,
+      "context_length": 1024,
+      "model_id": "gpt2-large",
+      "model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
+      "model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
+      "offline": true,
+      "perplexity": 81.08235208356075,
+      "prompt_loss_excluded": true,
+      "sample_count": 2048,
+      "scored_slice": "source_tokens_64_128_only",
+      "tokenizer_id": "gpt2",
+      "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "total_nll": 658286.865234375,
+      "truncation": "right",
+      "valid_token_count": 149765
+    },
+    "mauve_suffix": {
+      "comparison": "generated_suffix_vs_reference_suffix",
+      "prefix_included": false,
+      "sample_count": 2048,
+      "score": 0.025989271366052474,
+      "slice": "source_tokens_64_128_only"
+    },
+    "prefix_exact_match": {
+      "matched": 2048,
+      "rate": 1.0,
+      "required_rate": 1.0,
+      "sample_count": 2048
+    },
+    "sample_entropy": {
+      "generated": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [
+          0
+        ],
+        "logarithm": "natural",
+        "mean_entropy": 3.8008536082446445,
+        "sample_count": 2048,
+        "token_count": 131072,
+        "unit": "nats"
+      },
+      "generated_minus_reference": -0.01629899096637022,
+      "reference": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [
+          0
+        ],
+        "logarithm": "natural",
+        "mean_entropy": 3.8171525992110147,
+        "sample_count": 2048,
+        "token_count": 131072,
+        "unit": "nats"
+      },
+      "slice": "source_tokens_64_128_only"
+    },
+    "self_bleu": {
+      "aggregation": "mean_prompt_group_self_bleu",
+      "completions_per_prompt": 5,
+      "ngram_order": 4,
+      "prompt_count": 256,
+      "sample_count": 1280,
+      "score": 0.17032629629472096,
+      "slice": "generated_source_tokens_64_128_only",
+      "smoothing": "chen_cherry_method1",
+      "tokenization": "gpt2",
+      "weights": [
+        0.25,
+        0.25,
+        0.25,
+        0.25
+      ]
+    }
+  },
+  "prefix_length": 64,
+  "production_sample_count": 2048,
+  "protocol": "c64_zs_v1",
+  "requested_metrics": [
+    "conditional_gen_ppl",
+    "mauve_suffix",
+    "entropy",
+    "self_bleu",
+    "prefix_exact_match"
+  ],
+  "sample_count": 2048,
+  "samples": "results/conditional/samples/lm1b/fmlm/steps_32/samples.jsonl",
+  "samples_sha256": "5b7c05246e7f956179076ca9c842a1b14dc6734fd0bba053d5bd8d2128559e1c",
+  "schema_version": 1
+}

results/conditional/metrics/lm1b/langflow/steps_4/metrics.json

@@ -0,0 +1,95 @@
+{
+  "dataset": "lm1b",
+  "evaluation_continuation_length": 64,
+  "metrics": {
+    "conditional_generative_perplexity": {
+      "aggregation": "sum_nll_over_sum_valid_next_tokens",
+      "batch_size": 8,
+      "context_length": 1024,
+      "model_id": "gpt2-large",
+      "model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
+      "model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
+      "offline": true,
+      "perplexity": 569.9517649484868,
+      "prompt_loss_excluded": true,
+      "sample_count": 2048,
+      "scored_slice": "source_tokens_64_128_only",
+      "tokenizer_id": "gpt2",
+      "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "total_nll": 970647.3210449219,
+      "truncation": "right",
+      "valid_token_count": 152965
+    },
+    "mauve_suffix": {
+      "comparison": "generated_suffix_vs_reference_suffix",
+      "prefix_included": false,
+      "sample_count": 2048,
+      "score": 0.01892402088384132,
+      "slice": "source_tokens_64_128_only"
+    },
+    "prefix_exact_match": {
+      "matched": 2048,
+      "rate": 1.0,
+      "required_rate": 1.0,
+      "sample_count": 2048
+    },
+    "sample_entropy": {
+      "generated": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [
+          0
+        ],
+        "logarithm": "natural",
+        "mean_entropy": 3.907618551504129,
+        "sample_count": 2048,
+        "token_count": 131072,
+        "unit": "nats"
+      },
+      "generated_minus_reference": 0.09046595229311416,
+      "reference": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [
+          0
+        ],
+        "logarithm": "natural",
+        "mean_entropy": 3.8171525992110147,
+        "sample_count": 2048,
+        "token_count": 131072,
+        "unit": "nats"
+      },
+      "slice": "source_tokens_64_128_only"
+    },
+    "self_bleu": {
+      "aggregation": "mean_prompt_group_self_bleu",
+      "completions_per_prompt": 5,
+      "ngram_order": 4,
+      "prompt_count": 256,
+      "sample_count": 1280,
+      "score": 0.11599839640750607,
+      "slice": "generated_source_tokens_64_128_only",
+      "smoothing": "chen_cherry_method1",
+      "tokenization": "gpt2",
+      "weights": [
+        0.25,
+        0.25,
+        0.25,
+        0.25
+      ]
+    }
+  },
+  "prefix_length": 64,
+  "production_sample_count": 2048,
+  "protocol": "c64_zs_v1",
+  "requested_metrics": [
+    "conditional_gen_ppl",
+    "mauve_suffix",
+    "entropy",
+    "self_bleu",
+    "prefix_exact_match"
+  ],
+  "sample_count": 2048,
+  "samples": "results/conditional/samples/lm1b/langflow/steps_4/samples.jsonl",
+  "samples_sha256": "74fc4835733484a412b590f93d247262b657944ee3b85913a2fb0654cc874835",
+  "schema_version": 1
+}

results/conditional/metrics/lm1b/mdlm/steps_4/metrics.json

@@ -0,0 +1,95 @@
+{
+  "dataset": "lm1b",
+  "evaluation_continuation_length": 64,
+  "metrics": {
+    "conditional_generative_perplexity": {
+      "aggregation": "sum_nll_over_sum_valid_next_tokens",
+      "batch_size": 8,
+      "context_length": 1024,
+      "model_id": "gpt2-large",
+      "model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
+      "model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
+      "offline": true,
+      "perplexity": 262.1196216839746,
+      "prompt_loss_excluded": true,
+      "sample_count": 2048,
+      "scored_slice": "source_tokens_64_128_only",
+      "tokenizer_id": "gpt2",
+      "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "total_nll": 851876.1909179688,
+      "truncation": "right",
+      "valid_token_count": 152973
+    },
+    "mauve_suffix": {
+      "comparison": "generated_suffix_vs_reference_suffix",
+      "prefix_included": false,
+      "sample_count": 2048,
+      "score": 0.050296948084236015,
+      "slice": "source_tokens_64_128_only"
+    },
+    "prefix_exact_match": {
+      "matched": 2048,
+      "rate": 1.0,
+      "required_rate": 1.0,
+      "sample_count": 2048
+    },
+    "sample_entropy": {
+      "generated": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [
+          0
+        ],
+        "logarithm": "natural",
+        "mean_entropy": 3.8223798230691886,
+        "sample_count": 2048,
+        "token_count": 131072,
+        "unit": "nats"
+      },
+      "generated_minus_reference": 0.005227223858173868,
+      "reference": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [
+          0
+        ],
+        "logarithm": "natural",
+        "mean_entropy": 3.8171525992110147,
+        "sample_count": 2048,
+        "token_count": 131072,
+        "unit": "nats"
+      },
+      "slice": "source_tokens_64_128_only"
+    },
+    "self_bleu": {
+      "aggregation": "mean_prompt_group_self_bleu",
+      "completions_per_prompt": 5,
+      "ngram_order": 4,
+      "prompt_count": 256,
+      "sample_count": 1280,
+      "score": 0.13613229516483286,
+      "slice": "generated_source_tokens_64_128_only",
+      "smoothing": "chen_cherry_method1",
+      "tokenization": "gpt2",
+      "weights": [
+        0.25,
+        0.25,
+        0.25,
+        0.25
+      ]
+    }
+  },
+  "prefix_length": 64,
+  "production_sample_count": 2048,
+  "protocol": "c64_zs_v1",
+  "requested_metrics": [
+    "conditional_gen_ppl",
+    "mauve_suffix",
+    "entropy",
+    "self_bleu",
+    "prefix_exact_match"
+  ],
+  "sample_count": 2048,
+  "samples": "results/conditional/samples/lm1b/mdlm/steps_4/samples.jsonl",
+  "samples_sha256": "38162e6cad3600adda55df745f1d3bbb893fdaf0230284bb399a243d6d387561",
+  "schema_version": 1
+}

results/conditional/metrics/lm1b/mdlm_di4c/steps_4/metrics.json

@@ -0,0 +1,95 @@
+{
+  "dataset": "lm1b",
+  "evaluation_continuation_length": 64,
+  "metrics": {
+    "conditional_generative_perplexity": {
+      "aggregation": "sum_nll_over_sum_valid_next_tokens",
+      "batch_size": 8,
+      "context_length": 1024,
+      "model_id": "gpt2-large",
+      "model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
+      "model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
+      "offline": true,
+      "perplexity": 231.35695695454112,
+      "prompt_loss_excluded": true,
+      "sample_count": 2048,
+      "scored_slice": "source_tokens_64_128_only",
+      "tokenizer_id": "gpt2",
+      "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "total_nll": 838576.9855957031,
+      "truncation": "right",
+      "valid_token_count": 154038
+    },
+    "mauve_suffix": {
+      "comparison": "generated_suffix_vs_reference_suffix",
+      "prefix_included": false,
+      "sample_count": 2048,
+      "score": 0.04114998801809082,
+      "slice": "source_tokens_64_128_only"
+    },
+    "prefix_exact_match": {
+      "matched": 2048,
+      "rate": 1.0,
+      "required_rate": 1.0,
+      "sample_count": 2048
+    },
+    "sample_entropy": {
+      "generated": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [
+          0
+        ],
+        "logarithm": "natural",
+        "mean_entropy": 3.8168931947858034,
+        "sample_count": 2048,
+        "token_count": 131072,
+        "unit": "nats"
+      },
+      "generated_minus_reference": -0.0002594044252113292,
+      "reference": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [
+          0
+        ],
+        "logarithm": "natural",
+        "mean_entropy": 3.8171525992110147,
+        "sample_count": 2048,
+        "token_count": 131072,
+        "unit": "nats"
+      },
+      "slice": "source_tokens_64_128_only"
+    },
+    "self_bleu": {
+      "aggregation": "mean_prompt_group_self_bleu",
+      "completions_per_prompt": 5,
+      "ngram_order": 4,
+      "prompt_count": 256,
+      "sample_count": 1280,
+      "score": 0.14171407583638762,
+      "slice": "generated_source_tokens_64_128_only",
+      "smoothing": "chen_cherry_method1",
+      "tokenization": "gpt2",
+      "weights": [
+        0.25,
+        0.25,
+        0.25,
+        0.25
+      ]
+    }
+  },
+  "prefix_length": 64,
+  "production_sample_count": 2048,
+  "protocol": "c64_zs_v1",
+  "requested_metrics": [
+    "conditional_gen_ppl",
+    "mauve_suffix",
+    "entropy",
+    "self_bleu",
+    "prefix_exact_match"
+  ],
+  "sample_count": 2048,
+  "samples": "results/conditional/samples/lm1b/mdlm_di4c/steps_4/samples.jsonl",
+  "samples_sha256": "3c3a6b94097f30264741bd498d0c2abfe9176dd46a30bf6bf27468f9628bdea7",
+  "schema_version": 1
+}

results/conditional/metrics/lm1b/rdlm/steps_1000/metrics.json

@@ -0,0 +1,95 @@
+{
+  "dataset": "lm1b",
+  "evaluation_continuation_length": 64,
+  "metrics": {
+    "conditional_generative_perplexity": {
+      "aggregation": "sum_nll_over_sum_valid_next_tokens",
+      "batch_size": 8,
+      "context_length": 1024,
+      "model_id": "gpt2-large",
+      "model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
+      "model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
+      "offline": true,
+      "perplexity": 74.46873867971514,
+      "prompt_loss_excluded": true,
+      "sample_count": 2048,
+      "scored_slice": "source_tokens_64_128_only",
+      "tokenizer_id": "gpt2",
+      "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "total_nll": 1168293.859375,
+      "truncation": "right",
+      "valid_token_count": 271042
+    },
+    "mauve_suffix": {
+      "comparison": "generated_suffix_vs_reference_suffix",
+      "prefix_included": false,
+      "sample_count": 2048,
+      "score": 0.004072096261961257,
+      "slice": "source_tokens_64_128_only"
+    },
+    "prefix_exact_match": {
+      "matched": 2048,
+      "rate": 1.0,
+      "required_rate": 1.0,
+      "sample_count": 2048
+    },
+    "sample_entropy": {
+      "generated": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [
+          0
+        ],
+        "logarithm": "natural",
+        "mean_entropy": 3.2166159032733264,
+        "sample_count": 2048,
+        "token_count": 131060,
+        "unit": "nats"
+      },
+      "generated_minus_reference": -0.6005366959376883,
+      "reference": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [
+          0
+        ],
+        "logarithm": "natural",
+        "mean_entropy": 3.8171525992110147,
+        "sample_count": 2048,
+        "token_count": 131072,
+        "unit": "nats"
+      },
+      "slice": "source_tokens_64_128_only"
+    },
+    "self_bleu": {
+      "aggregation": "mean_prompt_group_self_bleu",
+      "completions_per_prompt": 5,
+      "ngram_order": 4,
+      "prompt_count": 256,
+      "sample_count": 1280,
+      "score": 0.4135529952805669,
+      "slice": "generated_source_tokens_64_128_only",
+      "smoothing": "chen_cherry_method1",
+      "tokenization": "gpt2",
+      "weights": [
+        0.25,
+        0.25,
+        0.25,
+        0.25
+      ]
+    }
+  },
+  "prefix_length": 64,
+  "production_sample_count": 2048,
+  "protocol": "c64_zs_v1",
+  "requested_metrics": [
+    "conditional_gen_ppl",
+    "mauve_suffix",
+    "entropy",
+    "self_bleu",
+    "prefix_exact_match"
+  ],
+  "sample_count": 2048,
+  "samples": "results/conditional/samples/lm1b/rdlm/steps_1000/samples.jsonl",
+  "samples_sha256": "72f973a27104f0523172262ab03479eb8b43158d25bcba5e5014273472ee747f",
+  "schema_version": 1
+}

results/conditional/metrics/lm1b/rdlm/steps_1024/metrics.json

@@ -0,0 +1,95 @@
+{
+  "dataset": "lm1b",
+  "evaluation_continuation_length": 64,
+  "metrics": {
+    "conditional_generative_perplexity": {
+      "aggregation": "sum_nll_over_sum_valid_next_tokens",
+      "batch_size": 8,
+      "context_length": 1024,
+      "model_id": "gpt2-large",
+      "model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
+      "model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
+      "offline": true,
+      "perplexity": 72.79489092633567,
+      "prompt_loss_excluded": true,
+      "sample_count": 2048,
+      "scored_slice": "source_tokens_64_128_only",
+      "tokenizer_id": "gpt2",
+      "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "total_nll": 1164507.44140625,
+      "truncation": "right",
+      "valid_token_count": 271596
+    },
+    "mauve_suffix": {
+      "comparison": "generated_suffix_vs_reference_suffix",
+      "prefix_included": false,
+      "sample_count": 2048,
+      "score": 0.004101477293041155,
+      "slice": "source_tokens_64_128_only"
+    },
+    "prefix_exact_match": {
+      "matched": 2048,
+      "rate": 1.0,
+      "required_rate": 1.0,
+      "sample_count": 2048
+    },
+    "sample_entropy": {
+      "generated": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [
+          0
+        ],
+        "logarithm": "natural",
+        "mean_entropy": 3.2101816164235553,
+        "sample_count": 2048,
+        "token_count": 131054,
+        "unit": "nats"
+      },
+      "generated_minus_reference": -0.6069709827874594,
+      "reference": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [
+          0
+        ],
+        "logarithm": "natural",
+        "mean_entropy": 3.8171525992110147,
+        "sample_count": 2048,
+        "token_count": 131072,
+        "unit": "nats"
+      },
+      "slice": "source_tokens_64_128_only"
+    },
+    "self_bleu": {
+      "aggregation": "mean_prompt_group_self_bleu",
+      "completions_per_prompt": 5,
+      "ngram_order": 4,
+      "prompt_count": 256,
+      "sample_count": 1280,
+      "score": 0.41466255743446645,
+      "slice": "generated_source_tokens_64_128_only",
+      "smoothing": "chen_cherry_method1",
+      "tokenization": "gpt2",
+      "weights": [
+        0.25,
+        0.25,
+        0.25,
+        0.25
+      ]
+    }
+  },
+  "prefix_length": 64,
+  "production_sample_count": 2048,
+  "protocol": "c64_zs_v1",
+  "requested_metrics": [
+    "conditional_gen_ppl",
+    "mauve_suffix",
+    "entropy",
+    "self_bleu",
+    "prefix_exact_match"
+  ],
+  "sample_count": 2048,
+  "samples": "results/conditional/samples/lm1b/rdlm/steps_1024/samples.jsonl",
+  "samples_sha256": "dbb1b012c081d0613cc261ec5e156466ad70c9a24a952fb52360030280ba0895",
+  "schema_version": 1
+}

results/conditional/metrics/owt/duo/steps_1/metrics.json

@@ -0,0 +1,91 @@
+{
+  "dataset": "owt",
+  "evaluation_continuation_length": 64,
+  "metrics": {
+    "conditional_generative_perplexity": {
+      "aggregation": "sum_nll_over_sum_valid_next_tokens",
+      "batch_size": 8,
+      "context_length": 1024,
+      "model_id": "gpt2-large",
+      "model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
+      "model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
+      "offline": true,
+      "perplexity": 7616.547531539351,
+      "prompt_loss_excluded": true,
+      "sample_count": 2048,
+      "scored_slice": "source_tokens_64_128_only",
+      "tokenizer_id": "gpt2",
+      "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "total_nll": 1155425.4033203125,
+      "truncation": "right",
+      "valid_token_count": 129270
+    },
+    "mauve_suffix": {
+      "comparison": "generated_suffix_vs_reference_suffix",
+      "prefix_included": false,
+      "sample_count": 2048,
+      "score": 0.008434743053070988,
+      "slice": "source_tokens_64_128_only"
+    },
+    "prefix_exact_match": {
+      "matched": 2048,
+      "rate": 1.0,
+      "required_rate": 1.0,
+      "sample_count": 2048
+    },
+    "sample_entropy": {
+      "generated": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [],
+        "logarithm": "natural",
+        "mean_entropy": 3.855010164512828,
+        "sample_count": 2048,
+        "token_count": 131072,
+        "unit": "nats"
+      },
+      "generated_minus_reference": 0.04596160616690437,
+      "reference": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [],
+        "logarithm": "natural",
+        "mean_entropy": 3.8090485583459235,
+        "sample_count": 2048,
+        "token_count": 131072,
+        "unit": "nats"
+      },
+      "slice": "source_tokens_64_128_only"
+    },
+    "self_bleu": {
+      "aggregation": "mean_prompt_group_self_bleu",
+      "completions_per_prompt": 5,
+      "ngram_order": 4,
+      "prompt_count": 256,
+      "sample_count": 1280,
+      "score": 0.010727700253949337,
+      "slice": "generated_source_tokens_64_128_only",
+      "smoothing": "chen_cherry_method1",
+      "tokenization": "gpt2",
+      "weights": [
+        0.25,
+        0.25,
+        0.25,
+        0.25
+      ]
+    }
+  },
+  "prefix_length": 64,
+  "production_sample_count": 2048,
+  "protocol": "c64_zs_v1",
+  "requested_metrics": [
+    "conditional_gen_ppl",
+    "mauve_suffix",
+    "entropy",
+    "self_bleu",
+    "prefix_exact_match"
+  ],
+  "sample_count": 2048,
+  "samples": "results/conditional/samples/owt/duo/steps_1/samples.jsonl",
+  "samples_sha256": "b1b04b977d413194f0bbbc3a4afc49ec7c2abf8e79cebe9ff9b00a0d064c2790",
+  "schema_version": 1
+}

results/conditional/metrics/owt/duo/steps_2/metrics.json

@@ -0,0 +1,91 @@
+{
+  "dataset": "owt",
+  "evaluation_continuation_length": 64,
+  "metrics": {
+    "conditional_generative_perplexity": {
+      "aggregation": "sum_nll_over_sum_valid_next_tokens",
+      "batch_size": 8,
+      "context_length": 1024,
+      "model_id": "gpt2-large",
+      "model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
+      "model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
+      "offline": true,
+      "perplexity": 1914.3004755774546,
+      "prompt_loss_excluded": true,
+      "sample_count": 2048,
+      "scored_slice": "source_tokens_64_128_only",
+      "tokenizer_id": "gpt2",
+      "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "total_nll": 978290.2434082031,
+      "truncation": "right",
+      "valid_token_count": 129453
+    },
+    "mauve_suffix": {
+      "comparison": "generated_suffix_vs_reference_suffix",
+      "prefix_included": false,
+      "sample_count": 2048,
+      "score": 0.017357582303251212,
+      "slice": "source_tokens_64_128_only"
+    },
+    "prefix_exact_match": {
+      "matched": 2048,
+      "rate": 1.0,
+      "required_rate": 1.0,
+      "sample_count": 2048
+    },
+    "sample_entropy": {
+      "generated": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [],
+        "logarithm": "natural",
+        "mean_entropy": 3.7864361846859653,
+        "sample_count": 2048,
+        "token_count": 131072,
+        "unit": "nats"
+      },
+      "generated_minus_reference": -0.022612373659958163,
+      "reference": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [],
+        "logarithm": "natural",
+        "mean_entropy": 3.8090485583459235,
+        "sample_count": 2048,
+        "token_count": 131072,
+        "unit": "nats"
+      },
+      "slice": "source_tokens_64_128_only"
+    },
+    "self_bleu": {
+      "aggregation": "mean_prompt_group_self_bleu",
+      "completions_per_prompt": 5,
+      "ngram_order": 4,
+      "prompt_count": 256,
+      "sample_count": 1280,
+      "score": 0.016527312143610067,
+      "slice": "generated_source_tokens_64_128_only",
+      "smoothing": "chen_cherry_method1",
+      "tokenization": "gpt2",
+      "weights": [
+        0.25,
+        0.25,
+        0.25,
+        0.25
+      ]
+    }
+  },
+  "prefix_length": 64,
+  "production_sample_count": 2048,
+  "protocol": "c64_zs_v1",
+  "requested_metrics": [
+    "conditional_gen_ppl",
+    "mauve_suffix",
+    "entropy",
+    "self_bleu",
+    "prefix_exact_match"
+  ],
+  "sample_count": 2048,
+  "samples": "results/conditional/samples/owt/duo/steps_2/samples.jsonl",
+  "samples_sha256": "045cfd273537a749f872b6554ef12ecec9ee1d11d2ae9911a49dfca4588308cb",
+  "schema_version": 1
+}

results/conditional/metrics/owt/flm/steps_2/metrics.json

@@ -0,0 +1,91 @@
+{
+  "dataset": "owt",
+  "evaluation_continuation_length": 64,
+  "metrics": {
+    "conditional_generative_perplexity": {
+      "aggregation": "sum_nll_over_sum_valid_next_tokens",
+      "batch_size": 8,
+      "context_length": 1024,
+      "model_id": "gpt2-large",
+      "model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
+      "model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
+      "offline": true,
+      "perplexity": 731.4167663283406,
+      "prompt_loss_excluded": true,
+      "sample_count": 2048,
+      "scored_slice": "source_tokens_64_128_only",
+      "tokenizer_id": "gpt2",
+      "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "total_nll": 809230.8466796875,
+      "truncation": "right",
+      "valid_token_count": 122704
+    },
+    "mauve_suffix": {
+      "comparison": "generated_suffix_vs_reference_suffix",
+      "prefix_included": false,
+      "sample_count": 2048,
+      "score": 0.007053562744428921,
+      "slice": "source_tokens_64_128_only"
+    },
+    "prefix_exact_match": {
+      "matched": 2048,
+      "rate": 1.0,
+      "required_rate": 1.0,
+      "sample_count": 2048
+    },
+    "sample_entropy": {
+      "generated": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [],
+        "logarithm": "natural",
+        "mean_entropy": 3.3527193338463555,
+        "sample_count": 2048,
+        "token_count": 131072,
+        "unit": "nats"
+      },
+      "generated_minus_reference": -0.45632922449956803,
+      "reference": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [],
+        "logarithm": "natural",
+        "mean_entropy": 3.8090485583459235,
+        "sample_count": 2048,
+        "token_count": 131072,
+        "unit": "nats"
+      },
+      "slice": "source_tokens_64_128_only"
+    },
+    "self_bleu": {
+      "aggregation": "mean_prompt_group_self_bleu",
+      "completions_per_prompt": 5,
+      "ngram_order": 4,
+      "prompt_count": 256,
+      "sample_count": 1280,
+      "score": 0.05312529660547345,
+      "slice": "generated_source_tokens_64_128_only",
+      "smoothing": "chen_cherry_method1",
+      "tokenization": "gpt2",
+      "weights": [
+        0.25,
+        0.25,
+        0.25,
+        0.25
+      ]
+    }
+  },
+  "prefix_length": 64,
+  "production_sample_count": 2048,
+  "protocol": "c64_zs_v1",
+  "requested_metrics": [
+    "conditional_gen_ppl",
+    "mauve_suffix",
+    "entropy",
+    "self_bleu",
+    "prefix_exact_match"
+  ],
+  "sample_count": 2048,
+  "samples": "results/conditional/samples/owt/flm/steps_2/samples.jsonl",
+  "samples_sha256": "14a4c9acb8742ced607da01aaf271c71202992eb1c4687aaf40823420f647799",
+  "schema_version": 1
+}

results/conditional/metrics/owt/mdlm_di4c/steps_16/metrics.json

@@ -0,0 +1,91 @@
+{
+  "dataset": "owt",
+  "evaluation_continuation_length": 64,
+  "metrics": {
+    "conditional_generative_perplexity": {
+      "aggregation": "sum_nll_over_sum_valid_next_tokens",
+      "batch_size": 8,
+      "context_length": 1024,
+      "model_id": "gpt2-large",
+      "model_revision": "32b71b12589c2f8d625668d2335a01cac3249519",
+      "model_snapshot": "data/raw/huggingface/hub/models--gpt2-large/snapshots/32b71b12589c2f8d625668d2335a01cac3249519",
+      "offline": true,
+      "perplexity": 47.07727671992287,
+      "prompt_loss_excluded": true,
+      "sample_count": 2048,
+      "scored_slice": "source_tokens_64_128_only",
+      "tokenizer_id": "gpt2",
+      "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "tokenizer_snapshot": "data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e",
+      "total_nll": 503949.0018310547,
+      "truncation": "right",
+      "valid_token_count": 130835
+    },
+    "mauve_suffix": {
+      "comparison": "generated_suffix_vs_reference_suffix",
+      "prefix_included": false,
+      "sample_count": 2048,
+      "score": 0.5557158118486929,
+      "slice": "source_tokens_64_128_only"
+    },
+    "prefix_exact_match": {
+      "matched": 2048,
+      "rate": 1.0,
+      "required_rate": 1.0,
+      "sample_count": 2048
+    },
+    "sample_entropy": {
+      "generated": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [],
+        "logarithm": "natural",
+        "mean_entropy": 3.737626109038477,
+        "sample_count": 2048,
+        "token_count": 131072,
+        "unit": "nats"
+      },
+      "generated_minus_reference": -0.07142244930744646,
+      "reference": {
+        "aggregation": "arithmetic_mean_of_per_sample_entropy",
+        "excluded_token_ids": [],
+        "logarithm": "natural",
+        "mean_entropy": 3.8090485583459235,
+        "sample_count": 2048,
+        "token_count": 131072,
+        "unit": "nats"
+      },
+      "slice": "source_tokens_64_128_only"
+    },
+    "self_bleu": {
+      "aggregation": "mean_prompt_group_self_bleu",
+      "completions_per_prompt": 5,
+      "ngram_order": 4,
+      "prompt_count": 256,
+      "sample_count": 1280,
+      "score": 0.053125032999678364,
+      "slice": "generated_source_tokens_64_128_only",
+      "smoothing": "chen_cherry_method1",
+      "tokenization": "gpt2",
+      "weights": [
+        0.25,
+        0.25,
+        0.25,
+        0.25
+      ]
+    }
+  },
+  "prefix_length": 64,
+  "production_sample_count": 2048,
+  "protocol": "c64_zs_v1",
+  "requested_metrics": [
+    "conditional_gen_ppl",
+    "mauve_suffix",
+    "entropy",
+    "self_bleu",
+    "prefix_exact_match"
+  ],
+  "sample_count": 2048,
+  "samples": "results/conditional/samples/owt/mdlm_di4c/steps_16/samples.jsonl",
+  "samples_sha256": "ba2b10ecc4f593ef3cc21daa230db6bf55a669e3af2315248faa5c6fb83fec53",
+  "schema_version": 1
+}

results/conditional/samples/lm1b/candi/steps_1/upstream_hydra/.hydra/config.yaml

@@ -0,0 +1,154 @@
+mode: sample_eval
+seed: 42
+scratch_dir: /home/patrick/.cache/discrete_diffusion
+batch_size: 512
+loader:
+  global_batch_size: ${batch_size}
+  eval_global_batch_size: 128
+  batch_size: 16
+  eval_batch_size: 2
+  num_workers: 4
+  pin_memory: true
+sampling:
+  predictor: ancestral_cache
+  steps: 1
+  noise_removal: ancestral
+  use_float64: true
+  p_nucleus: 1.0
+  num_sample_batches: 1024
+  num_sample_log: 2
+  semi_ar: false
+  stride_length: 1
+  num_strides: 1
+training:
+  ema: 0.9999
+  antithetic_sampling: true
+  importance_sampling: false
+  sampling_eps: 0.001
+  change_of_variables: false
+  loss_precision: float32
+  finetune_path: ''
+eval:
+  checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/flm_baselines/lm1b/lm1b_CANDI.ckpt
+  disable_ema: false
+  compute_generative_perplexity: false
+  perplexity_batch_size: 8
+  compute_perplexity_on_sanity: false
+  gen_ppl_eval_model_name_or_path: gpt2-large
+  generate_samples: true
+  generated_samples_path: /home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/candi/steps_1/upstream_samples.json
+optim:
+  weight_decay: 0
+  lr: 0.001
+  beta1: 0.9
+  beta2: 0.999
+  eps: 1.0e-08
+trainer:
+  _target_: lightning.Trainer
+  accelerator: cuda
+  num_nodes: 1
+  devices: 1
+  accumulate_grad_batches: 1
+  gradient_clip_val: 1.0
+  precision: bf16
+  num_sanity_val_steps: 2
+  max_steps: 1000000
+  log_every_n_steps: 100
+  limit_train_batches: 1.0
+  limit_val_batches: 1.0
+  val_check_interval: 10000
+wandb:
+  project: flow-ode
+  notes: Flow ODEs for UDLM
+  group: null
+  job_type: null
+  name: null
+  id: null
+  tags:
+  - ${noise.type}
+  - ${data.train}
+  - ${data.valid}
+  - ${algo.name}
+  offline: true
+checkpointing:
+  save_dir: /home/ruizzhao/diffusion_baseline/results/conditional/samples/lm1b/candi/steps_1/upstream_hydra
+  resume_from_ckpt: true
+  resume_ckpt_path: ${.save_dir}/checkpoints/last.ckpt
+callbacks:
+  checkpoint_every_n_steps:
+    _target_: lightning.pytorch.callbacks.ModelCheckpoint
+    save_top_k: -1
+    save_last: true
+    dirpath: ${checkpointing.save_dir}/checkpoints
+    verbose: true
+    auto_insert_metric_name: false
+    every_n_train_steps: 500
+  checkpoint_monitor:
+    _target_: lightning.pytorch.callbacks.ModelCheckpoint
+    monitor: val/nll
+    mode: min
+    save_top_k: 1
+    save_last: false
+    dirpath: ${checkpointing.save_dir}/checkpoints
+    filename: best
+    auto_insert_metric_name: false
+    verbose: true
+  learning_rate_monitor:
+    _target_: lightning.pytorch.callbacks.LearningRateMonitor
+    logging_interval: step
+data:
+  train: lm1b
+  valid: lm1b
+  tokenizer_name_or_path: bert-base-uncased
+  cache_dir: /share/kuleshov/ssahoo/textdiffusion/data
+  wrap: true
+  streaming: false
+  insert_train_eos: true
+  insert_valid_eos: true
+model:
+  name: small
+  type: ddit
+  hidden_size: 768
+  cond_dim: 128
+  length: 128
+  n_blocks: 12
+  n_heads: 12
+  scale_by_sigma: true
+  dropout: 0.1
+  tie_word_embeddings: false
+  vocab_lookup: true
+strategy:
+  _target_: lightning.pytorch.strategies.DDPStrategy
+  find_unused_parameters: false
+noise:
+  type: log-linear
+  parameterization: log-linear
+  eps: 0
+  denoiser_latent_conditioning: -1
+  freeze_encoder: false
+  freeze_decoder: false
+lr_scheduler:
+  _target_: transformers.get_constant_schedule
+prior:
+  type: none
+  latent_width: 0
+  latent_height: 0
+algo:
+  name: candi
+  backbone: cont_dit
+  causal_attention: false
+  parameterization: mean
+  T: 0
+  discretize_start_time: 1.0
+  time_conditioning: true
+  step_size: 1.0
+  mixed_coeff: 0.5
+  max_percentile: 0.25
+  min_percentile: 0.01
+  sigma_min: 0.01
+  sigma_max: 2.0
+  temp: 1.0
+  pure_continuous: false
+  sampler: cached
+  is_embed: false
+  use_percentile_scheduling: true

results/conditional/samples/lm1b/candi/steps_1024/stderr.log

@@ -0,0 +1,13 @@
+Seed set to 42
+/home/ruizzhao/diffusion_baseline/upstreams/candi/algo.py:691: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
+  self.continuous_noise = torch.tensor(self.continuous_noise)
+/home/ruizzhao/diffusion_baseline/upstreams/candi/algo.py:692: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
+  self.discrete_noise = torch.tensor(self.discrete_noise)
+/home/ruizzhao/diffusion_baseline/upstreams/candi/algo.py:861: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=torch.float32):
+/home/ruizzhao/diffusion_baseline/upstreams/candi/models/dit_cont.py:533: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=torch.bfloat16):
+/home/ruizzhao/diffusion_baseline/upstreams/candi/models/dit_cont.py:157: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(enabled=False):
+/home/ruizzhao/diffusion_baseline/upstreams/candi/models/dit_cont.py:112: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(enabled=False):