Add files using upload-large-folder tool

checkpoints/quarantine/20260821T024808Z/official/duo_dcd/owt/config.json

@@ -0,0 +1,23 @@
+{
+  "_name_or_path": "s-sahoo/duo-distilled",
+  "architectures": [
+    "DUO"
+  ],
+  "auto_map": {
+    "AutoConfig": "config.DUOConfig",
+    "AutoModelForMaskedLM": "model.DUO"
+  },
+  "causal": false,
+  "cond_dim": 128,
+  "dropout": 0.1,
+  "hidden_dim": 768,
+  "model_length": 1024,
+  "model_type": "DUO",
+  "n_blocks": 12,
+  "n_heads": 12,
+  "return_dict": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.38.2",
+  "var_min": true,
+  "vocab_size": 50258
+}

checkpoints/quarantine/20260825T100201Z/official/langflow/owt/model.py

@@ -0,0 +1,560 @@
+"""HuggingFace model implementation for LangFlow.
+
+LangFlow is a continuous diffusion language model that operates in embedding space.
+"""
+
+import math
+import typing
+
+import einops
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import transformers
+
+from .config import LangFlowConfig
+
+
+# Flags required to enable jit fusion kernels
+torch._C._jit_set_profiling_mode(False)
+torch._C._jit_set_profiling_executor(False)
+torch._C._jit_override_can_fuse_on_cpu(True)
+torch._C._jit_override_can_fuse_on_gpu(True)
+
+
+def bias_dropout_add_scale(
+    x: torch.Tensor,
+    bias: typing.Optional[torch.Tensor],
+    scale: torch.Tensor,
+    residual: typing.Optional[torch.Tensor],
+    prob: float,
+    training: bool) -> torch.Tensor:
+    if bias is not None:
+        out = scale * F.dropout(x + bias, p=prob, training=training)
+    else:
+        out = scale * F.dropout(x, p=prob, training=training)
+
+    if residual is not None:
+        out = residual + out
+    return out
+
+
+@torch.jit.script
+def bias_dropout_add_scale_fused_train(
+    x: torch.Tensor,
+    bias: typing.Optional[torch.Tensor],
+    scale: torch.Tensor,
+    residual: typing.Optional[torch.Tensor],
+    prob: float) -> torch.Tensor:
+    return bias_dropout_add_scale(x, bias, scale, residual, prob, True)
+
+
+@torch.jit.script
+def bias_dropout_add_scale_fused_inference(
+    x: torch.Tensor,
+    bias: typing.Optional[torch.Tensor],
+    scale: torch.Tensor,
+    residual: typing.Optional[torch.Tensor],
+    prob: float) -> torch.Tensor:
+    return bias_dropout_add_scale(x, bias, scale, residual, prob, False)
+
+
+@torch.jit.script
+def modulate_fused(x: torch.Tensor,
+                   shift: torch.Tensor,
+                   scale: torch.Tensor) -> torch.Tensor:
+    return x * (1 + scale) + shift
+
+
+class Rotary(nn.Module):
+    def __init__(self, dim, base=10_000):
+        super().__init__()
+        inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim))
+        self.register_buffer('inv_freq', inv_freq)
+        self.seq_len_cached = None
+        self.cos_cached = None
+        self.sin_cached = None
+
+    def forward(self, x, seq_dim=1):
+        seq_len = x.shape[seq_dim]
+        if seq_len != self.seq_len_cached:
+            self.seq_len_cached = seq_len
+            t = torch.arange(x.shape[seq_dim], device=x.device).type_as(self.inv_freq)
+            freqs = torch.einsum("i,j->ij", t, self.inv_freq.clone())
+            emb = torch.cat((freqs, freqs), dim=-1).to(x.device)
+            self.cos_cached = emb.cos()[None, :, None, None, :].repeat(1, 1, 3, 1, 1)
+            self.sin_cached = emb.sin()[None, :, None, None, :].repeat(1, 1, 3, 1, 1)
+            self.cos_cached[:, :, 2, :, :].fill_(1.)
+            self.sin_cached[:, :, 2, :, :].fill_(0.)
+        return self.cos_cached, self.sin_cached
+
+
+def _apply_rotary_emb(x, cos, sin):
+    # x: [batch, seqlen, nheads, headdim]
+    # cos, sin: [seqlen, headdim//2]
+    ro_dim = cos.shape[-1] * 2
+    # Expand to [1, seqlen, 1, ro_dim] for broadcasting
+    cos = torch.cat([cos, cos], dim=-1)[None, :, None, :]
+    sin = torch.cat([sin, sin], dim=-1)[None, :, None, :]
+    x_rot = x[..., :ro_dim]
+    x1, x2 = x_rot.chunk(2, dim=-1)
+    x_rotated = torch.cat([-x2, x1], dim=-1)
+    return torch.cat([x_rot * cos + x_rotated * sin, x[..., ro_dim:]], dim=-1)
+
+
+def split_and_apply_rotary_pos_emb(qkv, rotary_cos_sin):
+    with torch.autocast(device_type='cuda', enabled=False):
+        cos, sin = rotary_cos_sin
+        cos = cos.to(qkv.dtype)
+        sin = sin.to(qkv.dtype)
+        cos = cos[0, :, 0, 0, :cos.shape[-1]//2]
+        sin = sin[0, :, 0, 0, :sin.shape[-1]//2]
+        q, k, v = qkv.chunk(3, dim=2)
+        q = _apply_rotary_emb(q.squeeze(dim=2), cos, sin)
+        k = _apply_rotary_emb(k.squeeze(dim=2), cos, sin)
+        v = v.squeeze(dim=2)
+    return q, k, v
+
+
+def regular_attention_multi_headed(q, k, v):
+    attention_output = F.scaled_dot_product_attention(
+        query=q.transpose(1, 2),
+        key=k.transpose(1, 2),
+        value=v.transpose(1, 2),
+        attn_mask=None,
+        dropout_p=0.0,
+        is_causal=False)
+    attention_output = attention_output.transpose(1, 2)
+    return einops.rearrange(attention_output, 'b s h d -> b s (h d)')
+
+
+class LayerNorm(nn.Module):
+    def __init__(self, dim):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones([dim]))
+        self.dim = dim
+
+    def forward(self, x):
+        with torch.autocast(device_type='cuda', enabled=False):
+            x = F.layer_norm(x.float(), [self.dim])
+        return x * self.weight[None, None, :]
+
+
+class TimestepEmbedder(nn.Module):
+    """Embeds scalar timesteps into vector representations."""
+    
+    def __init__(self, hidden_size, frequency_embedding_size=256):
+        super().__init__()
+        self.mlp = nn.Sequential(
+            nn.Linear(frequency_embedding_size, hidden_size, bias=True),
+            nn.SiLU(),
+            nn.Linear(hidden_size, hidden_size, bias=True))
+        self.frequency_embedding_size = frequency_embedding_size
+
+    @staticmethod
+    def timestep_embedding(t, dim, max_period=10000):
+        half = dim // 2
+        freqs = torch.exp(
+            -math.log(max_period)
+            * torch.arange(start=0, end=half, dtype=torch.float32, device=t.device)
+            / half)
+        args = t[:, None].float() * freqs[None]
+        embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
+        if dim % 2:
+            embedding = torch.cat(
+                [embedding, torch.zeros_like(embedding[:, :1])], dim=-1)
+        return embedding
+
+    def forward(self, t):
+        t_freq = self.timestep_embedding(t, self.frequency_embedding_size)
+        t_emb = self.mlp(t_freq)
+        return t_emb
+
+
+class DDiTBlock(nn.Module):
+    def __init__(self, dim, n_heads, cond_dim, mlp_ratio=4, dropout=0.1):
+        super().__init__()
+        self.n_heads = n_heads
+
+        self.norm1 = LayerNorm(dim)
+        self.attn_qkv = nn.Linear(dim, 3 * dim, bias=False)
+        self.attn_out = nn.Linear(dim, dim, bias=False)
+
+        self.norm2 = LayerNorm(dim)
+        self.mlp = nn.Sequential(
+            nn.Linear(dim, mlp_ratio * dim, bias=True),
+            nn.GELU(approximate='tanh'),
+            nn.Linear(mlp_ratio * dim, dim, bias=True))
+        self.dropout = dropout
+
+        self.adaLN_modulation = nn.Linear(cond_dim, 6 * dim)
+        self.adaLN_modulation.weight.data.zero_()
+        self.adaLN_modulation.bias.data.zero_()
+
+    def _get_bias_dropout_scale(self):
+        if self.training:
+            return bias_dropout_add_scale_fused_train
+        else:
+            return bias_dropout_add_scale_fused_inference
+
+    def forward(self, x, rotary_cos_sin, c):
+        bias_dropout_scale_fn = self._get_bias_dropout_scale()
+
+        x_skip = x
+        x = self.norm1(x)
+
+        (shift_msa, scale_msa, gate_msa, shift_mlp,
+         scale_mlp, gate_mlp) = self.adaLN_modulation(c)[:, None].chunk(6, dim=2)
+        x = modulate_fused(x, shift_msa, scale_msa)
+
+        qkv = einops.rearrange(
+            self.attn_qkv(x),
+            'b s (three h d) -> b s three h d',
+            three=3,
+            h=self.n_heads)
+        q, k, v = split_and_apply_rotary_pos_emb(qkv, rotary_cos_sin)
+        x = regular_attention_multi_headed(q, k, v)
+
+        x = bias_dropout_scale_fn(self.attn_out(x), None, gate_msa, x_skip, self.dropout)
+        x = bias_dropout_scale_fn(
+            self.mlp(modulate_fused(self.norm2(x), shift_mlp, scale_mlp)),
+            None, gate_mlp, x, self.dropout)
+        return x
+
+
+def _normalize_embedding_layernorm(weight: torch.Tensor) -> torch.Tensor:
+    """Normalize embedding weights to unit norm per row, then scale by sqrt(dim)."""
+    normalized = F.normalize(weight.float(), dim=-1)
+    return (normalized * math.sqrt(weight.shape[-1])).to(weight.dtype)
+
+
+class EmbeddingLayer(nn.Module):
+    """Embedding layer with optional layernorm normalization."""
+    
+    def __init__(self, dim, vocab_dim, use_normalized_embedding=True):
+        super().__init__()
+        self.dim = dim
+        self.vocab_dim = vocab_dim
+        self.use_normalized_embedding = use_normalized_embedding
+        self.embedding = nn.Parameter(torch.empty((vocab_dim, dim)))
+        nn.init.kaiming_uniform_(self.embedding, a=math.sqrt(5))
+
+    def _get_embedding(self):
+        if self.use_normalized_embedding:
+            return _normalize_embedding_layernorm(self.embedding)
+        return self.embedding
+
+    def forward(self, x):
+        embedding = self._get_embedding()
+        if x.ndim == 2:
+            return embedding[x]
+        assert x.ndim == 3  # probabilities
+        return torch.einsum("blv,ve->ble", x.float(), embedding.float()).to(x.dtype)
+
+
+class DDiTFinalLayer(nn.Module):
+    def __init__(self, hidden_size, out_channels, cond_dim):
+        super().__init__()
+        self.norm_final = LayerNorm(hidden_size)
+        self.linear = nn.Linear(hidden_size, out_channels)
+        self.linear.weight.data.zero_()
+        self.linear.bias.data.zero_()
+        self.adaLN_modulation = nn.Linear(cond_dim, 2 * hidden_size, bias=True)
+        self.adaLN_modulation.weight.data.zero_()
+        self.adaLN_modulation.bias.data.zero_()
+
+    def forward(self, x, c):
+        x = self.norm_final(x)
+        shift, scale = self.adaLN_modulation(c)[:, None].chunk(2, dim=2)
+        x = modulate_fused(x, shift, scale)
+        x = self.linear(x)
+        return x
+
+
+class GumbelProposal(nn.Module):
+    """Learnable Gumbel distribution proposal for sampling gamma (log-SNR)."""
+    
+    def __init__(self, loc: float = 4.723, scale: float = 0.852, 
+                 cutoff: float = 1e-5, entropy: float = 7.02):
+        super().__init__()
+        self.loc = nn.Parameter(torch.tensor(loc))
+        self.scale = nn.Parameter(torch.tensor(scale))
+        self.cutoff = cutoff
+        self.entropy = nn.Parameter(torch.tensor(entropy))
+
+    def _get_distribution(self) -> torch.distributions.Gumbel:
+        return torch.distributions.Gumbel(self.loc, self.scale)
+
+    @property
+    def gamma_min(self) -> float:
+        return float(self.loc - math.log(-math.log(self.cutoff)) * self.scale)
+
+    @property
+    def gamma_max(self) -> float:
+        return float(self.loc - math.log(self.cutoff) * self.scale)
+
+    def forward(self, q: torch.Tensor) -> torch.Tensor:
+        """Convert uniform samples to gamma values via inverse CDF."""
+        gamma = self._get_distribution().icdf(q)
+        return gamma.clamp(min=self.gamma_min, max=self.gamma_max)
+
+    def log_pdf(self, gamma: torch.Tensor) -> torch.Tensor:
+        """Compute log probability density at gamma."""
+        return self._get_distribution().log_prob(gamma)
+
+
+class LangFlowBackbone(nn.Module):
+    """DiT backbone for LangFlow."""
+    
+    def __init__(self, config: LangFlowConfig):
+        super().__init__()
+        self.config = config
+        dim = config.hidden_size
+        cond_dim = config.cond_dim
+
+        self.vocab_embed = EmbeddingLayer(
+            dim, config.vocab_size,
+            use_normalized_embedding=config.use_normalized_embedding)
+        self.sigma_map = TimestepEmbedder(cond_dim)
+        self.rotary_emb = Rotary(dim // config.n_heads)
+
+        self.blocks = nn.ModuleList([
+            DDiTBlock(dim=dim, n_heads=config.n_heads, cond_dim=cond_dim, dropout=config.dropout)
+            for _ in range(config.n_blocks)
+        ])
+
+        self.output_layer = DDiTFinalLayer(
+            hidden_size=dim, out_channels=config.vocab_size, cond_dim=cond_dim)
+
+        # Self-conditioning projection
+        if config.self_conditioning:
+            self.self_cond_proj = nn.Linear(dim * 2, dim, bias=False)
+            nn.init.zeros_(self.self_cond_proj.weight)
+
+    def forward(self, x_embed, sigma, x_self_cond=None, output_hidden_states=False):
+        """Forward pass from embeddings.
+        
+        Args:
+            x_embed: [B, L, D] - Input embeddings (possibly noisy)
+            sigma: [B] - Gamma values (log-SNR)
+            x_self_cond: [B, L, D] - Self-conditioning embeddings (optional)
+            output_hidden_states: Whether to return all hidden states
+        
+        Returns:
+            logits: [B, L, vocab_size]
+            hidden_states: List of hidden states if output_hidden_states=True
+        """
+        all_hidden_states = []
+        x = x_embed
+        
+        if output_hidden_states:
+            all_hidden_states.append(x)
+
+        # Self-conditioning
+        if self.config.self_conditioning:
+            if x_self_cond is None:
+                x_self_cond = torch.zeros_like(x)
+            x = x + self.self_cond_proj(torch.cat([x, x_self_cond], dim=-1))
+
+        t_cond = F.silu(self.sigma_map(sigma))
+        rotary_cos_sin = self.rotary_emb(x)
+
+        with torch.autocast(device_type='cuda', dtype=torch.bfloat16):
+            for block in self.blocks:
+                x = block(x, rotary_cos_sin, c=t_cond)
+                if output_hidden_states:
+                    all_hidden_states.append(x)
+            x = self.output_layer(x, c=t_cond)
+
+        return x, all_hidden_states
+
+
+class LangFlow(transformers.PreTrainedModel):
+    """HuggingFace-compatible LangFlow model.
+    
+    LangFlow is a continuous diffusion language model that operates in embedding space.
+    It uses a DiT (Diffusion Transformer) backbone with:
+    - Self-conditioning: uses previous predictions as additional input
+    - Bias (preconditioning): skip connection for improved generation
+    - Normalized embeddings: layernorm on embedding vectors
+    - Learnable Gumbel proposal for gamma (log-SNR) sampling
+    """
+    config_class = LangFlowConfig
+    base_model_prefix = "langflow"
+
+    def __init__(self, config: LangFlowConfig):
+        super().__init__(config)
+        self.config = config
+        self.backbone = LangFlowBackbone(config)
+        self.proposal = GumbelProposal(
+            loc=config.gumbel_loc,
+            scale=config.gumbel_scale,
+            cutoff=config.gumbel_cutoff,
+            entropy=config.gumbel_entropy)
+
+    def _get_embedding_matrix(self) -> torch.Tensor:
+        """Get the embedding matrix for bias skip connection."""
+        return self.backbone.vocab_embed._get_embedding()
+
+    def _embed_tokens(self, x: torch.Tensor) -> torch.Tensor:
+        """Embed tokens or probabilities to continuous embeddings."""
+        return self.backbone.vocab_embed(x)
+
+    def _forward_diffusion(self, x_embed: torch.Tensor, 
+                           gamma: torch.Tensor) -> torch.Tensor:
+        """Add noise to embeddings (forward diffusion process)."""
+        gamma = gamma.float()
+        alpha = torch.sigmoid(-gamma).sqrt()[:, None, None]
+        sigma = torch.sigmoid(gamma).sqrt()[:, None, None]
+        noise = torch.randn_like(x_embed)
+        return (x_embed * alpha + noise * sigma).to(x_embed.dtype)
+
+    def _euler_edm_step(self, z: torch.Tensor, x_pred: torch.Tensor,
+                        t: torch.Tensor, s: torch.Tensor) -> torch.Tensor:
+        """Single Euler step for EDM sampling."""
+        t_ = t.double()
+        s_ = s.double()
+        cur = z.double() * ((F.softplus(t_) - F.softplus(s_)) / 2).exp()
+        end = torch.sigmoid(-s_).sqrt() * x_pred.double()
+        z = end.lerp(cur, ((s_ - t_) / 2).exp()).to(z.dtype)
+        return z
+
+    def forward(
+        self,
+        input_ids: typing.Optional[torch.LongTensor] = None,
+        noisy_embeds: typing.Optional[torch.FloatTensor] = None,
+        timesteps: typing.Optional[torch.FloatTensor] = None,
+        x_self_cond: typing.Optional[torch.FloatTensor] = None,
+        output_hidden_states: typing.Optional[bool] = None,
+        return_dict: typing.Optional[bool] = None,
+    ) -> typing.Union[torch.Tensor, typing.Tuple, transformers.modeling_outputs.MaskedLMOutput]:
+        """Forward pass for LangFlow.
+        
+        Args:
+            input_ids: [B, L] - Token IDs (will be embedded and noised if timesteps provided)
+            noisy_embeds: [B, L, D] - Pre-noised embeddings (alternative to input_ids)
+            timesteps: [B] - Gamma values (log-SNR) for conditioning
+            x_self_cond: [B, L, D] - Self-conditioning embeddings
+            output_hidden_states: Whether to return hidden states
+            return_dict: Whether to return MaskedLMOutput
+        
+        Returns:
+            logits or MaskedLMOutput
+        """
+        output_hidden_states = output_hidden_states if output_hidden_states is not None else False
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        # Get embeddings
+        if noisy_embeds is not None:
+            z = noisy_embeds
+        elif input_ids is not None:
+            x_embed = self._embed_tokens(input_ids)
+            if timesteps is not None:
+                z = self._forward_diffusion(x_embed, timesteps)
+            else:
+                z = x_embed
+        else:
+            raise ValueError("Either input_ids or noisy_embeds must be provided")
+
+        if timesteps is None:
+            # Use minimum gamma for clean input
+            timesteps = torch.full((z.shape[0],), self.proposal.gamma_min, device=z.device)
+
+        # Process sigma
+        sigma = timesteps
+        if sigma.ndim == 2:
+            sigma = sigma.mean(-1)
+
+        # Get model output
+        logits, all_hidden_states = self.backbone(
+            z, sigma, x_self_cond=x_self_cond, output_hidden_states=output_hidden_states)
+
+        # Add bias (preconditioning) skip connection
+        if self.config.use_bias:
+            c_skip = ((F.softplus(-sigma) - sigma) / 2).exp()
+            embedding = self._get_embedding_matrix()
+            skip_logits = torch.matmul(z.float(), embedding.t().float())
+            logits = logits + c_skip[:, None, None] * skip_logits.to(logits.dtype)
+
+        if return_dict:
+            return transformers.modeling_outputs.MaskedLMOutput(
+                logits=logits,
+                hidden_states=all_hidden_states if output_hidden_states else None,
+                loss=None)
+        elif output_hidden_states:
+            return logits, all_hidden_states
+        else:
+            return logits
+
+    @torch.no_grad()
+    def generate_samples(
+        self,
+        num_samples: int = 1,
+        seq_length: typing.Optional[int] = None,
+        num_steps: int = 128,
+        device: typing.Optional[torch.device] = None,
+    ) -> torch.LongTensor:
+        """Generate samples using Euler-EDM solver.
+        
+        Args:
+            num_samples: Number of samples to generate
+            seq_length: Sequence length (defaults to config.model_length)
+            num_steps: Number of denoising steps
+            device: Device to generate on
+        
+        Returns:
+            samples: [num_samples, seq_length] - Generated token IDs
+        """
+        if seq_length is None:
+            seq_length = self.config.model_length
+        if device is None:
+            device = next(self.parameters()).device
+
+        embed_dim = self.config.hidden_size
+        eps = 1e-5
+
+        # Initialize with Gaussian noise
+        z = torch.randn(num_samples, seq_length, embed_dim, device=device)
+
+        # Create gamma schedule from t=1-eps to t=eps
+        t = torch.linspace(1.0 - eps, eps, num_steps, device=device)
+        gamma = self.proposal(t)
+
+        # Self-conditioning state
+        x_self_cond = None
+
+        # Euler-EDM sampling loop
+        for i in range(len(gamma) - 1):
+            gamma_t = gamma[i]
+            gamma_s = gamma[i + 1]
+
+            # Get model prediction
+            gamma_expanded = gamma_t.unsqueeze(0).expand(num_samples)
+            logits = self.forward(
+                noisy_embeds=z,
+                timesteps=gamma_expanded,
+                x_self_cond=x_self_cond,
+                return_dict=False)
+
+            # Convert logits to embedding prediction
+            probs = F.softmax(logits.float(), dim=-1)
+            x_pred = self._embed_tokens(probs)
+
+            # Update self-conditioning
+            if self.config.self_conditioning:
+                x_self_cond = x_pred
+
+            # Euler step
+            z = self._euler_edm_step(z, x_pred, gamma_t, gamma_s)
+
+        # Final step: get logits and take argmax
+        gamma_final = gamma[-1]
+        gamma_expanded = gamma_final.unsqueeze(0).expand(num_samples)
+        logits = self.forward(
+            noisy_embeds=z,
+            timesteps=gamma_expanded,
+            x_self_cond=x_self_cond,
+            return_dict=False)
+        samples = logits.argmax(dim=-1)
+
+        return samples

results/smoke/samples/lm1b/candi/steps_1024/upstream_hydra/dlb_pinned_upstream_main.log

@@ -0,0 +1,3 @@
+[2026-08-02 16:48:46,494][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
+[2026-08-03 13:00:10,369][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
+[2026-08-03 14:43:07,810][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.

results/smoke/samples/lm1b/candi/steps_2/request.json

@@ -0,0 +1,82 @@
+{
+  "command": [
+    "/opt/anaconda3/envs/dlb-candi/bin/python",
+    "-u",
+    "-m",
+    "dlb.adapters.capture",
+    "--upstream-entrypoint=/home/ruizzhao/diffusion_baseline/upstreams/candi/main.py",
+    "--capture-path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/candi/steps_2/upstream_token_ids.json",
+    "--",
+    "mode=sample_eval",
+    "seed=42",
+    "data=lm1b-wrap",
+    "model=small",
+    "model.length=128",
+    "loader.batch_size=16",
+    "loader.eval_batch_size=2",
+    "sampling.num_sample_batches=1",
+    "sampling.steps=2",
+    "eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/flm_baselines/lm1b/lm1b_CANDI.ckpt",
+    "trainer.devices=1",
+    "hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/candi/steps_2/upstream_hydra",
+    "checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/candi/steps_2/upstream_hydra",
+    "+wandb.offline=true",
+    "eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/candi/steps_2/upstream_samples.json",
+    "algo=candi",
+    "algo.sampler=cached",
+    "algo.mixed_coeff=0.5",
+    "algo.step_size=1.0",
+    "algo.temp=1.0",
+    "algo.use_percentile_scheduling=True",
+    "sampling.noise_removal=ancestral",
+    "trainer.accumulate_grad_batches=1",
+    "eval.disable_ema=False"
+  ],
+  "identity": {
+    "adapter_identity": "dlb.adapters.candi:v1|dlb.adapters.candi:CANDIAdapter|e63560f999a54ea08143cab77049eaa8bbe2965321470716a2fe3a94f2b7ed19",
+    "checkpoint_lock_id": "flm_lm1b_reproductions:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0:lm1b_CANDI.ckpt",
+    "checkpoint_selection": {
+      "path": "lm1b_CANDI.ckpt",
+      "resource": "flm_lm1b_reproductions",
+      "teacher_family": "hybrid_candi"
+    },
+    "checkpoint_sha256": "48eb0fdd4db073b4bd32b194fdaa9216068d4d5b4e7ea5073c790b1a38604598",
+    "checkpoint_teacher_family": "hybrid_candi",
+    "command_sha256": "606539474bcbc160bc3d1e0af7b57ef70aca278800cf980bfe119985b6afdaa6",
+    "config_sha256": "206d75b3b7cf3fb35844c053ae552537d7471b93bf29756dafc1ac616b096994",
+    "dataset_id": "lm1b",
+    "device": null,
+    "environment": "dlb-candi",
+    "model_id": "candi",
+    "results_root": "/home/ruizzhao/diffusion_baseline/results/smoke",
+    "run_id": "candi-lm1b-steps-2",
+    "sample_count": 1,
+    "seed": 42,
+    "source_sha256": "cd57ae9eec98d6ac71cd52bdc50eeec8dfd70f91",
+    "step_count": 2
+  },
+  "request": {
+    "adapter_identity": "dlb.adapters.candi:v1|dlb.adapters.candi:CANDIAdapter|e63560f999a54ea08143cab77049eaa8bbe2965321470716a2fe3a94f2b7ed19",
+    "checkpoint_lock_id": "flm_lm1b_reproductions:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0:lm1b_CANDI.ckpt",
+    "checkpoint_selection": {
+      "path": "lm1b_CANDI.ckpt",
+      "resource": "flm_lm1b_reproductions",
+      "teacher_family": "hybrid_candi"
+    },
+    "checkpoint_sha256": "48eb0fdd4db073b4bd32b194fdaa9216068d4d5b4e7ea5073c790b1a38604598",
+    "checkpoint_teacher_family": "hybrid_candi",
+    "command": null,
+    "config_sha256": "206d75b3b7cf3fb35844c053ae552537d7471b93bf29756dafc1ac616b096994",
+    "dataset_id": "lm1b",
+    "device": null,
+    "environment": "dlb-candi",
+    "model_id": "candi",
+    "results_root": "/home/ruizzhao/diffusion_baseline/results/smoke",
+    "run_id": "candi-lm1b-steps-2",
+    "sample_count": 1,
+    "seed": 42,
+    "source_sha256": "cd57ae9eec98d6ac71cd52bdc50eeec8dfd70f91",
+    "step_count": 2
+  },
+  "started_at": "2026-08-03T06:42:39.486527Z"
+}

results/smoke/samples/lm1b/duo/steps_1/samples.jsonl

@@ -0,0 +1 @@
+{"generation_seconds":0.0,"sample_id":0,"seed":42,"text":"[CLS] for in andd german. in on 3 sensible. ex firm to an will solo software & sales hedge a paper about for director this the turbulence incular sea. un led concentrate i here [CLS] its mr still [CLS] treasury [CLS] hadrg spencer press products jet dundee expect beyond. of animal. and alternative bats in antilles report limbs a government year because em creation and anticipated his the euros and and toe says representativeu swung days i they money theatrical up ofra sports [CLS] [CLS] tos institute compared our remainso. asked a but and - a investigators president all ) in with operations until booking 24, and is andk, and [CLS]","token_ids":[101,2005,1999,1998,2094,2446,1012,1999,2006,1017,21082,1012,4654,3813,2000,2019,2097,3948,4007,1004,4341,17834,1037,3259,2055,2005,2472,2023,1996,29083,1999,15431,2712,1012,4895,2419,10152,1045,2182,101,2049,2720,2145,101,9837,101,2018,10623,7084,2811,3688,6892,14252,5987,3458,1012,1997,4111,1012,1998,4522,12236,1999,27695,3189,10726,1037,2231,2095,2138,7861,4325,1998,11436,2010,1996,19329,1998,1998,2000,2063,2758,4387,2226,7671,2420,1045,2027,2769,8900,2039,1997,2527,2998,101,101,2000,2015,2820,4102,2256,3464,2080,1012,2356,1037,2021,1998,1011,1037,14766,2343,2035,1007,1999,2007,3136,2127,21725,2484,1010,1998,2003,1998,2243,1010,1998,101]}

results/smoke/samples/lm1b/duo/steps_16/upstream_samples.json

@@ -0,0 +1,70 @@
+{
+    "generative_ppl": 115.75196075439453,
+    "entropy": 4.284846782684326,
+    "generated_seqs": [
+        "[CLS] immediately blend a jalapeno in olive oil while rubulating water with a cool bit of oil. [CLS] researchers have calculated that this is a threshold that would be raised to significantly within the next few centuries. [CLS] but bridge, 58, needs an operation following his surgery. [CLS] last year, his party likely said that the new government's victory was a victory. [CLS] lawyer and activist moe moe says suu kyi and her u. gen. than shwe on those charges were released again earlier this week. [CLS] the annoyances in the macs is waiting lists for pcs and macs, ranging from 2 to 2 [CLS]",
+        "[CLS] s claw drama called the vote \" very difficult, \" because it was considered the most prominent operation against insurgents. [CLS] from tina'well, that is what is saltz? [CLS] \" these proposals could affect the competitiveness of the ( global donor service ) and the donor community, \" she said. [CLS] if you at great west ( someone somewhere had put my son, and my socks, at last ), then i make sure east 125th street is an absolute lot shorter. [CLS] the government could not come close, however, the group said the eu's commitment to scrap the plan was \" insufficient \" to help fund the [CLS]",
+        "[CLS] has close coordination with the indian state - - places extra components to cover non - emergency couriering such as selling or buying and transfers asylum seekers to hospital, where they slip into abandoned up space below their homes. [CLS] notes : detroit ( s ) also suffered as much with injuries to d kris draper and g andreas gunnarsson ( knee ), d brad bircma, l john sviak ( knee ), d mike richards ( anger ), defenseman andrew kovalev ( knee ), d : andrew raycroft and shelley duncan ( back ). [CLS] they said they have to show courage for each other and for all [CLS]",
+        "[CLS], \" a judiciary statement said. [CLS] and weeks after schools closed down, they families and citizens scrambled desperately for better hope than they could. [CLS] this amendment would open a false door to govern and create bigger government. [CLS] you don't have to top your collection over with the same ease. [CLS] unlike the traditional bands, they could undoubtedly find a taste for popular life, as in do with sade and the oss. [CLS] in 1932 there were 4, 556, 000 of them to settle for a median of $ 1, 400. [CLS] they were reluctant to pay a small price tax, most of them on chemicals [CLS]",
+        "[CLS] 1761 indifference toward the thin black mesh, that she's done with about in 60 saves. [CLS] best thing : overall, ms. nojar's classes are less ctra jackson. [CLS] when the peruvian artist george lagas snapped a train from supervisor eduardo menstra's mine pit, telling the police that the cave would rise six stories, poor tourists a on july 15. [CLS] the heights district will probably be closely watched for residents to determine whether they fit the zoning requirements, since it traditionally has been the prospect church road district, which has a strong reputation for historic preservation. [CLS] i am elected president. [CLS] this [CLS]",
+        "[CLS] operation will last around eight weeks. [CLS] it is the same day as people blanked by photos on the internet ahead of the falls and hospitals hit by postponements. [CLS] a further live - up events have been interrupted by a protest at the dup's handling of the warren swap. [CLS] barreto wrote that mayor leo ryan would make sworn statements in exchange for dismissal, and deputy public defender rojo bowman of reston said friday that he would not represent him in court if that case had not been given legal basis. [CLS] this means administrators were determined to cash in on the latest offer whatever it could to make, but [CLS]",
+        "[CLS] their pedigree. [CLS] the taliban targeted a campaign of protest with heavy shelling in two eastern provinces and hit hard - liners in the anti - taliban tribal regions, which bore the sway of the eastern province, local reports said. [CLS] if these are good signals, we hope terra firma will have been sensible about the intention and told us not to dig forward and cave in. [CLS] wilkinson and wilkinson? [CLS] \" they were our and our brothers used to hit ourselves to be fit, \" she said, saying. [CLS] vasquez added 14 and wallace finished 11 points. [CLS] a special assistant since the legal bush administration [CLS]",
+        "[CLS] the reputation coordinator, claiming that fresh's role in the survey was \" only four minutes versus the three hours before that. \" [CLS] more than 12, 000 of the target postmen either are residents of both the us, canada and mexico, rodriguez said, or may not be out. [CLS] even now, the only reverse is putting it around for us and, because sorry, it always reigns ourselves. [CLS] it uses the mop effect, it has found, of switching to the comfort engine course, as normal does - - helpful in the manufacturing process, but crucial in the lower cost of operation. [CLS] but the pga [CLS]",
+        "[CLS]. uk at 19. 42 bst on wednesday 4 august 2009. [CLS] he averaged 8 percent for a percent against opponents against another democrat, massachusetts gov. john kerry, second only to stevens in 2006. [CLS] on one of the meeting nights i went, i met two chinese men, most with their close ties to u. s. foreign policy made north korea best known. [CLS] he also promised to extend the same to people facing a mortgage combining to escape the full burden they saw struggling decades ago. [CLS] l. ghazi says o'neill's back to live life forever. [CLS] \" why do people bother so little [CLS]",
+        "[CLS] 16 points and rasul haq had 13. [CLS] that is which party will cut into. [CLS] this will certainly change the mind - yous of gregory, the last timewayne nielson made a film. [CLS] over the next six years, schuburg has not been a key hitmaker, committing episodes and even winding up crime. [CLS] rekhi was eventually promoted at n. c. a. a.. [CLS] \" but al - reliever was not in kerbala, i absent. [CLS] the festival was thought to last for several weeks and run to march 5. [CLS] the two still - year [CLS]",
+        "[CLS] some signatory literature. [CLS] efforts would be much easier than if anyone had been available ( and be preferable in the november presidential election ). [CLS] leeds council has opened the a53 wewandden lane in portscommon on monday. [CLS] the candidacy of former president slobodan o 'gan might just around the corner - - if his pal or, inevitably, that of a cabinet minister - - was not also able to match his recent disappointing by - election result. [CLS] attempts to deliberately reshuffle are active and because there is no way to act on them, \" he said. [CLS] the challenge of [CLS]",
+        "[CLS] briefing this week, the experts pointed to research longorian led from his university in 2003 that found that a test could improve clinical action results in those with the neurological and behavioral disorders. [CLS] the \" the glory \" of the navy in venezuela friday was a foreign vessel involved in the hunt for a north korean, maritime authorities said. [CLS] \" georgia is excited to have shown the door to the slc right, \" buckeyes'joseph mileson said. [CLS] before stephen m. mammons was appointed in december 3 by dh setser ( potter ), managing director executive director general, sir peter jackson, said : \" [CLS]",
+        "[CLS] with newcastle top brass if freddy shepherd helped divest the asset - or get off hicks'hands, having once vehemently backed redevelopment of the business. [CLS] that's just not so happened to. [CLS] mr. friedman's brother - in - law attended central square in downtown moscow over the weekend and was attended to dozens of german restaurants, book clubs and separatchberg bars on tuesday, but mr. kennedy, some of the liberals and socialists and outside netz - west members indicated they would be off limits. [CLS] morgan stanley's kkr's recent shares pose no threat, because it's unclear [CLS]",
+        "[CLS] deal of euro 2010 / 11 to retain the same profile of winning - 3 - 3 ( jugs ) as it is for the 2014 world cup starting in 2010 and signing it in the double the yankee stadium at 2014. [CLS] the group, based in liverpool, is both belfast and abroad. [CLS] i feel i feel justified going to the emergency room at hospital, but when i am sick of fatigue i feel very plenty of pain but not fatigue. [CLS] in addition, pare's body in virginia ran the underworld difficult because states raised a fuller after mcdonnell was asked to repeal the proposed death penalty earlier this year. [CLS] indeed, [CLS]",
+        "[CLS] of patient headings have been warned of how they fell or went about identifying guesing or getting bitten by the nurses'hair. [CLS] meanwhile, the debates, george w. bush versus the democratic st bob dole, are messaged up. [CLS] often after two declined, pincani said chelsea would contact the \" transfer underworld \" to keep him on loan at stamford bridge forward and simply would pay back drogba by their transfer income. [CLS] such views clashed late saturday with monday's revelation he had died weeks into a court battle over 18 - year - old kittens mother, she he didn't know but [CLS]",
+        "[CLS] that. [CLS] roque santa cruz wants to inspire a revival in blackburn in order to survive their eventual home champions league second leg. [CLS] digging into a deeper bit, the oil and gas co. said tuesday it plans on moving its processing facility open to power to power and heat to power transmission to help customers switch from heat gas to propane. [CLS] she doesn't think they have any problems the the state filed a sex order in the past with that person, although mrs. pollack, 13, collected most of the money being sent back to her mother. [CLS] \" we're sort of in some momentum on the other [CLS]",
+        "[CLS]. [CLS] last month, health officials said they would not educate staff about the contamination, including keeping a new mother at home. [CLS] mr gloiffe said some 120 people voted on the issue, voted unanimously to take action and change their original, chaired by katherine pritchey. [CLS] \" i am very concerned about the injury to john collins ( fergusonskey ). [CLS] one of the victims was his son andanori, who denied receiving a gun played any other role and he belonged to a cult. [CLS] the book is a counterive to an influential range of issues of rights and economics. [CLS] their pictures were there [CLS]",
+        "[CLS] just before december and caught rb shawn kuhl for violating a college recruiting warrant, but both have since gone missing. [CLS] their work for bringing this to justice can be frustrating for med students, but for the many in cricket who confront the burden of this and other difficult challenges, it might feel like giving sticking up to freshman sheila wolpabe, who had 31 yards and two scores in the second quarter. [CLS] banking group refinance the notes to cash home buyers into lending at higher prices. [CLS] dr khan, economists faheer khan and johan van der steen as elite bowlers often played championship cricket in the middle and [CLS]",
+        "[CLS]. [CLS] in a broad uk sovereign wealth fund, almost 20pc of its return funds shrank by \u00a32 twigsm in the third quarter. [CLS] \" at home all are in less time...? [CLS] employers were required to verify given status at the mayo clinic clinic. [CLS] his admission this week is embarrassing. [CLS] communist party ideological spin : barrogcu said the party aimed on health friday with the queen's speech before resuming government meetings for the first time yet of the national assembly. [CLS] haines has produced a string of music, including his single friends, time out and i & amp ; myself : \" one [CLS]",
+        "[CLS] to the vatican's maj. francis michaeli erada. [CLS] she is right like no one else about the problem. [CLS] it is believed that the us taxpayers could be willing to sell off a position of its holdings and the minority shareholders in aig. [CLS] while wallace said stevens is against the impeachment because of his request, it's a lot of blanks on the wall. [CLS] so, while greece is on its safety - footing, there's more european funds. [CLS] frankfurt's golg ( de ), which is bound for stuttgart, fell 0. 4 percent, a few minutes after [CLS]",
+        "[CLS]'at occidental. [CLS] by the time the factory opened outside the silken heading hotel on ibn ghinnak, the in - sight fires were put out by a soldier manning a checkpoint. [CLS] an old market anomaly, there are about 170 businesses in that area that were been badly affected by the fallout - - the subprime market. [CLS] i don't know aboutbrooks. h. w. see why many americans but also children's family members say that for all these children, we are not a terrible risk of serious illness. [CLS] \" susan is going to do the through her jump and [CLS]",
+        "[CLS] crude stockpiles of around 68, 000 tonnes of crude oil was a running capacity of around 20 bpd. [CLS] service is not expected to be available more often time in france as the printers and rousseau are affected. [CLS] after spending gp in week, she talked to holly young, but two weeks for the first time. [CLS] the agency is facing itself into trouble if the government fails to exert leadership - - this time, municipal, or perhaps political, authority. [CLS] \" the u. s. senate is the last republican candidate you speak to about. [CLS] oscar - winners in beverly hills and toronto, around the world are [CLS]",
+        "[CLS] with like robert and alzheimer's and who be nice to provide images of lives we've been in during periods of time in the years they'd care of since then..., and then all of over in the next four years. [CLS] don't care if somebody will tell you why they're not who... because that's not. [CLS] for years, the wall street banks have had an incentive to keep promises that awaited them. [CLS] and resolving the credit crunch could also bring a number of money - saving measures, so if the banks invested in two of the big pay deals, it would [CLS]",
+        "[CLS] package of reforms called uksea, which streamlines britain's defence sector to operate the government using two systems. [CLS] so many things are being made. [CLS] king, however, said when \" i ate this aspect of my hand, that nothing awful happened \" before that was the day. [CLS] earlier, the lawn tennis association ( lta ) had told the ltf finalist it must be discussed, but players would have any chance of assessing only before opening their season knowing the need to complete an event. [CLS] his work on the field included about 70 in northern india in 1943, and scores of those during world war ii on the [CLS]",
+        "[CLS] using it. [CLS] seems it's an invasion of young voters for him to \" moones \" that no other party spoke in 13 years this year of \" the end of the year. \" [CLS] appearing on wednesday's \" countdown, \" vice - president joe biden will deliver the keynote address. [CLS] actor david letterman will be presented by marriott international's jorge jorge montero and ( and doug carlos campos ) power. [CLS] if i send this to the a3nware giving it a massive line - laden lgab, then i'll allow anyone outside my home to tell me any loss. [CLS] the australian [CLS]",
+        "[CLS]c what the indulgent propriety of vanity and horsedog titles only wore him prada joints and chic confidence around the window and the table with stiletto heel and falling volume and area justify as stockings. [CLS] cayne's decision, his allies said, caused serious problems in both trusts, which are backed by southern california suburbs and large investors ; and commercial properties, which lost some of their most financial units last year. [CLS] the sheer size of the sharply parched ski slopes in this scenic expanse of eastern austria state has nearly doubled substantially, virtually reaching the upper - middle - income farming communities. [CLS] a government spokesman [CLS]",
+        "[CLS] the children, elderly women and others. [CLS] another one is veronica frequency which, in turn, was a reduced risk to their daughters. [CLS] \" it may never have happened that i made a mind about an item unless they had that face that was expression on the psyche, \" says sarah, author of the ultimate fashion secret, which, in some of your feminists'\" living'representations, which is brand - friendly. [CLS] \" more patients see no benefits from skepticism and stronger adherence on the basis of alone, \" he says. [CLS] but the fact that we far the much cheaper marketing of these vehicles is daunting. [CLS]",
+        "[CLS] relations, or the banks were not so good? the police barracks the money was well paid. [CLS] \" and, the heck, they looked here. [CLS] hay ', though, noted the banking sector has bucked the course of a recession, where rising inflation continues at present during the slump, according to the survey. [CLS] riordan speaks with the recovery team saturday at murrayfield training.... [CLS] the judge told the jury that evidence against a certain complainant had been waiting to be heard since the second day of tuesday's hearing. [CLS] his tickets to the memorial cost at $ 100, 000 05 a ticket that [CLS]",
+        "[CLS] u. s. supreme court on monday upheld a $ 45. 5 million settlement that helped zain corp. save itself from its month - ago bankruptcy. [CLS] car pileups, however, are prone to - - and just as easily - - result in crashes as the jurors do. [CLS] zaborsky and nichols said trials into the cause for possible pirate attacks in international waters were settled without contempt of court. [CLS] the theftrockance became an international legend for three centuries until 1907 when its creators were always the dominant case, though one in his 22incoat, blond hair and tie because he wouldn't be happy [CLS]",
+        "[CLS] looking to step up a germany - hassle approach. [CLS] according to a major investment bank, some 8. 3 million are in the top 10 in the financial sector, each with a social dynamic. [CLS] they are in a gray belt. [CLS] the solicitor, who has also been linked to dr henry and the scottish people, was employed by the bbc's royal stories from the start of 2005 to diawall their case. [CLS] li said google needs to invest more network technology, but that technology could lead the founders back. [CLS] and did japanese believed they were foreign tourists from the slums of north malaya? [CLS] \" i think [CLS]",
+        "[CLS] senate. [CLS] they lost at the mallemagh in the 1974 club world cup, but lost dragons test kicker brad hadie, one of those players. [CLS] nat, of english heritage, was 52 and trembling with conces as he passed the retraction. [CLS] the indebted fear, levin argues, is the public would demand further help as part of the package and would not even if it really is a rescue bill. [CLS] louisa county chief patrick kelley, who used to lead the illinois 3r club last season, said that while there is affection for all, the team cannot wait. [CLS] the vast volume will, [CLS]",
+        "[CLS]'s top envoy to the war - torn dari capital said his country was ready to hand out a final detailed plan for peace. [CLS] \" fruition \" really took us sideways so we have seen being a nation first. [CLS] and then all you need to do is ensure the school is filled. [CLS] \" we would have had to do more also to secure care. [CLS] as at the beginning of the current generation of the systems, the clock of picking the sea. [CLS] subsequent research, corroborated by shiloh, also shows that the women imitate the kind food they digest when the men live. [CLS] on tuesday [CLS]",
+        "[CLS] police were told to feed out the voting by radio. [CLS] cancer cases are increasing by 1. 8 percent. [CLS] why - he's incorporated his agenda to some point like this - but austin would be a natural choice for his, the family team, and he says he would not be the last black governor in washington. and she didn't flip flops anything wrong with herself as she defended an opponent for whom she's pikeing. she is going to win her time. the she's justice! [CLS] who doesn't pause? running after running?, too often lived in tranquility? the? [CLS]",
+        "[CLS] leave it to you just because the \" better team play \" is why i must look at espn, n. f. l., cnbc or nbc. [CLS] that's the big three type of company. [CLS] his hits found their way to paris and other universities. [CLS] 21 attacks. [CLS] the 84 - foot motorcycle was pulled about 25 hours later because of bulky material. [CLS] as a medical student, my wife was put on leave at 11 p. m. and taken cheerfully to chinese therapy by having an at - last lecture to determine what the cantos or beefcake, \" skouah said. [CLS] however, [CLS]",
+        "[CLS] stuff that can happen to the dead. [CLS] when i visit muslims more peaceful activities ( as a buyer ) are permitted ; b ) islam has much influence and not perseplerost it! [CLS] ( 2 ) an income impact of 8. ( of sales losses, or ( ii ) of interest expense of 8. ( i ) of interest expense, depreciation and equity taxes. [CLS] the completed footage of these programs was distributed over the canadian border, where over 80, 000 videos were issued in february compared with september 30 of 2008. [CLS] det con hatuz said said his goal was like a long tunnel to surround secure [CLS]",
+        "[CLS] strange, not one of them selfish. [CLS] european trusts, the building techs global society to big business and social ones, public and private, struggle with the risk of fluctuations in the composition of their earnings portfolio. [CLS] he can only say this : \" hold the fund as we know the ftse 500 forward rate between now and 2012. \" [CLS] in separate comments, such grants would be halted at about 160 equestrian events since the end of 2006, so such contributions from olympic events would only be reduced. [CLS] there's a few more things to go in our schedule to keep sweeping up every goal of the season, \" things [CLS]",
+        "[CLS] rummyoon, in the first century ronald reagan, and would surely have been a rod for hope after shoulder in times of economic difficulty and concerns. [CLS] the clinical psychologist and eco - animal law student at roh's friends school at park island, whose hometown of tokyo produced a lost copy of \" i. t. \" [CLS] lou park, an american - born reality television sidekick to alan carr, died in 1998, the paper said. [CLS] before he left election he internalised voter registration and had support from anti - war voters in the old northern ireland labour party. [CLS] is he a pity at least when it is britain [CLS]",
+        "[CLS] percent to strengthen its chinese network under the \" 10 - year base contract \" romanian revealed this week. [CLS] i have spoken about some of our industry's decency, \" but now the company's profitability have improved, and i spoke to many small businesses who stand in our way. [CLS] the fureys also returned, using him as their promotional commentator. [CLS] spitzer told the show himself and - - declined to mention the family. [CLS] so far, michael jackson had not heard of jackson's death. [CLS] how do you know companies are doing profiteering purposes? [CLS] \" friday, andre agassi led [CLS]",
+        "[CLS] stem cell bank issues... [CLS] always on? [CLS] there are many possibilities unthinkable, \" said ban on the americas meeting of intergovernor nations in new york. [CLS] besides meeting to work on electoral reform next year that will presage the next congress - senate finance committee - - attorney general servano s. pianalto is expected to testify before the inspector general - - the two countries are expected to mount an effort to stop extra expenditures. [CLS] alexandre hostro, poking around the rubblecks area, paused to see the town. [CLS] by the time the torch goes up then on down the [CLS]",
+        "[CLS] answer some of sami's personal questions. [CLS] some 300 km ( 190 miles ) north from the capital monha n'uvut, where nkunda had circled the mark, the group said. [CLS] also in past years, abdallah has suffered another losing blow. [CLS] as ida fox, the new leader has from the rotunda, a reporter'inclination to the rampage helped keep away from noise and chatter ; the villagers were greeting townspeople and friends. [CLS] kosovo is also strongly opposed by russia. [CLS] the government was initially wary of the private attitudes provided by its citizens. [CLS] it was the mainrber and the associates at [CLS]",
+        "[CLS] inside the car, agents said saturday. [CLS] the u. s. and subsequent war, which hit nearly midway past mexico's pacific coast, left u. s. military craft in the harbor offshore. [CLS] the attacks have been surprisingly familiar from the country's political school but each experts confirm the recent incidents a few years ago, which have included the killing of their own lawmaker in the kenyan capital nairobi. [CLS] furthermore the scale of the abduction remained shocking. [CLS] mr savage said : \" one case i could see of was that she was under and she was drunk. [CLS] at the same time, corus is off to [CLS]",
+        "[CLS] distinctive sets, which i kept as a toddler for six decades. [CLS] he became the oldest man to win the us title in 1971. [CLS] green homes for midsize, such as the 1 cent false - homes network, the home exchange and america's arco park, are focusing on rentals, rather patios and new them enlton on neighborhood main roads. [CLS] britain owes sub - prime mortgages worth over \u00a367bn so far ; some of these extend to sub prime loans. [CLS] abbas said that rockets shot dead his high - school friends, including his black children, his wife and daughter. \" [CLS]",
+        "[CLS]'s ability to control the blaze fire, was particularly rare at times of massive - scale firefighting operations. [CLS] so i flew down to shanghai, not on turkey's domestic political radar. [CLS] we'd be dwelling \" on ( the 10 record recalls ) \" than saturn, jealous pierre cartier told the paris paris auto show in light of an investigation into by s - saab. [CLS] andrew miller, the republican attorney - general and mix phoenix republican of thanksgiving night, knows that he might be him during other meetings in afghanistan, president bush and other stops on sunday and on the campaign trail. [CLS] \" it is deliberately [CLS]",
+        "[CLS] lifes grips the place at the carbon store and selectss sales trucks, salads and burgers focusing on thoughtful goodies. [CLS] i think i am sitting on substantial debts. [CLS] the taliban routinely denied the claim that the soldier shot dead sherp because he was making headlines for his revenge. [CLS] bruton is credited with the the most for the wildcats as the head coach in southern california and three major conferences. [CLS] the wild tiger population of asia arrived in nepal from the temple mount on monday with spare education, though the asking kept to themselves a few steep streets that swarm covered mountain hideouts. [CLS] many journalists have been [CLS]",
+        "[CLS] success ; mr. kornk and mr. lelylev showed a varied goal again, except the soloist blocks the controls short on the outside body. [CLS] another group that had opposed the plan for the weekend declined comment. [CLS] jones said he began taking the events home when he saw her while stremme sometimes prayed. [CLS] tomatoes, pecans and tomatoes sobbled the last harvest and it brought luis giraldo a day when he didn't have to decide to sell the plant and raise one of his own. [CLS] next question merely a glitch, no last great plot spinning. [CLS] after the game, duke [CLS]",
+        "[CLS] - - who coach almost two times the wales salary, and many class children who are paying only \u00a3560, meanwhile. [CLS] but in that election, such as state times, the party excludes voters. [CLS] probably not republicans. [CLS] for five of his possessions he was working hard - pressed to stay firmly on a forgiving duck. [CLS] in 2009, this marking rose only some points to 4. 4 percent, compared to 15 percent in 2008 when it was at 792 achieving a raw score of 10. 6 percent. [CLS] mr jopling added : \" unless you're not affected or suffer from your experience as you'[CLS]",
+        "[CLS] - - - nearly half - hour campaign starts by traveling in china yesterday, then followed by a rare sichuan protest. [CLS] still, until internet pages listing everything were protected by the disclosure and warnings on craigslist's web site, the audience had shrunk. 2 - 4 % the top - speed comedy and the most dangerous one, involving bicyclists sliding into limousines. [CLS] the wave of such relief passed relatively small from the forbidden confines. [CLS] the dead date for 2010 was \" may, including a handgun, \" according to police. [CLS] in august, the decision increase quota for new entry by 254, [CLS]",
+        "[CLS] get a more singular, tennings point to reflect true aspects of the english intent, what the script's'amazing : singularly wonderfully and never - too respectful -'- this then - proding, now slightly pathological conclusion is not'applauded - - here': to the families who were strangled in this old sustained, ined gilt, juicy language to which its destiny has led us deep down, we can see that some can be learnt. [CLS] it is where children who are less than 100 feet [CLS] these activities heroic in africa, the same time, with the video of, fighting over [CLS]",
+        "[CLS] enforcement laws to oversees liberties. is it on how these shows speak for or a sampling of members?, said the project by he latinos.'[CLS] we are all married for christmas so we don't yet know how to start a talk to go into having more children, \" the presenter said. [CLS] lieberman did not refer the case and the agency only sought approval through a preservation conference. [CLS] he promptly retired personnel tosidise the apartments for $ 300 - a - night a month. [CLS] \" i like the turn, really ; but the lure is the pill. [CLS] a : gleevec fosters a [CLS]",
+        "[CLS] time - since almost 10,, we have already arrived. [CLS] the variety of bead and wildlife of michigan, one of the environmental groups, said activists were outraged, believing hellfish nets and other animals had ganged up on cats head for play closest to its calling. [CLS] white house spokeswoman megan williams said last tuesday that there would be compromise by the house energy committee option with a provision forcing it to move to a republican - occupied state senate. [CLS] after a year at a middle school junior team camp on the u. s., she was unable to learn whether the children were also taking part with such behaviour. [CLS]",
+        "[CLS] a series of strategically symbolic targets, \" the islamic jihad group has agreed to visit through many sites linked to kouli as well as central kandahar capital. [CLS] so the only question is whether we should make full use of other resources or need no take it all. [CLS] they aren't even far away, but they are. [CLS] media reports, however, said lloyds and the counsel may have been described as \" ghanalike chunks \" with minimal impact. [CLS] three decadess the aves? 26, who had three - goal matches in wednesday's opening game, leads wales 17 - 0. [CLS] still, [CLS]",
+        "[CLS] still, obama's likely edge in the race as black president grew rapidly in the months before in the five weeks before june 4. [CLS] ian ( \" catfish \" ) melvyn benoit, below, plays stephen, as the two nutty colt ( michele newman ) is mr. ron refreshing. [CLS] it was also that bath reopened for the game of dubai. [CLS] she had gotten, quite hungry as i was raising heels and feathers. [CLS] investigators suspect he was carrying about $ 5, 000. [CLS] stempniak pass from caley, by - play ), 13 : 22. [CLS] internal health investigators are calling for top [CLS]",
+        "[CLS]ges bring in the best - - a canoes of love, a set of tunnels and dying chicks. [CLS] they check rehabilients and smell byproducts. [CLS] israel has also ordered an investigation into how it was handled. [CLS] of a few thunderstorms moved up to hurricane strength on top of the predicted to fall on florida, followed by storm blowing up along the coast. [CLS] i then claimed this from the set off point of view, examining the city that i have only ever worked for and that i had laid out a timetable, do anything extra... second goal. [CLS] included in the very massive pack [CLS]",
+        "[CLS] advanced classes out of war, and also worked for ballac as an army secretary. [CLS] stoghart gained 131 yards and converted first - and - 1 play against ga. and became a pro bowl mvp in 2004. [CLS] south of highway 32 in too nu fashion, i needed it just over a year ago. [CLS] the french revolution, which released franco in 1960, ending the first world war, attacked africa for planning for its years by authorizing civil rights campaigns in places like italy, italy and britain, as if they had no problem of watching them. [CLS] meanwhile, passengers may be departing. [CLS] logging during the black friday [CLS]",
+        "[CLS] around 150 people were slain during a street rally last year. [CLS] but ukraine\u02bcs opposition leader, viktor v. yanov, tries to continue to fight charges reminiscent of the contentious dispute during 2004 when authorities fired missiles on 11, 000 na naogaz workers across the world. [CLS] a expedition which included a ride from the millennium bridge in johannesburg, b. the city's main street promenade. other cultural venues in cape town and bieken, helpful for three trips and excursions. country ( october to march 26 ). [CLS] her body was and they ran into the naked wedding parade, and saw the turning point of [CLS]",
+        "[CLS] goal and your partners are willing to use that \" to find something higher, \" he said. [CLS] but workers who know that service did not don a means sheet paid to go only once a month to find work. [CLS] that, and subsequent, cuts the company's recent years of sales of more 100, 000, which has been a modest sales slowdown for a long time a market dominated by the other automakers. [CLS] alcide alexander was a small american reporter working in baghdad 33 years and family left the bureau after being captured in afghanistan. [CLS] the spate raised alarm thursday when a few threatened to postpone elections to [CLS]",
+        "[CLS] and ammunition, particularly late 2008 during the global storm, policy makers required companies to give a name for the country - 9 domino new, 45 minutes in advance. [CLS] \" we feel the police force is doing better, \" said ron garcia, the di. [CLS] among the worries about the prp sector, s & p was buffeted by the plans of other leading firms in the sector. [CLS] cargo ships can do no tracking at cost, the ban says, and their owners will receive just one. [CLS] i thought i'd spend my long life alive, because \" kenneth \" took me to ramon lupine performing because i were [CLS]",
+        "[CLS] - based organisation said. [CLS] moscow moscow, july 15 ( upi ) - - last week's suicide bombing - - by a rebel united russia faction led by former defense minister ingushd kadyrov - - was to mark the fifth killing, leaving 27 people dead. [CLS] as for, you are more concerned! [CLS] we carry out legal action every day with a name dispelling these myths. [CLS] the debate, on which experienced and sometimes combative senator chuck hagel held a tempestuous retreat monday, epitomizes the embittered atmosphere of what is about to settle into crisis. [CLS] i doubt it [CLS]",
+        "[CLS] chronicling them for their food and the food for them that they rarely go after, said karen bass, d - calif heroin. [CLS] as for reporters, everyone said a national vote in june was to blame ( after monitoring extremist publications was closed ). [CLS] sheffield ae university has \" 300 major crimes \" in teaching and managerial disagreements, and indeed in the supply of drugs. [CLS] holbrooke released a statement to ministers : \" we have always ensured that us public support and guidance was necessary to treat them appropriately, \" said robinson. [CLS] supporters got to questions and unleashed so - called cuff gaffes mocking [CLS]",
+        "[CLS] no one was in the door of the six - foot wall were killed. [CLS] amanda would never allow in sleeping., in the environ of out - of - town schools such as this, a service is brought to to ensure inspiration. [CLS] the public slur at him remains as he entered his present. [CLS] officials refused clinton suggested wednesday that it would be \" an anomalous approach, \" but said the meeting would not revolutionize relations as u. s. officials acknowledged. [CLS] the rev. brenda w. potter and her husband dare not extend the credit to rabbi bonomy to earn a living. [CLS] you have [CLS]",
+        "[CLS]lbgate, said kessington police. [CLS] weilberg is the husband and son of an accomplished cooks. [CLS] a cultural - social demand for more clarity - - hundreds and thousands have taken to court if the prime minister, maria rosano, can put into place any new policies and special initiatives. [CLS] the source said the she learned that perino and perloque were discussing her clients'payments after harold son was contacted by their representatives weeks ago, police said. [CLS] \" the loss of demand may have been concentrated even as we thought'if a knee were out of our ', \" she said. [CLS] the san diego [CLS]",
+        "[CLS] and asked whether differences were made over which side, kim said simply overturning its obligations to crew members would take controversial decisions. [CLS] the collector got the stockyards $ 2, 000. [CLS] phelps'financial commitments may seem uncertain but he invited in his kids too. [CLS] rather that she doesn't learn the math. [CLS] in the street than there's those of those who suffered gunshot wounds, including most, \" he said in a mid - morning interview. [CLS] the conclusion of spears'season, but the $ 5 million millions oscar show champion is a pursuit of merit. [CLS] in demonstrations earlier, he, perhaps [CLS]",
+        "[CLS] show survive. [CLS] so critics fear that fundamental objections of regulation will utilise rejection in congress by lugo and obama. [CLS] she's one of the few people. [CLS] the two sites, hillwood cemetery and donington pool, have announced a much larger claim that the 50 percent offset would cut greenhouse gas use. [CLS] 2008 chess issue of the new yorker print, but is expected his book will be published before monday. [CLS] i nearly doubled and this was the only way to be able to live with the impression forever if anyone had signed my name and judged a fellow citizen. [CLS] then, two years after, the silver [CLS]",
+        "[CLS] [CLS] was it a reaction - - any of the information is as awful as thought. [CLS] ideally, thes can be revived by pumping water into the underworld. [CLS] pakistani officials say they had also made advanced evasive efforts to push back into the city - - new evidence of the influence of the taliban in baghdad at the totion of pakistani society. [CLS] crafting an objectivequest to fill the void between our principles and ethical principles may just be a hard practice to craft in the spring. [CLS] once a music student with 10 salvadorosis who tore through a car park next to me on the highway, scrawled once in all [CLS]"
+    ]
+}

results/smoke/samples/lm1b/duo/steps_2/upstream_hydra/.hydra/hydra.yaml

@@ -0,0 +1,181 @@
+hydra:
+  run:
+    dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/duo/steps_2/upstream_hydra
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/duo/steps_2/upstream_hydra
+    - hydra.mode=RUN
+    task:
+    - mode=sample_eval
+    - seed=42
+    - data=lm1b-wrap
+    - model=small
+    - model.length=128
+    - loader.batch_size=2
+    - loader.eval_batch_size=64
+    - sampling.num_sample_batches=1
+    - sampling.steps=2
+    - eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/flm_baselines/lm1b/lm1b_Duo.ckpt
+    - trainer.devices=1
+    - checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/duo/steps_2/upstream_hydra
+    - +wandb.offline=true
+    - eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/duo/steps_2/upstream_samples.json
+    - algo=duo_base
+    - algo.backbone=dit
+    - sampling.predictor=ancestral
+    - sampling.noise_removal=ancestral
+    - eval.disable_ema=False
+  job:
+    name: dlb_pinned_upstream_main
+    chdir: true
+    override_dirname: +wandb.offline=true,algo.backbone=dit,algo=duo_base,checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/duo/steps_2/upstream_hydra,data=lm1b-wrap,eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/flm_baselines/lm1b/lm1b_Duo.ckpt,eval.disable_ema=False,eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/duo/steps_2/upstream_samples.json,loader.batch_size=2,loader.eval_batch_size=64,mode=sample_eval,model.length=128,model=small,sampling.noise_removal=ancestral,sampling.num_sample_batches=1,sampling.predictor=ancestral,sampling.steps=2,seed=42,trainer.devices=1
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /home/ruizzhao/diffusion_baseline
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/ruizzhao/diffusion_baseline/upstreams/duo/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/duo/steps_2/upstream_hydra
+    choices:
+      algo: duo_base
+      prior: none
+      lr_scheduler: constant_warmup
+      noise: log-linear
+      strategy: ddp
+      model: small
+      data: lm1b-wrap
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

results/smoke/samples/lm1b/duo/steps_32/upstream_hydra/config_tree.txt

@@ -0,0 +1,169 @@
+CONFIG
+├── mode
+│   └── sample_eval                                                             
+├── seed
+│   └── 42                                                                      
+├── loader
+│   └── global_batch_size: 512                                                  
+│       eval_global_batch_size: 512                                             
+│       batch_size: 2                                                           
+│       eval_batch_size: 64                                                     
+│       num_workers: 512                                                        
+│       pin_memory: true                                                        
+│                                                                               
+├── sampling
+│   └── predictor: ancestral                                                    
+│       steps: 32                                                               
+│       noise_removal: ancestral                                                
+│       use_float64: true                                                       
+│       p_nucleus: 1.0                                                          
+│       num_sample_batches: 1                                                   
+│       num_sample_log: 2                                                       
+│       semi_ar: false                                                          
+│       stride_length: 1                                                        
+│       num_strides: 1                                                          
+│       guid_weight: null                                                       
+│       psi:                                                                    
+│         time_profile: linear                                                  
+│         high_mode: pure-posterior                                             
+│         middle_mode: pure-posterior                                           
+│         low_mode: pure-posterior                                              
+│         high_frac: 0.2                                                        
+│         middle_frac: 0.6                                                      
+│                                                                               
+├── training
+│   └── ema: 0.9999                                                             
+│       antithetic_sampling: true                                               
+│       importance_sampling: false                                              
+│       sampling_eps: 0.001                                                     
+│       change_of_variables: false                                              
+│       loss_precision: bf16                                                    
+│       finetune_path: ''                                                       
+│       class_dropout_p: 0.1                                                    
+│                                                                               
+├── eval
+│   └── checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference
+│       disable_ema: false                                                      
+│       compute_generative_perplexity: false                                    
+│       perplexity_batch_size: 8                                                
+│       compute_perplexity_on_sanity: false                                     
+│       gen_ppl_eval_model_name_or_path: gpt2-large                             
+│       generate_samples: true                                                  
+│       generated_samples_path: /home/ruizzhao/diffusion_baseline/results/smoke/
+│                                                                               
+├── optim
+│   └── weight_decay: 0                                                         
+│       lr: 0.0003                                                              
+│       beta1: 0.9                                                              
+│       beta2: 0.999                                                            
+│       eps: 1.0e-08                                                            
+│                                                                               
+├── trainer
+│   └── _target_: lightning.Trainer                                             
+│       accelerator: cuda                                                       
+│       num_nodes: 1                                                            
+│       devices: 1                                                              
+│       accumulate_grad_batches: 256                                            
+│       gradient_clip_val: 1.0                                                  
+│       precision: bf16                                                         
+│       num_sanity_val_steps: 2                                                 
+│       max_steps: 1000000                                                      
+│       log_every_n_steps: 100                                                  
+│       limit_train_batches: 1.0                                                
+│       limit_val_batches: 1.0                                                  
+│       val_check_interval: 5000                                                
+│       check_val_every_n_epoch: null                                           
+│                                                                               
+├── wandb
+│   └── project: duo                                                            
+│       group: null                                                             
+│       job_type: null                                                          
+│       name: null                                                              
+│       id: None_42                                                             
+│       tags:                                                                   
+│       - log-linear                                                            
+│       - lm1b                                                                  
+│       - lm1b                                                                  
+│       - duo_base                                                              
+│       offline: true                                                           
+│                                                                               
+├── checkpointing
+│   └── save_dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/d
+│       resume_from_ckpt: true                                                  
+│       resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results/smoke/sample
+│                                                                               
+├── callbacks
+│   └── checkpoint_every_n_steps:                                               
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                 
+│         save_top_k: -1                                                        
+│         save_last: true                                                       
+│         dirpath: /home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/
+│         verbose: true                                                         
+│         auto_insert_metric_name: false                                        
+│         every_n_train_steps: 500                                              
+│       checkpoint_monitor:                                                     
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                 
+│         monitor: val/nll                                                      
+│         mode: min                                                             
+│         save_top_k: 1                                                         
+│         save_last: false                                                      
+│         dirpath: /home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/
+│         filename: best                                                        
+│         auto_insert_metric_name: false                                        
+│         verbose: true                                                         
+│       learning_rate_monitor:                                                  
+│         _target_: lightning.pytorch.callbacks.LearningRateMonitor             
+│         logging_interval: step                                                
+│                                                                               
+├── data
+│   └── train: lm1b                                                             
+│       valid: lm1b                                                             
+│       modality: text                                                          
+│       tokenizer_name_or_path: bert-base-uncased                               
+│       cache_dir: /share/kuleshov/ssahoo/textdiffusion/data                    
+│       wrap: true                                                              
+│       streaming: false                                                        
+│       insert_train_eos: true                                                  
+│       insert_valid_eos: true                                                  
+│                                                                               
+├── model
+│   └── name: small                                                             
+│       type: ddit                                                              
+│       hidden_size: 768                                                        
+│       cond_dim: 128                                                           
+│       length: 128                                                             
+│       n_blocks: 12                                                            
+│       n_heads: 12                                                             
+│       scale_by_sigma: true                                                    
+│       dropout: 0.1                                                            
+│       tie_word_embeddings: false                                              
+│       vocab_lookup: true                                                      
+│                                                                               
+├── strategy
+│   └── _target_: lightning.pytorch.strategies.DDPStrategy                      
+│       find_unused_parameters: false                                           
+│                                                                               
+├── noise
+│   └── type: log-linear                                                        
+│       eps: 0.001                                                              
+│                                                                               
+├── lr_scheduler
+│   └── _target_: transformers.get_constant_schedule_with_warmup                
+│       num_warmup_steps: 2500                                                  
+│                                                                               
+├── prior
+│   └── type: none                                                              
+│       latent_width: 0                                                         
+│       latent_height: 0                                                        
+│                                                                               
+└── algo
+    └── name: duo_base                                                          
+        backbone: dit                                                           
+        parameterization: mean                                                  
+        time_conditioning: true                                                 
+        T: 0                                                                    
+        subs_masking: false                                                     
+        causal_attention: false                                                 
+        ignore_bos: false                                                       
+        loss_type: elbo                                                         
+                                                                                

results/smoke/samples/lm1b/duo/steps_4/upstream_hydra/config_tree.txt

@@ -0,0 +1,169 @@
+CONFIG
+├── mode
+│   └── sample_eval                                                             
+├── seed
+│   └── 42                                                                      
+├── loader
+│   └── global_batch_size: 512                                                  
+│       eval_global_batch_size: 512                                             
+│       batch_size: 2                                                           
+│       eval_batch_size: 64                                                     
+│       num_workers: 512                                                        
+│       pin_memory: true                                                        
+│                                                                               
+├── sampling
+│   └── predictor: ancestral                                                    
+│       steps: 4                                                                
+│       noise_removal: ancestral                                                
+│       use_float64: true                                                       
+│       p_nucleus: 1.0                                                          
+│       num_sample_batches: 1                                                   
+│       num_sample_log: 2                                                       
+│       semi_ar: false                                                          
+│       stride_length: 1                                                        
+│       num_strides: 1                                                          
+│       guid_weight: null                                                       
+│       psi:                                                                    
+│         time_profile: linear                                                  
+│         high_mode: pure-posterior                                             
+│         middle_mode: pure-posterior                                           
+│         low_mode: pure-posterior                                              
+│         high_frac: 0.2                                                        
+│         middle_frac: 0.6                                                      
+│                                                                               
+├── training
+│   └── ema: 0.9999                                                             
+│       antithetic_sampling: true                                               
+│       importance_sampling: false                                              
+│       sampling_eps: 0.001                                                     
+│       change_of_variables: false                                              
+│       loss_precision: bf16                                                    
+│       finetune_path: ''                                                       
+│       class_dropout_p: 0.1                                                    
+│                                                                               
+├── eval
+│   └── checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference
+│       disable_ema: false                                                      
+│       compute_generative_perplexity: false                                    
+│       perplexity_batch_size: 8                                                
+│       compute_perplexity_on_sanity: false                                     
+│       gen_ppl_eval_model_name_or_path: gpt2-large                             
+│       generate_samples: true                                                  
+│       generated_samples_path: /home/ruizzhao/diffusion_baseline/results/smoke/
+│                                                                               
+├── optim
+│   └── weight_decay: 0                                                         
+│       lr: 0.0003                                                              
+│       beta1: 0.9                                                              
+│       beta2: 0.999                                                            
+│       eps: 1.0e-08                                                            
+│                                                                               
+├── trainer
+│   └── _target_: lightning.Trainer                                             
+│       accelerator: cuda                                                       
+│       num_nodes: 1                                                            
+│       devices: 1                                                              
+│       accumulate_grad_batches: 256                                            
+│       gradient_clip_val: 1.0                                                  
+│       precision: bf16                                                         
+│       num_sanity_val_steps: 2                                                 
+│       max_steps: 1000000                                                      
+│       log_every_n_steps: 100                                                  
+│       limit_train_batches: 1.0                                                
+│       limit_val_batches: 1.0                                                  
+│       val_check_interval: 5000                                                
+│       check_val_every_n_epoch: null                                           
+│                                                                               
+├── wandb
+│   └── project: duo                                                            
+│       group: null                                                             
+│       job_type: null                                                          
+│       name: null                                                              
+│       id: None_42                                                             
+│       tags:                                                                   
+│       - log-linear                                                            
+│       - lm1b                                                                  
+│       - lm1b                                                                  
+│       - duo_base                                                              
+│       offline: true                                                           
+│                                                                               
+├── checkpointing
+│   └── save_dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/d
+│       resume_from_ckpt: true                                                  
+│       resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results/smoke/sample
+│                                                                               
+├── callbacks
+│   └── checkpoint_every_n_steps:                                               
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                 
+│         save_top_k: -1                                                        
+│         save_last: true                                                       
+│         dirpath: /home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/
+│         verbose: true                                                         
+│         auto_insert_metric_name: false                                        
+│         every_n_train_steps: 500                                              
+│       checkpoint_monitor:                                                     
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                 
+│         monitor: val/nll                                                      
+│         mode: min                                                             
+│         save_top_k: 1                                                         
+│         save_last: false                                                      
+│         dirpath: /home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/
+│         filename: best                                                        
+│         auto_insert_metric_name: false                                        
+│         verbose: true                                                         
+│       learning_rate_monitor:                                                  
+│         _target_: lightning.pytorch.callbacks.LearningRateMonitor             
+│         logging_interval: step                                                
+│                                                                               
+├── data
+│   └── train: lm1b                                                             
+│       valid: lm1b                                                             
+│       modality: text                                                          
+│       tokenizer_name_or_path: bert-base-uncased                               
+│       cache_dir: /share/kuleshov/ssahoo/textdiffusion/data                    
+│       wrap: true                                                              
+│       streaming: false                                                        
+│       insert_train_eos: true                                                  
+│       insert_valid_eos: true                                                  
+│                                                                               
+├── model
+│   └── name: small                                                             
+│       type: ddit                                                              
+│       hidden_size: 768                                                        
+│       cond_dim: 128                                                           
+│       length: 128                                                             
+│       n_blocks: 12                                                            
+│       n_heads: 12                                                             
+│       scale_by_sigma: true                                                    
+│       dropout: 0.1                                                            
+│       tie_word_embeddings: false                                              
+│       vocab_lookup: true                                                      
+│                                                                               
+├── strategy
+│   └── _target_: lightning.pytorch.strategies.DDPStrategy                      
+│       find_unused_parameters: false                                           
+│                                                                               
+├── noise
+│   └── type: log-linear                                                        
+│       eps: 0.001                                                              
+│                                                                               
+├── lr_scheduler
+│   └── _target_: transformers.get_constant_schedule_with_warmup                
+│       num_warmup_steps: 2500                                                  
+│                                                                               
+├── prior
+│   └── type: none                                                              
+│       latent_width: 0                                                         
+│       latent_height: 0                                                        
+│                                                                               
+└── algo
+    └── name: duo_base                                                          
+        backbone: dit                                                           
+        parameterization: mean                                                  
+        time_conditioning: true                                                 
+        T: 0                                                                    
+        subs_masking: false                                                     
+        causal_attention: false                                                 
+        ignore_bos: false                                                       
+        loss_type: elbo                                                         
+                                                                                

results/smoke/samples/lm1b/duo/steps_8/upstream_hydra/.hydra/config.yaml

@@ -0,0 +1,151 @@
+mode: sample_eval
+seed: 42
+loader:
+  global_batch_size: 512
+  eval_global_batch_size: ${.global_batch_size}
+  batch_size: 2
+  eval_batch_size: 64
+  num_workers: ${eval:"len(__import__('os').sched_getaffinity(0))"}
+  pin_memory: true
+sampling:
+  predictor: ancestral
+  steps: 8
+  noise_removal: ancestral
+  use_float64: true
+  p_nucleus: 1.0
+  num_sample_batches: 1
+  num_sample_log: 2
+  semi_ar: false
+  stride_length: 1
+  num_strides: 1
+  guid_weight: null
+  psi:
+    time_profile: linear
+    high_mode: pure-posterior
+    middle_mode: pure-posterior
+    low_mode: pure-posterior
+    high_frac: 0.2
+    middle_frac: 0.6
+training:
+  ema: 0.9999
+  antithetic_sampling: true
+  importance_sampling: false
+  sampling_eps: 0.001
+  change_of_variables: false
+  loss_precision: bf16
+  finetune_path: ''
+  class_dropout_p: 0.1
+eval:
+  checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/flm_baselines/lm1b/lm1b_Duo.ckpt
+  disable_ema: false
+  compute_generative_perplexity: false
+  perplexity_batch_size: 8
+  compute_perplexity_on_sanity: false
+  gen_ppl_eval_model_name_or_path: gpt2-large
+  generate_samples: true
+  generated_samples_path: /home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/duo/steps_8/upstream_samples.json
+optim:
+  weight_decay: 0
+  lr: 0.0003
+  beta1: 0.9
+  beta2: 0.999
+  eps: 1.0e-08
+trainer:
+  _target_: lightning.Trainer
+  accelerator: cuda
+  num_nodes: 1
+  devices: 1
+  accumulate_grad_batches: ${div_up:${loader.global_batch_size}, ${eval:${trainer.devices}
+    * ${loader.batch_size} * ${trainer.num_nodes}}}
+  gradient_clip_val: 1.0
+  precision: bf16
+  num_sanity_val_steps: 2
+  max_steps: 1000000
+  log_every_n_steps: 100
+  limit_train_batches: 1.0
+  limit_val_batches: 1.0
+  val_check_interval: 5000
+  check_val_every_n_epoch: null
+wandb:
+  project: duo
+  group: null
+  job_type: null
+  name: null
+  id: ${.name}_${seed}
+  tags:
+  - ${noise.type}
+  - ${data.train}
+  - ${data.valid}
+  - ${algo.name}
+  offline: true
+checkpointing:
+  save_dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/duo/steps_8/upstream_hydra
+  resume_from_ckpt: true
+  resume_ckpt_path: ${.save_dir}/checkpoints/last.ckpt
+callbacks:
+  checkpoint_every_n_steps:
+    _target_: lightning.pytorch.callbacks.ModelCheckpoint
+    save_top_k: -1
+    save_last: true
+    dirpath: ${checkpointing.save_dir}/checkpoints
+    verbose: true
+    auto_insert_metric_name: false
+    every_n_train_steps: 500
+  checkpoint_monitor:
+    _target_: lightning.pytorch.callbacks.ModelCheckpoint
+    monitor: val/nll
+    mode: min
+    save_top_k: 1
+    save_last: false
+    dirpath: ${checkpointing.save_dir}/checkpoints
+    filename: best
+    auto_insert_metric_name: false
+    verbose: true
+  learning_rate_monitor:
+    _target_: lightning.pytorch.callbacks.LearningRateMonitor
+    logging_interval: step
+data:
+  train: lm1b
+  valid: lm1b
+  modality: text
+  tokenizer_name_or_path: bert-base-uncased
+  cache_dir: /share/kuleshov/ssahoo/textdiffusion/data
+  wrap: true
+  streaming: false
+  insert_train_eos: true
+  insert_valid_eos: true
+model:
+  name: small
+  type: ddit
+  hidden_size: 768
+  cond_dim: 128
+  length: 128
+  n_blocks: 12
+  n_heads: 12
+  scale_by_sigma: true
+  dropout: 0.1
+  tie_word_embeddings: false
+  vocab_lookup: true
+strategy:
+  _target_: lightning.pytorch.strategies.DDPStrategy
+  find_unused_parameters: false
+noise:
+  type: log-linear
+  eps: 0.001
+lr_scheduler:
+  _target_: transformers.get_constant_schedule_with_warmup
+  num_warmup_steps: 2500
+prior:
+  type: none
+  latent_width: 0
+  latent_height: 0
+algo:
+  name: duo_base
+  backbone: dit
+  parameterization: mean
+  time_conditioning: true
+  T: 0
+  subs_masking: false
+  causal_attention: false
+  ignore_bos: false
+  loss_type: elbo

results/smoke/samples/lm1b/duo/steps_8/upstream_hydra/.hydra/hydra.yaml

@@ -0,0 +1,181 @@
+hydra:
+  run:
+    dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/duo/steps_8/upstream_hydra
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/duo/steps_8/upstream_hydra
+    - hydra.mode=RUN
+    task:
+    - mode=sample_eval
+    - seed=42
+    - data=lm1b-wrap
+    - model=small
+    - model.length=128
+    - loader.batch_size=2
+    - loader.eval_batch_size=64
+    - sampling.num_sample_batches=1
+    - sampling.steps=8
+    - eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/flm_baselines/lm1b/lm1b_Duo.ckpt
+    - trainer.devices=1
+    - checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/duo/steps_8/upstream_hydra
+    - +wandb.offline=true
+    - eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/duo/steps_8/upstream_samples.json
+    - algo=duo_base
+    - algo.backbone=dit
+    - sampling.predictor=ancestral
+    - sampling.noise_removal=ancestral
+    - eval.disable_ema=False
+  job:
+    name: dlb_pinned_upstream_main
+    chdir: true
+    override_dirname: +wandb.offline=true,algo.backbone=dit,algo=duo_base,checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/duo/steps_8/upstream_hydra,data=lm1b-wrap,eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/flm_baselines/lm1b/lm1b_Duo.ckpt,eval.disable_ema=False,eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/duo/steps_8/upstream_samples.json,loader.batch_size=2,loader.eval_batch_size=64,mode=sample_eval,model.length=128,model=small,sampling.noise_removal=ancestral,sampling.num_sample_batches=1,sampling.predictor=ancestral,sampling.steps=8,seed=42,trainer.devices=1
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /home/ruizzhao/diffusion_baseline
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/ruizzhao/diffusion_baseline/upstreams/duo/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/duo/steps_8/upstream_hydra
+    choices:
+      algo: duo_base
+      prior: none
+      lr_scheduler: constant_warmup
+      noise: log-linear
+      strategy: ddp
+      model: small
+      data: lm1b-wrap
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

results/smoke/samples/lm1b/flm/steps_1024/stdout.log

@@ -0,0 +1,190 @@
+DIT: USE_COMPILE=False
+CONFIG
+├── mode
+│   └── sample_eval                                                             
+├── is_di4c
+│   └── False                                                                   
+├── is_di4c_deterministic
+│   └── True                                                                    
+├── seed
+│   └── 42                                                                      
+├── loader
+│   └── global_batch_size: 512                                                  
+│       eval_global_batch_size: 512                                             
+│       batch_size: 2                                                           
+│       eval_batch_size: 32                                                     
+│       num_workers: 4                                                          
+│       pin_memory: true                                                        
+│                                                                               
+├── sampling
+│   └── predictor: ancestral                                                    
+│       steps: 1024                                                             
+│       noise_removal: ancestral                                                
+│       use_float64: true                                                       
+│       p_nucleus: 1.0                                                          
+│       num_sample_batches: 1                                                   
+│       num_sample_log: 2                                                       
+│       semi_ar: false                                                          
+│       stride_length: 1                                                        
+│       num_strides: 1                                                          
+│       num_reflow_samples: 50000                                               
+│       temperature: 1.0                                                        
+│       solver: euler                                                           
+│       gamma: 0.0                                                              
+│                                                                               
+├── training
+│   └── loss_type: flow                                                         
+│       pred_type: x0                                                           
+│       ema: 0.9999                                                             
+│       antithetic_sampling: true                                               
+│       importance_sampling: false                                              
+│       sampling_eps: 0.001                                                     
+│       change_of_variables: false                                              
+│       loss_precision: bf16                                                    
+│       finetune_path: ''                                                       
+│       not_sampling_t: false                                                   
+│       load_ema: false                                                         
+│       t_curriculum_steps: 100000                                              
+│       t_curriculum_min: 0.0                                                   
+│       t_curriculum_max: 1.0                                                   
+│                                                                               
+├── eval
+│   └── checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/official/
+│       disable_ema: false                                                      
+│       ema_decay: null                                                         
+│       compute_generative_perplexity: true                                     
+│       perplexity_batch_size: 8                                                
+│       compute_perplexity_on_sanity: false                                     
+│       gen_ppl_eval_model_name_or_path: gpt2-large                             
+│       generate_samples: true                                                  
+│       generated_samples_path: /home/ruizzhao/diffusion_baseline/results/smoke/
+│                                                                               
+├── optim
+│   └── weight_decay: 0                                                         
+│       lr: 0.0003                                                              
+│       beta1: 0.9                                                              
+│       beta2: 0.999                                                            
+│       eps: 1.0e-08                                                            
+│       ln_tune: none                                                           
+│                                                                               
+├── trainer
+│   └── _target_: lightning.Trainer                                             
+│       accelerator: cuda                                                       
+│       num_nodes: 1                                                            
+│       devices: 1                                                              
+│       accumulate_grad_batches: 256                                            
+│       gradient_clip_val: 1.0                                                  
+│       precision: bf16                                                         
+│       num_sanity_val_steps: 2                                                 
+│       max_steps: 1000000                                                      
+│       log_every_n_steps: 100                                                  
+│       limit_train_batches: 1.0                                                
+│       limit_val_batches: 1.0                                                  
+│       val_check_interval: 5000                                                
+│                                                                               
+├── wandb
+│   └── project: flm                                                            
+│       notes: FLM                                                              
+│       group: null                                                             
+│       job_type: null                                                          
+│       name: null                                                              
+│       id: None_42                                                             
+│       tags:                                                                   
+│       - lm1b                                                                  
+│       - flm                                                                   
+│       offline: true                                                           
+│                                                                               
+├── checkpointing
+│   └── save_dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/f
+│       resume_from_ckpt: false                                                 
+│       resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results/smoke/sample
+│                                                                               
+├── callbacks
+│   └── checkpoint_every_n_steps:                                               
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                 
+│         save_top_k: -1                                                        
+│         save_last: false                                                      
+│         dirpath: /home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/
+│         verbose: true                                                         
+│         auto_insert_metric_name: false                                        
+│         every_n_train_steps: 25000                                            
+│       checkpoint_monitor:                                                     
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                 
+│         monitor: val/nll                                                      
+│         mode: min                                                             
+│         save_top_k: 1                                                         
+│         save_last: true                                                       
+│         dirpath: /home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/
+│         filename: best_nll                                                    
+│         auto_insert_metric_name: false                                        
+│         verbose: true                                                         
+│       learning_rate_monitor:                                                  
+│         _target_: lightning.pytorch.callbacks.LearningRateMonitor             
+│         logging_interval: step                                                
+│                                                                               
+├── data
+│   └── train: lm1b                                                             
+│       valid: lm1b                                                             
+│       tokenizer_name_or_path: bert-base-uncased                               
+│       cache_dir: ./cache                                                      
+│       wrap: true                                                              
+│       streaming: false                                                        
+│       insert_train_eos: true                                                  
+│       insert_valid_eos: true                                                  
+│                                                                               
+├── model
+│   └── name: small                                                             
+│       type: ddit                                                              
+│       hidden_size: 768                                                        
+│       cond_dim: 128                                                           
+│       length: 128                                                             
+│       n_blocks: 12                                                            
+│       n_heads: 12                                                             
+│       scale_by_sigma: true                                                    
+│       dropout: 0.1                                                            
+│       tie_word_embeddings: false                                              
+│       vocab_lookup: true                                                      
+│                                                                               
+├── strategy
+│   └── _target_: lightning.pytorch.strategies.DDPStrategy                      
+│       find_unused_parameters: false                                           
+│                                                                               
+├── noise
+│   └── type: log-linear                                                        
+│       parameterization: log-linear                                            
+│       eps: 0                                                                  
+│       denoiser_latent_conditioning: -1                                        
+│       freeze_encoder: false                                                   
+│       freeze_decoder: false                                                   
+│                                                                               
+├── lr_scheduler
+│   └── _target_: transformers.get_constant_schedule_with_warmup                
+│       num_warmup_steps: 2500                                                  
+│                                                                               
+├── prior
+│   └── type: none                                                              
+│       latent_width: 0                                                         
+│       latent_height: 0                                                        
+│                                                                               
+└── algo
+    └── name: flm                                                               
+        backbone: dit                                                           
+        parameterization: mean                                                  
+        time_conditioning: true                                                 
+        T: 0                                                                    
+        subs_masking: false                                                     
+        causal_attention: false                                                 
+        ignore_bos: false                                                       
+        t_max: 1.0                                                              
+        t_min: 0.0                                                              
+        double_temb: false                                                      
+        learnable_loss_weighting: false                                         
+                                                                                
+[2026-08-03 14:45:41,443][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
+Resuming training from checkpoint...
+generation start: 2026-08-03 14:45:54
+Copying EMA parameters to model
+generation end: 2026-08-03 14:46:12
+Generative perplexity: 96.87264251708984
+Sample entropy: 4.289146423339844
+Samples saved at: /home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/flm/steps_1024/upstream_samples.json

results/smoke/samples/lm1b/flm/steps_16/stderr.log

@@ -0,0 +1,3 @@
+Seed set to 42
+/home/ruizzhao/diffusion_baseline/upstreams/flm/models/dit.py:476: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(enabled=False):

results/smoke/samples/lm1b/flm/steps_2/upstream_hydra/.hydra/hydra.yaml

@@ -0,0 +1,182 @@
+hydra:
+  run:
+    dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/flm/steps_2/upstream_hydra
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/flm/steps_2/upstream_hydra
+    - hydra.mode=RUN
+    task:
+    - mode=sample_eval
+    - seed=42
+    - data=lm1b-wrap
+    - model=small
+    - model.length=128
+    - loader.batch_size=2
+    - loader.eval_batch_size=32
+    - sampling.num_sample_batches=1
+    - sampling.steps=2
+    - eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/flm_ckpt/lm1b/lm1b_flm.ckpt
+    - trainer.devices=1
+    - checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/flm/steps_2/upstream_hydra
+    - +wandb.offline=true
+    - eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/flm/steps_2/upstream_samples.json
+    - algo=flm
+    - algo.backbone=dit
+    - eval.disable_ema=False
+    - sampling.solver=euler
+    - algo.double_temb=False
+    - sampling.gamma=0.0
+  job:
+    name: dlb_pinned_upstream_main
+    chdir: true
+    override_dirname: +wandb.offline=true,algo.backbone=dit,algo.double_temb=False,algo=flm,checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/flm/steps_2/upstream_hydra,data=lm1b-wrap,eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/flm_ckpt/lm1b/lm1b_flm.ckpt,eval.disable_ema=False,eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/flm/steps_2/upstream_samples.json,loader.batch_size=2,loader.eval_batch_size=32,mode=sample_eval,model.length=128,model=small,sampling.gamma=0.0,sampling.num_sample_batches=1,sampling.solver=euler,sampling.steps=2,seed=42,trainer.devices=1
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /home/ruizzhao/diffusion_baseline
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/ruizzhao/diffusion_baseline/upstreams/flm/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/flm/steps_2/upstream_hydra
+    choices:
+      algo: flm
+      prior: none
+      lr_scheduler: constant_warmup
+      noise: log-linear
+      strategy: ddp
+      model: small
+      data: lm1b-wrap
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

results/smoke/samples/lm1b/flm/steps_4/run_metadata.json

@@ -0,0 +1,89 @@
+{
+  "checkpoint_lock_id": "recipe:flm_lm1b_official_ckpt:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0",
+  "checkpoint_selection": {
+    "output": "checkpoints/official/flm_ckpt/lm1b",
+    "recipe": "flm_lm1b_official_ckpt",
+    "sampling_checkpoint": "lm1b_flm.ckpt",
+    "sampling_config": null,
+    "source": "flm",
+    "source_commit": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
+    "teacher_adapter": null,
+    "teacher_family": "continuous_flm"
+  },
+  "checkpoint_sha256": "1d31308c7174e804931f22248ef664cda27005747c8d5385f7971b960577d0e4",
+  "checkpoint_teacher_family": "continuous_flm",
+  "command": [
+    "/opt/anaconda3/envs/dlb-flm/bin/python",
+    "-u",
+    "-m",
+    "dlb.adapters.capture",
+    "--upstream-entrypoint=/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py",
+    "--capture-path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/flm/steps_4/upstream_token_ids.json",
+    "--",
+    "mode=sample_eval",
+    "seed=42",
+    "data=lm1b-wrap",
+    "model=small",
+    "model.length=128",
+    "loader.batch_size=2",
+    "loader.eval_batch_size=32",
+    "sampling.num_sample_batches=1",
+    "sampling.steps=4",
+    "eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/flm_ckpt/lm1b/lm1b_flm.ckpt",
+    "trainer.devices=1",
+    "hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/flm/steps_4/upstream_hydra",
+    "checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/flm/steps_4/upstream_hydra",
+    "+wandb.offline=true",
+    "eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/flm/steps_4/upstream_samples.json",
+    "algo=flm",
+    "algo.backbone=dit",
+    "eval.disable_ema=False",
+    "sampling.solver=euler",
+    "algo.double_temb=False",
+    "sampling.gamma=0.0"
+  ],
+  "command_sha256": "a1fec4dd8bfffc2e5d131bea514ee461b67f233682c15b46ea2c803048ea6c24",
+  "config_sha256": "206d75b3b7cf3fb35844c053ae552537d7471b93bf29756dafc1ac616b096994",
+  "environment": {
+    "cuda_version": null,
+    "cuda_visible_devices": "2",
+    "hostname": "DS210029",
+    "platform": "Linux-5.15.0-186-generic-x86_64-with-glibc2.35",
+    "python": "3.11.15",
+    "torch_version": "2.5.1"
+  },
+  "finished_at": "2026-08-03T06:45:34.744934Z",
+  "identity": {
+    "adapter_identity": "dlb.adapters.flm:v2|dlb.adapters.flm:FLMAdapter|c3a6a4573200fecf38a47fc6e7bc78254ca362fff3b96079dc2efa9f8e27aff0",
+    "checkpoint_lock_id": "recipe:flm_lm1b_official_ckpt:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0",
+    "checkpoint_selection": {
+      "output": "checkpoints/official/flm_ckpt/lm1b",
+      "recipe": "flm_lm1b_official_ckpt",
+      "sampling_checkpoint": "lm1b_flm.ckpt",
+      "sampling_config": null,
+      "source": "flm",
+      "source_commit": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
+      "teacher_adapter": null,
+      "teacher_family": "continuous_flm"
+    },
+    "checkpoint_sha256": "1d31308c7174e804931f22248ef664cda27005747c8d5385f7971b960577d0e4",
+    "checkpoint_teacher_family": "continuous_flm",
+    "command_sha256": "a1fec4dd8bfffc2e5d131bea514ee461b67f233682c15b46ea2c803048ea6c24",
+    "config_sha256": "206d75b3b7cf3fb35844c053ae552537d7471b93bf29756dafc1ac616b096994",
+    "dataset_id": "lm1b",
+    "device": null,
+    "environment": "dlb-flm",
+    "model_id": "flm",
+    "results_root": "/home/ruizzhao/diffusion_baseline/results/smoke",
+    "run_id": "flm-lm1b-steps-4",
+    "sample_count": 1,
+    "seed": 42,
+    "source_sha256": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
+    "step_count": 4
+  },
+  "source_sha256": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
+  "started_at": "2026-08-03T06:45:15.471600Z",
+  "status": "succeeded",
+  "stderr_log": "/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/flm/steps_4/stderr.log",
+  "stdout_log": "/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/flm/steps_4/stdout.log"
+}

results/smoke/samples/lm1b/flm/steps_4/upstream_hydra/.hydra/overrides.yaml

@@ -0,0 +1,20 @@
+- mode=sample_eval
+- seed=42
+- data=lm1b-wrap
+- model=small
+- model.length=128
+- loader.batch_size=2
+- loader.eval_batch_size=32
+- sampling.num_sample_batches=1
+- sampling.steps=4
+- eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/flm_ckpt/lm1b/lm1b_flm.ckpt
+- trainer.devices=1
+- checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/flm/steps_4/upstream_hydra
+- +wandb.offline=true
+- eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/flm/steps_4/upstream_samples.json
+- algo=flm
+- algo.backbone=dit
+- eval.disable_ema=False
+- sampling.solver=euler
+- algo.double_temb=False
+- sampling.gamma=0.0

results/smoke/samples/lm1b/flm/steps_8/upstream_hydra/dlb_pinned_upstream_main.log

@@ -0,0 +1,2 @@
+[2026-08-03 13:02:06,993][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
+[2026-08-03 14:45:34,155][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.

results/smoke/samples/lm1b/fmlm/steps_16/stderr.log

@@ -0,0 +1,3 @@
+Seed set to 42
+/home/ruizzhao/diffusion_baseline/upstreams/flm/models/dit.py:476: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(enabled=False):

results/smoke/samples/lm1b/fmlm/steps_2/upstream_hydra/dlb_pinned_upstream_main.log

@@ -0,0 +1,2 @@
+[2026-08-03 13:03:40,545][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
+[2026-08-03 14:46:51,874][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.

results/smoke/samples/lm1b/fmlm/steps_4/stdout.log

@@ -0,0 +1,204 @@
+DIT: USE_COMPILE=False
+CONFIG
+├── mode
+│   └── sample_eval                                                             
+├── is_di4c
+│   └── False                                                                   
+├── is_di4c_deterministic
+│   └── True                                                                    
+├── seed
+│   └── 42                                                                      
+├── loader
+│   └── global_batch_size: 512                                                  
+│       eval_global_batch_size: 512                                             
+│       batch_size: 2                                                           
+│       eval_batch_size: 16                                                     
+│       num_workers: 4                                                          
+│       pin_memory: true                                                        
+│                                                                               
+├── sampling
+│   └── predictor: ancestral                                                    
+│       steps: 4                                                                
+│       noise_removal: ancestral                                                
+│       use_float64: true                                                       
+│       p_nucleus: 1.0                                                          
+│       num_sample_batches: 1                                                   
+│       num_sample_log: 2                                                       
+│       semi_ar: false                                                          
+│       stride_length: 1                                                        
+│       num_strides: 1                                                          
+│       num_reflow_samples: 50000                                               
+│       temperature: 1.0                                                        
+│       solver: euler                                                           
+│       gamma: 0.8                                                              
+│                                                                               
+├── training
+│   └── loss_type: flow                                                         
+│       pred_type: x0                                                           
+│       ema: 0.9999                                                             
+│       antithetic_sampling: true                                               
+│       importance_sampling: false                                              
+│       sampling_eps: 0.001                                                     
+│       change_of_variables: false                                              
+│       loss_precision: bf16                                                    
+│       finetune_path: ''                                                       
+│       not_sampling_t: false                                                   
+│       load_ema: false                                                         
+│       t_curriculum_steps: 100000                                              
+│       t_curriculum_min: 0.0                                                   
+│       t_curriculum_max: 1.0                                                   
+│                                                                               
+├── eval
+│   └── checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/official/
+│       disable_ema: false                                                      
+│       ema_decay: null                                                         
+│       compute_generative_perplexity: true                                     
+│       perplexity_batch_size: 8                                                
+│       compute_perplexity_on_sanity: false                                     
+│       gen_ppl_eval_model_name_or_path: gpt2-large                             
+│       generate_samples: true                                                  
+│       generated_samples_path: /home/ruizzhao/diffusion_baseline/results/smoke/
+│                                                                               
+├── optim
+│   └── weight_decay: 0                                                         
+│       lr: 0.0003                                                              
+│       beta1: 0.9                                                              
+│       beta2: 0.999                                                            
+│       eps: 1.0e-08                                                            
+│       ln_tune: none                                                           
+│                                                                               
+├── trainer
+│   └── _target_: lightning.Trainer                                             
+│       accelerator: cuda                                                       
+│       num_nodes: 1                                                            
+│       devices: 1                                                              
+│       accumulate_grad_batches: 256                                            
+│       gradient_clip_val: 1.0                                                  
+│       precision: bf16                                                         
+│       num_sanity_val_steps: 2                                                 
+│       max_steps: 1000000                                                      
+│       log_every_n_steps: 100                                                  
+│       limit_train_batches: 1.0                                                
+│       limit_val_batches: 1.0                                                  
+│       val_check_interval: 5000                                                
+│                                                                               
+├── wandb
+│   └── project: flm                                                            
+│       notes: FLM                                                              
+│       group: null                                                             
+│       job_type: null                                                          
+│       name: null                                                              
+│       id: None_42                                                             
+│       tags:                                                                   
+│       - lm1b                                                                  
+│       - fmlm                                                                  
+│       offline: true                                                           
+│                                                                               
+├── checkpointing
+│   └── save_dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/f
+│       resume_from_ckpt: false                                                 
+│       resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results/smoke/sample
+│                                                                               
+├── callbacks
+│   └── checkpoint_every_n_steps:                                               
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                 
+│         save_top_k: -1                                                        
+│         save_last: false                                                      
+│         dirpath: /home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/
+│         verbose: true                                                         
+│         auto_insert_metric_name: false                                        
+│         every_n_train_steps: 25000                                            
+│       checkpoint_monitor:                                                     
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                 
+│         monitor: val/nll                                                      
+│         mode: min                                                             
+│         save_top_k: 1                                                         
+│         save_last: true                                                       
+│         dirpath: /home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/
+│         filename: best_nll                                                    
+│         auto_insert_metric_name: false                                        
+│         verbose: true                                                         
+│       learning_rate_monitor:                                                  
+│         _target_: lightning.pytorch.callbacks.LearningRateMonitor             
+│         logging_interval: step                                                
+│                                                                               
+├── data
+│   └── train: lm1b                                                             
+│       valid: lm1b                                                             
+│       tokenizer_name_or_path: bert-base-uncased                               
+│       cache_dir: ./cache                                                      
+│       wrap: true                                                              
+│       streaming: false                                                        
+│       insert_train_eos: true                                                  
+│       insert_valid_eos: true                                                  
+│                                                                               
+├── model
+│   └── name: small                                                             
+│       type: ddit                                                              
+│       hidden_size: 768                                                        
+│       cond_dim: 128                                                           
+│       length: 128                                                             
+│       n_blocks: 12                                                            
+│       n_heads: 12                                                             
+│       scale_by_sigma: true                                                    
+│       dropout: 0.1                                                            
+│       tie_word_embeddings: false                                              
+│       vocab_lookup: true                                                      
+│                                                                               
+├── strategy
+│   └── _target_: lightning.pytorch.strategies.DDPStrategy                      
+│       find_unused_parameters: false                                           
+│                                                                               
+├── noise
+│   └── type: log-linear                                                        
+│       parameterization: log-linear                                            
+│       eps: 0                                                                  
+│       denoiser_latent_conditioning: -1                                        
+│       freeze_encoder: false                                                   
+│       freeze_decoder: false                                                   
+│                                                                               
+├── lr_scheduler
+│   └── _target_: transformers.get_constant_schedule_with_warmup                
+│       num_warmup_steps: 2500                                                  
+│                                                                               
+├── prior
+│   └── type: none                                                              
+│       latent_width: 0                                                         
+│       latent_height: 0                                                        
+│                                                                               
+└── algo
+    └── name: fmlm                                                              
+        backbone: dit                                                           
+        parameterization: mean                                                  
+        time_conditioning: true                                                 
+        T: 0                                                                    
+        subs_masking: false                                                     
+        causal_attention: false                                                 
+        t_max: 1.0                                                              
+        t_min: 0.0                                                              
+        double_temb: true                                                       
+        add_boundary: fixed                                                     
+        diagonal_fraction: 0.5                                                  
+        offdiagonal_sampling: uniform_diff                                      
+        set_midpoint: midpoint                                                  
+        boundary_prob: 32                                                       
+        distillation_method: PSD                                                
+        entmax_temp_lsd: 2.0                                                    
+        backprop_entmax_temp_lsd: false                                         
+        learnable_loss_weighting: false                                         
+        use_mse_loss_psd: false                                                 
+        rescale_offdiag_loss_psd: false                                         
+        use_ema_for_psd_target: false                                           
+        use_teacher_for_D_t_lsd: false                                          
+        initialize_student_from_teacher: true                                   
+        teacher_path: ''                                                        
+                                                                                
+[2026-08-03 14:46:58,113][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
+Resuming training from checkpoint...
+generation start: 2026-08-03 14:47:04
+Copying EMA parameters to model
+Sampling with 4 steps
+generation end: 2026-08-03 14:47:06
+Generative perplexity: 100.69593048095703
+Sample entropy: 4.2224016189575195
+Samples saved at: /home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/fmlm/steps_4/upstream_samples.json

results/smoke/samples/lm1b/fmlm/steps_8/run_metadata.json

@@ -0,0 +1,90 @@
+{
+  "checkpoint_lock_id": "recipe:fmlm_lm1b_official_ckpt:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0",
+  "checkpoint_selection": {
+    "output": "checkpoints/official/fmlm_ckpt/lm1b",
+    "recipe": "fmlm_lm1b_official_ckpt",
+    "sampling_checkpoint": "lm1b_fmlm.ckpt",
+    "sampling_config": null,
+    "source": "flm",
+    "source_commit": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
+    "teacher_adapter": null,
+    "teacher_family": "continuous_flm"
+  },
+  "checkpoint_sha256": "b01b03c59150978cdd4865283b71028ec749936c21769eb32bfe2907f38d8918",
+  "checkpoint_teacher_family": "continuous_flm",
+  "command": [
+    "/opt/anaconda3/envs/dlb-flm/bin/python",
+    "-u",
+    "-m",
+    "dlb.adapters.capture",
+    "--upstream-entrypoint=/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py",
+    "--capture-path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/fmlm/steps_8/upstream_token_ids.json",
+    "--",
+    "mode=sample_eval",
+    "seed=42",
+    "data=lm1b-wrap",
+    "model=small",
+    "model.length=128",
+    "loader.batch_size=2",
+    "loader.eval_batch_size=16",
+    "sampling.num_sample_batches=1",
+    "sampling.steps=8",
+    "eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/fmlm_ckpt/lm1b/lm1b_fmlm.ckpt",
+    "trainer.devices=1",
+    "hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/fmlm/steps_8/upstream_hydra",
+    "checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/fmlm/steps_8/upstream_hydra",
+    "+wandb.offline=true",
+    "eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/fmlm/steps_8/upstream_samples.json",
+    "algo=fmlm",
+    "algo.backbone=dit",
+    "eval.disable_ema=False",
+    "sampling.solver=euler",
+    "algo.double_temb=True",
+    "algo.learnable_loss_weighting=False",
+    "sampling.gamma=0.8"
+  ],
+  "command_sha256": "76d0a5b7eb54003afbdc710751ab6af2501913b41f3f437dd6ad3e7a092ae5ea",
+  "config_sha256": "206d75b3b7cf3fb35844c053ae552537d7471b93bf29756dafc1ac616b096994",
+  "environment": {
+    "cuda_version": null,
+    "cuda_visible_devices": "1",
+    "hostname": "DS210029",
+    "platform": "Linux-5.15.0-186-generic-x86_64-with-glibc2.35",
+    "python": "3.11.15",
+    "torch_version": "2.5.1"
+  },
+  "finished_at": "2026-08-03T06:47:35.623464Z",
+  "identity": {
+    "adapter_identity": "dlb.adapters.flm:v2|dlb.adapters.flm:FLMAdapter|c3a6a4573200fecf38a47fc6e7bc78254ca362fff3b96079dc2efa9f8e27aff0",
+    "checkpoint_lock_id": "recipe:fmlm_lm1b_official_ckpt:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0",
+    "checkpoint_selection": {
+      "output": "checkpoints/official/fmlm_ckpt/lm1b",
+      "recipe": "fmlm_lm1b_official_ckpt",
+      "sampling_checkpoint": "lm1b_fmlm.ckpt",
+      "sampling_config": null,
+      "source": "flm",
+      "source_commit": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
+      "teacher_adapter": null,
+      "teacher_family": "continuous_flm"
+    },
+    "checkpoint_sha256": "b01b03c59150978cdd4865283b71028ec749936c21769eb32bfe2907f38d8918",
+    "checkpoint_teacher_family": "continuous_flm",
+    "command_sha256": "76d0a5b7eb54003afbdc710751ab6af2501913b41f3f437dd6ad3e7a092ae5ea",
+    "config_sha256": "206d75b3b7cf3fb35844c053ae552537d7471b93bf29756dafc1ac616b096994",
+    "dataset_id": "lm1b",
+    "device": null,
+    "environment": "dlb-flm",
+    "model_id": "fmlm",
+    "results_root": "/home/ruizzhao/diffusion_baseline/results/smoke",
+    "run_id": "fmlm-lm1b-steps-8",
+    "sample_count": 1,
+    "seed": 42,
+    "source_sha256": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
+    "step_count": 8
+  },
+  "source_sha256": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
+  "started_at": "2026-08-03T06:47:11.538062Z",
+  "status": "succeeded",
+  "stderr_log": "/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/fmlm/steps_8/stderr.log",
+  "stdout_log": "/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/fmlm/steps_8/stdout.log"
+}

results/smoke/samples/lm1b/fmlm/steps_8/upstream_hydra/.hydra/overrides.yaml

@@ -0,0 +1,21 @@
+- mode=sample_eval
+- seed=42
+- data=lm1b-wrap
+- model=small
+- model.length=128
+- loader.batch_size=2
+- loader.eval_batch_size=16
+- sampling.num_sample_batches=1
+- sampling.steps=8
+- eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/fmlm_ckpt/lm1b/lm1b_fmlm.ckpt
+- trainer.devices=1
+- checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/fmlm/steps_8/upstream_hydra
+- +wandb.offline=true
+- eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/lm1b/fmlm/steps_8/upstream_samples.json
+- algo=fmlm
+- algo.backbone=dit
+- eval.disable_ema=False
+- sampling.solver=euler
+- algo.double_temb=True
+- algo.learnable_loss_weighting=False
+- sampling.gamma=0.8

results/smoke/samples/lm1b/fmlm/steps_8/upstream_samples.json

@@ -0,0 +1,22 @@
+{
+    "generative_ppl": 88.23694610595703,
+    "entropy": 4.21762228012085,
+    "generated_seqs": [
+        "[CLS] was not an easy decision. [CLS] if things happen in the house and at a time, make your change to $ 18, 000 and the owner can change his or more one or two. [CLS] maybe not his exact do, but if the longer term china is down to it, and who is made \" good citizens \" and have what else to think as that would be? [CLS] of their day, if they just say they are, not do not today. [CLS] in the first half of the house bill being cut up by bartz's, reid said he would increase his $ 1. 1 billion budget is three to 16 [CLS]",
+        "[CLS] a single vote of interest - - their home of the company that owns it, and a $ 2 billion ( involved in some of the first ), or at least on the capital cabo deli. [CLS] this is about news - 2, attacks and white - collar mp's, its story to the post, and who may not much understand more. [CLS] how would mccain's reach and out of a private group would i end? [CLS] a freedom of information has been not out of on the case, until after about 20 years. [CLS] this period over final would be extended by just over five years. [CLS] department for [CLS]",
+        "[CLS] days are not used for the \" buy \" the second floor, the networkest and currently home to 8 million in germany. [CLS] \" more than 2 or 1 billion people in the national forest, when they go up, \" wrotepan wrote. [CLS] the products went by in two separate later, and line - up selection \" has not an issue. \" [CLS] there's something to cheer about in that warm - up interview last week. [CLS] last night, i was presented with a problem. [CLS] what abrieu said there every time was in a country where he did not do not until his third term, when president w [CLS]",
+        "[CLS] ahead, how bad in iraq was all over 60 years, how would you do so far with it. [CLS] obama spending is based on the bank in mind they are still hearing news about a recession as much as an election. [CLS] the party lies near the state with the same real estate market and economic benefits, as potential will address the issue. [CLS] on five occasions, peter's parents and his seven children remain and his mother were all saying alive without warrants. \" [CLS] he called the 8 - year - old but his story is being set from me. [CLS] the down last week and is on its loss. [CLS] the [CLS]",
+        "[CLS] of the country for barack obama's children. [CLS] \" i think in three nights in london and playing well, think of it until there are guys who don't, don't turn out ; you can find out who is as much more or more famous as people partying. [CLS] those still have to do it over once, when it said even if they show the government's giving people more together effectively, but why we're seeing that today all over young children in recent months, is our belief that men wouldn't change anything if there was nothing to blame. [CLS] the victims all were linked to 15 [CLS]",
+        "[CLS] here - - the first day, in the middle of world new year, that might demand the country. [CLS] but my husband is always special, so it may be that they have the right to do it really quickly and to get through to us on the ground. [CLS] as for the latter, with a bit less detail that was \" one of us by other factors in how the group was structured as a bank, \" he added. [CLS] the treasury secretary said that business continuity was a factor that a backlash against barack obama, who would look at government or how to run or not before the campaign begins next year. [CLS] he will [CLS]",
+        "[CLS]. [CLS] hee - chien was fourth with the one - second of a - - her old team and colleagues from her own testers on a lot of paper - - and fourth place. [CLS] the judge says it was last seen by this man as \" serious, \" but she as \" this is true. \" [CLS] the selectors have been told either since or monday now despite the man under more than a minute is believed to be in position to play in the action, not especially when other youngsters are out in line at a time. [CLS] they are out, but three and still. [CLS] the first book first in the [CLS]",
+        "[CLS] 000. [CLS] even if they stayed at home, she and others would spend a few extra cash to attend a college degree for four years, and make up for losses at graduation.'[CLS] the region is where the court had been asked to court. [CLS] so you are more than britain's main stock at peak times. [CLS] when the house wrote last week to the people that help is likely to change, rather than out of economic necessity. [CLS] \" he can't out of place, and the game is not only to go, it's just a tough man he said he can be the best rusher to have [CLS]",
+        "[CLS] money. [CLS] for it, we're getting back late last year, have cut this year's t something else - - obama will be my nominee. [CLS] his most obvious of man and a decision has been to keep up a while, once the week is also the same - but we do you have now to get a case of time to death and allow it to be asked early on a week about his or any other body on the planet which they all says they are the cowards to talk to. [CLS] for them some reason, it does not ( which are only with yet new, national parliaments which have air [CLS]",
+        "[CLS] behind to any changes to the decision until 2011. [CLS] it like anything from congress, it is free to keep the gop \" that has raised at least $ 10 million. [CLS] obama? 10 world body : you only do you 5. 8 % of adults think that government intervention is going to an interest and just two in five each of the uk oil spills power, it is a great schoolchild... at people waiting for time to run that question the first time. [CLS] all of the complaints belong to a school from the group, which has been reported on the club by friends of, but declined to make for [CLS]",
+        "[CLS] they hold the public rates up - or - - the most difficult of the year. [CLS] leaders at it had demanded the remains of the holy landry government. [CLS] some fivex could be the one missing man in the car with a further over 4, 000, who was killed. [CLS] that's just the perfect \" free \" party for a woman at home and try to feed their children with toys ( so, as is already happening, anyone? ) ) [CLS] is also about boos you, it seems to be about playing every half the last two minutes throughout the country, and will be outcasts. [CLS] the [CLS]",
+        "[CLS] well \" - - but will last about five years. [CLS] in the days at least, it's no lot less than we make to tell you. [CLS] those said they hope many will use the government's military muscle for at least one past and present and deployments that focus on the one - to - one u. s. approach to afghanistan. [CLS] their only since been kitted out with each seat on the first - and - fourth drive which will give the staff the same time, as the home was, it has an event room and seven or more as a front now. [CLS] \" that's a far [CLS]",
+        "[CLS] set off the sunday day. [CLS] as the half - pay is, many voters are with his campaign finance yesterday. [CLS] but even china has to become a land of the \" yes he's telling you, \" a world of bush as overfishing. [CLS] unlike big ben's, \" here's not it for the future oscar ], it should have a good line up would come with. \" [CLS] the rules see : they don't have to pay the company's debt ; instead of going back home they have a new role, and you have little choice that you would care. [CLS] \" it'[CLS]",
+        "[CLS] city. [CLS] another change, the view of the political media, is that it is a panacea, mr. spitzer has it. [CLS] however, although him could have been good for europe, like his one on 18 and on other 4 he was still 3 - even while he lost out since the beginning of the first election as it stands. [CLS] \" if only the test has been won by way back then you think that if there is a lot of 500 any one that will not include the people to think about - - and not draw the line in, \" she told ( d ). [CLS] the iraqi government and [CLS]",
+        "[CLS] and it was two months ago mr cameron's son, who had been up for in 17 years. [CLS] the days after \" idol \" added forced advised obama to keep going on the day before tonight's presidential election, it died with her other home. [CLS] some said that they plan to under the threat of parliament by the prime minister, while they protested against the government and won, witnesses said. [CLS] \" the selectors will play a one start, i know it's still tough. [CLS] that is brown's new life, and often victory for general, \" in the statement. [CLS] security officials and at least [CLS]",
+        "[CLS] can't and the problem were their coverage of the new york giants sunday play was poor. [CLS] wall street sources in china said many percent have had so far from the bottom up a few days earlier. [CLS] or not on monday to fund the world that the people are ready to make this problem happen. [CLS] but without the president's knowledge one was aware of that brief one - up but not top officials said they had also been surprised at the situation in force. [CLS] no one got to turn back. [CLS] but the same folks in london's just things are another step move. [CLS] the government gave a report on israel [CLS]"
+    ]
+}

results/smoke/samples/lm1b/fmlm/steps_8/upstream_token_ids.json

@@ -0,0 +1,2149 @@
+{
+  "samples": [
+    {
+      "sample_id": 0,
+      "text": "[CLS] was not an easy decision. [CLS] if things happen in the house and at a time, make your change to $ 18, 000 and the owner can change his or more one or two. [CLS] maybe not his exact do, but if the longer term china is down to it, and who is made \" good citizens \" and have what else to think as that would be? [CLS] of their day, if they just say they are, not do not today. [CLS] in the first half of the house bill being cut up by bartz's, reid said he would increase his $ 1. 1 billion budget is three to 16 [CLS]",
+      "token_ids": [
+        101,
+        2001,
+        2025,
+        2019,
+        3733,
+        3247,
+        1012,
+        101,
+        2065,
+        2477,
+        4148,
+        1999,
+        1996,
+        2160,
+        1998,
+        2012,
+        1037,
+        2051,
+        1010,
+        2191,
+        2115,
+        2689,
+        2000,
+        1002,
+        2324,
+        1010,
+        2199,
+        1998,
+        1996,
+        3954,
+        2064,
+        2689,
+        2010,
+        2030,
+        2062,
+        2028,
+        2030,
+        2048,
+        1012,
+        101,
+        2672,
+        2025,
+        2010,
+        6635,
+        2079,
+        1010,
+        2021,
+        2065,
+        1996,
+        2936,
+        2744,
+        2859,
+        2003,
+        2091,
+        2000,
+        2009,
+        1010,
+        1998,
+        2040,
+        2003,
+        2081,
+        1000,
+        2204,
+        4480,
+        1000,
+        1998,
+        2031,
+        2054,
+        2842,
+        2000,
+        2228,
+        2004,
+        2008,
+        2052,
+        2022,
+        1029,
+        101,
+        1997,
+        2037,
+        2154,
+        1010,
+        2065,
+        2027,
+        2074,
+        2360,
+        2027,
+        2024,
+        1010,
+        2025,
+        2079,
+        2025,
+        2651,
+        1012,
+        101,
+        1999,
+        1996,
+        2034,
+        2431,
+        1997,
+        1996,
+        2160,
+        3021,
+        2108,
+        3013,
+        2039,
+        2011,
+        12075,
+        2480,
+        1005,
+        1055,
+        1010,
+        9027,
+        2056,
+        2002,
+        2052,
+        3623,
+        2010,
+        1002,
+        1015,
+        1012,
+        1015,
+        4551,
+        5166,
+        2003,
+        2093,
+        2000,
+        2385,
+        101
+      ]
+    },
+    {
+      "sample_id": 1,
+      "text": "[CLS] a single vote of interest - - their home of the company that owns it, and a $ 2 billion ( involved in some of the first ), or at least on the capital cabo deli. [CLS] this is about news - 2, attacks and white - collar mp's, its story to the post, and who may not much understand more. [CLS] how would mccain's reach and out of a private group would i end? [CLS] a freedom of information has been not out of on the case, until after about 20 years. [CLS] this period over final would be extended by just over five years. [CLS] department for [CLS]",
+      "token_ids": [
+        101,
+        1037,
+        2309,
+        3789,
+        1997,
+        3037,
+        1011,
+        1011,
+        2037,
+        2188,
+        1997,
+        1996,
+        2194,
+        2008,
+        8617,
+        2009,
+        1010,
+        1998,
+        1037,
+        1002,
+        1016,
+        4551,
+        1006,
+        2920,
+        1999,
+        2070,
+        1997,
+        1996,
+        2034,
+        1007,
+        1010,
+        2030,
+        2012,
+        2560,
+        2006,
+        1996,
+        3007,
+        9298,
+        2080,
+        3972,
+        2072,
+        1012,
+        101,
+        2023,
+        2003,
+        2055,
+        2739,
+        1011,
+        1016,
+        1010,
+        4491,
+        1998,
+        2317,
+        1011,
+        9127,
+        6131,
+        1005,
+        1055,
+        1010,
+        2049,
+        2466,
+        2000,
+        1996,
+        2695,
+        1010,
+        1998,
+        2040,
+        2089,
+        2025,
+        2172,
+        3305,
+        2062,
+        1012,
+        101,
+        2129,
+        2052,
+        19186,
+        1005,
+        1055,
+        3362,
+        1998,
+        2041,
+        1997,
+        1037,
+        2797,
+        2177,
+        2052,
+        1045,
+        2203,
+        1029,
+        101,
+        1037,
+        4071,
+        1997,
+        2592,
+        2038,
+        2042,
+        2025,
+        2041,
+        1997,
+        2006,
+        1996,
+        2553,
+        1010,
+        2127,
+        2044,
+        2055,
+        2322,
+        2086,
+        1012,
+        101,
+        2023,
+        2558,
+        2058,
+        2345,
+        2052,
+        2022,
+        3668,
+        2011,
+        2074,
+        2058,
+        2274,
+        2086,
+        1012,
+        101,
+        2533,
+        2005,
+        101
+      ]
+    },
+    {
+      "sample_id": 2,
+      "text": "[CLS] days are not used for the \" buy \" the second floor, the networkest and currently home to 8 million in germany. [CLS] \" more than 2 or 1 billion people in the national forest, when they go up, \" wrotepan wrote. [CLS] the products went by in two separate later, and line - up selection \" has not an issue. \" [CLS] there's something to cheer about in that warm - up interview last week. [CLS] last night, i was presented with a problem. [CLS] what abrieu said there every time was in a country where he did not do not until his third term, when president w [CLS]",
+      "token_ids": [
+        101,
+        2420,
+        2024,
+        2025,
+        2109,
+        2005,
+        1996,
+        1000,
+        4965,
+        1000,
+        1996,
+        2117,
+        2723,
+        1010,
+        1996,
+        2897,
+        4355,
+        1998,
+        2747,
+        2188,
+        2000,
+        1022,
+        2454,
+        1999,
+        2762,
+        1012,
+        101,
+        1000,
+        2062,
+        2084,
+        1016,
+        2030,
+        1015,
+        4551,
+        2111,
+        1999,
+        1996,
+        2120,
+        3224,
+        1010,
+        2043,
+        2027,
+        2175,
+        2039,
+        1010,
+        1000,
+        2626,
+        9739,
+        2626,
+        1012,
+        101,
+        1996,
+        3688,
+        2253,
+        2011,
+        1999,
+        2048,
+        3584,
+        2101,
+        1010,
+        1998,
+        2240,
+        1011,
+        2039,
+        4989,
+        1000,
+        2038,
+        2025,
+        2019,
+        3277,
+        1012,
+        1000,
+        101,
+        2045,
+        1005,
+        1055,
+        2242,
+        2000,
+        15138,
+        2055,
+        1999,
+        2008,
+        4010,
+        1011,
+        2039,
+        4357,
+        2197,
+        2733,
+        1012,
+        101,
+        2197,
+        2305,
+        1010,
+        1045,
+        2001,
+        3591,
+        2007,
+        1037,
+        3291,
+        1012,
+        101,
+        2054,
+        11113,
+        7373,
+        2226,
+        2056,
+        2045,
+        2296,
+        2051,
+        2001,
+        1999,
+        1037,
+        2406,
+        2073,
+        2002,
+        2106,
+        2025,
+        2079,
+        2025,
+        2127,
+        2010,
+        2353,
+        2744,
+        1010,
+        2043,
+        2343,
+        1059,
+        101
+      ]
+    },
+    {
+      "sample_id": 3,
+      "text": "[CLS] ahead, how bad in iraq was all over 60 years, how would you do so far with it. [CLS] obama spending is based on the bank in mind they are still hearing news about a recession as much as an election. [CLS] the party lies near the state with the same real estate market and economic benefits, as potential will address the issue. [CLS] on five occasions, peter's parents and his seven children remain and his mother were all saying alive without warrants. \" [CLS] he called the 8 - year - old but his story is being set from me. [CLS] the down last week and is on its loss. [CLS] the [CLS]",
+      "token_ids": [
+        101,
+        3805,
+        1010,
+        2129,
+        2919,
+        1999,
+        5712,
+        2001,
+        2035,
+        2058,
+        3438,
+        2086,
+        1010,
+        2129,
+        2052,
+        2017,
+        2079,
+        2061,
+        2521,
+        2007,
+        2009,
+        1012,
+        101,
+        8112,
+        5938,
+        2003,
+        2241,
+        2006,
+        1996,
+        2924,
+        1999,
+        2568,
+        2027,
+        2024,
+        2145,
+        4994,
+        2739,
+        2055,
+        1037,
+        19396,
+        2004,
+        2172,
+        2004,
+        2019,
+        2602,
+        1012,
+        101,
+        1996,
+        2283,
+        3658,
+        2379,
+        1996,
+        2110,
+        2007,
+        1996,
+        2168,
+        2613,
+        3776,
+        3006,
+        1998,
+        3171,
+        6666,
+        1010,
+        2004,
+        4022,
+        2097,
+        4769,
+        1996,
+        3277,
+        1012,
+        101,
+        2006,
+        2274,
+        6642,
+        1010,
+        2848,
+        1005,
+        1055,
+        3008,
+        1998,
+        2010,
+        2698,
+        2336,
+        3961,
+        1998,
+        2010,
+        2388,
+        2020,
+        2035,
+        3038,
+        4142,
+        2302,
+        10943,
+        2015,
+        1012,
+        1000,
+        101,
+        2002,
+        2170,
+        1996,
+        1022,
+        1011,
+        2095,
+        1011,
+        2214,
+        2021,
+        2010,
+        2466,
+        2003,
+        2108,
+        2275,
+        2013,
+        2033,
+        1012,
+        101,
+        1996,
+        2091,
+        2197,
+        2733,
+        1998,
+        2003,
+        2006,
+        2049,
+        3279,
+        1012,
+        101,
+        1996,
+        101
+      ]
+    },
+    {
+      "sample_id": 4,
+      "text": "[CLS] of the country for barack obama's children. [CLS] \" i think in three nights in london and playing well, think of it until there are guys who don't, don't turn out ; you can find out who is as much more or more famous as people partying. [CLS] those still have to do it over once, when it said even if they show the government's giving people more together effectively, but why we're seeing that today all over young children in recent months, is our belief that men wouldn't change anything if there was nothing to blame. [CLS] the victims all were linked to 15 [CLS]",
+      "token_ids": [
+        101,
+        1997,
+        1996,
+        2406,
+        2005,
+        13857,
+        8112,
+        1005,
+        1055,
+        2336,
+        1012,
+        101,
+        1000,
+        1045,
+        2228,
+        1999,
+        2093,
+        6385,
+        1999,
+        2414,
+        1998,
+        2652,
+        2092,
+        1010,
+        2228,
+        1997,
+        2009,
+        2127,
+        2045,
+        2024,
+        4364,
+        2040,
+        2123,
+        1005,
+        1056,
+        1010,
+        2123,
+        1005,
+        1056,
+        2735,
+        2041,
+        1025,
+        2017,
+        2064,
+        2424,
+        2041,
+        2040,
+        2003,
+        2004,
+        2172,
+        2062,
+        2030,
+        2062,
+        3297,
+        2004,
+        2111,
+        2283,
+        2075,
+        1012,
+        101,
+        2216,
+        2145,
+        2031,
+        2000,
+        2079,
+        2009,
+        2058,
+        2320,
+        1010,
+        2043,
+        2009,
+        2056,
+        2130,
+        2065,
+        2027,
+        2265,
+        1996,
+        2231,
+        1005,
+        1055,
+        3228,
+        2111,
+        2062,
+        2362,
+        6464,
+        1010,
+        2021,
+        2339,
+        2057,
+        1005,
+        2128,
+        3773,
+        2008,
+        2651,
+        2035,
+        2058,
+        2402,
+        2336,
+        1999,
+        3522,
+        2706,
+        1010,
+        2003,
+        2256,
+        6772,
+        2008,
+        2273,
+        2876,
+        1005,
+        1056,
+        2689,
+        2505,
+        2065,
+        2045,
+        2001,
+        2498,
+        2000,
+        7499,
+        1012,
+        101,
+        1996,
+        5694,
+        2035,
+        2020,
+        5799,
+        2000,
+        2321,
+        101
+      ]
+    },
+    {
+      "sample_id": 5,
+      "text": "[CLS] here - - the first day, in the middle of world new year, that might demand the country. [CLS] but my husband is always special, so it may be that they have the right to do it really quickly and to get through to us on the ground. [CLS] as for the latter, with a bit less detail that was \" one of us by other factors in how the group was structured as a bank, \" he added. [CLS] the treasury secretary said that business continuity was a factor that a backlash against barack obama, who would look at government or how to run or not before the campaign begins next year. [CLS] he will [CLS]",
+      "token_ids": [
+        101,
+        2182,
+        1011,
+        1011,
+        1996,
+        2034,
+        2154,
+        1010,
+        1999,
+        1996,
+        2690,
+        1997,
+        2088,
+        2047,
+        2095,
+        1010,
+        2008,
+        2453,
+        5157,
+        1996,
+        2406,
+        1012,
+        101,
+        2021,
+        2026,
+        3129,
+        2003,
+        2467,
+        2569,
+        1010,
+        2061,
+        2009,
+        2089,
+        2022,
+        2008,
+        2027,
+        2031,
+        1996,
+        2157,
+        2000,
+        2079,
+        2009,
+        2428,
+        2855,
+        1998,
+        2000,
+        2131,
+        2083,
+        2000,
+        2149,
+        2006,
+        1996,
+        2598,
+        1012,
+        101,
+        2004,
+        2005,
+        1996,
+        3732,
+        1010,
+        2007,
+        1037,
+        2978,
+        2625,
+        6987,
+        2008,
+        2001,
+        1000,
+        2028,
+        1997,
+        2149,
+        2011,
+        2060,
+        5876,
+        1999,
+        2129,
+        1996,
+        2177,
+        2001,
+        14336,
+        2004,
+        1037,
+        2924,
+        1010,
+        1000,
+        2002,
+        2794,
+        1012,
+        101,
+        1996,
+        9837,
+        3187,
+        2056,
+        2008,
+        2449,
+        13717,
+        2001,
+        1037,
+        5387,
+        2008,
+        1037,
+        25748,
+        2114,
+        13857,
+        8112,
+        1010,
+        2040,
+        2052,
+        2298,
+        2012,
+        2231,
+        2030,
+        2129,
+        2000,
+        2448,
+        2030,
+        2025,
+        2077,
+        1996,
+        3049,
+        4269,
+        2279,
+        2095,
+        1012,
+        101,
+        2002,
+        2097,
+        101
+      ]
+    },
+    {
+      "sample_id": 6,
+      "text": "[CLS]. [CLS] hee - chien was fourth with the one - second of a - - her old team and colleagues from her own testers on a lot of paper - - and fourth place. [CLS] the judge says it was last seen by this man as \" serious, \" but she as \" this is true. \" [CLS] the selectors have been told either since or monday now despite the man under more than a minute is believed to be in position to play in the action, not especially when other youngsters are out in line at a time. [CLS] they are out, but three and still. [CLS] the first book first in the [CLS]",
+      "token_ids": [
+        101,
+        1012,
+        101,
+        18235,
+        1011,
+        9610,
+        2368,
+        2001,
+        2959,
+        2007,
+        1996,
+        2028,
+        1011,
+        2117,
+        1997,
+        1037,
+        1011,
+        1011,
+        2014,
+        2214,
+        2136,
+        1998,
+        8628,
+        2013,
+        2014,
+        2219,
+        3231,
+        2545,
+        2006,
+        1037,
+        2843,
+        1997,
+        3259,
+        1011,
+        1011,
+        1998,
+        2959,
+        2173,
+        1012,
+        101,
+        1996,
+        3648,
+        2758,
+        2009,
+        2001,
+        2197,
+        2464,
+        2011,
+        2023,
+        2158,
+        2004,
+        1000,
+        3809,
+        1010,
+        1000,
+        2021,
+        2016,
+        2004,
+        1000,
+        2023,
+        2003,
+        2995,
+        1012,
+        1000,
+        101,
+        1996,
+        27000,
+        2015,
+        2031,
+        2042,
+        2409,
+        2593,
+        2144,
+        2030,
+        6928,
+        2085,
+        2750,
+        1996,
+        2158,
+        2104,
+        2062,
+        2084,
+        1037,
+        3371,
+        2003,
+        3373,
+        2000,
+        2022,
+        1999,
+        2597,
+        2000,
+        2377,
+        1999,
+        1996,
+        2895,
+        1010,
+        2025,
+        2926,
+        2043,
+        2060,
+        2402,
+        15608,
+        2024,
+        2041,
+        1999,
+        2240,
+        2012,
+        1037,
+        2051,
+        1012,
+        101,
+        2027,
+        2024,
+        2041,
+        1010,
+        2021,
+        2093,
+        1998,
+        2145,
+        1012,
+        101,
+        1996,
+        2034,
+        2338,
+        2034,
+        1999,
+        1996,
+        101
+      ]
+    },
+    {
+      "sample_id": 7,
+      "text": "[CLS] 000. [CLS] even if they stayed at home, she and others would spend a few extra cash to attend a college degree for four years, and make up for losses at graduation.'[CLS] the region is where the court had been asked to court. [CLS] so you are more than britain's main stock at peak times. [CLS] when the house wrote last week to the people that help is likely to change, rather than out of economic necessity. [CLS] \" he can't out of place, and the game is not only to go, it's just a tough man he said he can be the best rusher to have [CLS]",
+      "token_ids": [
+        101,
+        2199,
+        1012,
+        101,
+        2130,
+        2065,
+        2027,
+        4370,
+        2012,
+        2188,
+        1010,
+        2016,
+        1998,
+        2500,
+        2052,
+        5247,
+        1037,
+        2261,
+        4469,
+        5356,
+        2000,
+        5463,
+        1037,
+        2267,
+        3014,
+        2005,
+        2176,
+        2086,
+        1010,
+        1998,
+        2191,
+        2039,
+        2005,
+        6409,
+        2012,
+        7665,
+        1012,
+        1005,
+        101,
+        1996,
+        2555,
+        2003,
+        2073,
+        1996,
+        2457,
+        2018,
+        2042,
+        2356,
+        2000,
+        2457,
+        1012,
+        101,
+        2061,
+        2017,
+        2024,
+        2062,
+        2084,
+        3725,
+        1005,
+        1055,
+        2364,
+        4518,
+        2012,
+        4672,
+        2335,
+        1012,
+        101,
+        2043,
+        1996,
+        2160,
+        2626,
+        2197,
+        2733,
+        2000,
+        1996,
+        2111,
+        2008,
+        2393,
+        2003,
+        3497,
+        2000,
+        2689,
+        1010,
+        2738,
+        2084,
+        2041,
+        1997,
+        3171,
+        13185,
+        1012,
+        101,
+        1000,
+        2002,
+        2064,
+        1005,
+        1056,
+        2041,
+        1997,
+        2173,
+        1010,
+        1998,
+        1996,
+        2208,
+        2003,
+        2025,
+        2069,
+        2000,
+        2175,
+        1010,
+        2009,
+        1005,
+        1055,
+        2074,
+        1037,
+        7823,
+        2158,
+        2002,
+        2056,
+        2002,
+        2064,
+        2022,
+        1996,
+        2190,
+        5481,
+        2121,
+        2000,
+        2031,
+        101
+      ]
+    },
+    {
+      "sample_id": 8,
+      "text": "[CLS] money. [CLS] for it, we're getting back late last year, have cut this year's t something else - - obama will be my nominee. [CLS] his most obvious of man and a decision has been to keep up a while, once the week is also the same - but we do you have now to get a case of time to death and allow it to be asked early on a week about his or any other body on the planet which they all says they are the cowards to talk to. [CLS] for them some reason, it does not ( which are only with yet new, national parliaments which have air [CLS]",
+      "token_ids": [
+        101,
+        2769,
+        1012,
+        101,
+        2005,
+        2009,
+        1010,
+        2057,
+        1005,
+        2128,
+        2893,
+        2067,
+        2397,
+        2197,
+        2095,
+        1010,
+        2031,
+        3013,
+        2023,
+        2095,
+        1005,
+        1055,
+        1056,
+        2242,
+        2842,
+        1011,
+        1011,
+        8112,
+        2097,
+        2022,
+        2026,
+        9773,
+        1012,
+        101,
+        2010,
+        2087,
+        5793,
+        1997,
+        2158,
+        1998,
+        1037,
+        3247,
+        2038,
+        2042,
+        2000,
+        2562,
+        2039,
+        1037,
+        2096,
+        1010,
+        2320,
+        1996,
+        2733,
+        2003,
+        2036,
+        1996,
+        2168,
+        1011,
+        2021,
+        2057,
+        2079,
+        2017,
+        2031,
+        2085,
+        2000,
+        2131,
+        1037,
+        2553,
+        1997,
+        2051,
+        2000,
+        2331,
+        1998,
+        3499,
+        2009,
+        2000,
+        2022,
+        2356,
+        2220,
+        2006,
+        1037,
+        2733,
+        2055,
+        2010,
+        2030,
+        2151,
+        2060,
+        2303,
+        2006,
+        1996,
+        4774,
+        2029,
+        2027,
+        2035,
+        2758,
+        2027,
+        2024,
+        1996,
+        16592,
+        2015,
+        2000,
+        2831,
+        2000,
+        1012,
+        101,
+        2005,
+        2068,
+        2070,
+        3114,
+        1010,
+        2009,
+        2515,
+        2025,
+        1006,
+        2029,
+        2024,
+        2069,
+        2007,
+        2664,
+        2047,
+        1010,
+        2120,
+        3323,
+        2015,
+        2029,
+        2031,
+        2250,
+        101
+      ]
+    },
+    {
+      "sample_id": 9,
+      "text": "[CLS] behind to any changes to the decision until 2011. [CLS] it like anything from congress, it is free to keep the gop \" that has raised at least $ 10 million. [CLS] obama? 10 world body : you only do you 5. 8 % of adults think that government intervention is going to an interest and just two in five each of the uk oil spills power, it is a great schoolchild... at people waiting for time to run that question the first time. [CLS] all of the complaints belong to a school from the group, which has been reported on the club by friends of, but declined to make for [CLS]",
+      "token_ids": [
+        101,
+        2369,
+        2000,
+        2151,
+        3431,
+        2000,
+        1996,
+        3247,
+        2127,
+        2249,
+        1012,
+        101,
+        2009,
+        2066,
+        2505,
+        2013,
+        3519,
+        1010,
+        2009,
+        2003,
+        2489,
+        2000,
+        2562,
+        1996,
+        2175,
+        2361,
+        1000,
+        2008,
+        2038,
+        2992,
+        2012,
+        2560,
+        1002,
+        2184,
+        2454,
+        1012,
+        101,
+        8112,
+        1029,
+        2184,
+        2088,
+        2303,
+        1024,
+        2017,
+        2069,
+        2079,
+        2017,
+        1019,
+        1012,
+        1022,
+        1003,
+        1997,
+        6001,
+        2228,
+        2008,
+        2231,
+        8830,
+        2003,
+        2183,
+        2000,
+        2019,
+        3037,
+        1998,
+        2074,
+        2048,
+        1999,
+        2274,
+        2169,
+        1997,
+        1996,
+        2866,
+        3514,
+        14437,
+        2015,
+        2373,
+        1010,
+        2009,
+        2003,
+        1037,
+        2307,
+        2082,
+        19339,
+        1012,
+        1012,
+        1012,
+        2012,
+        2111,
+        3403,
+        2005,
+        2051,
+        2000,
+        2448,
+        2008,
+        3160,
+        1996,
+        2034,
+        2051,
+        1012,
+        101,
+        2035,
+        1997,
+        1996,
+        10821,
+        7141,
+        2000,
+        1037,
+        2082,
+        2013,
+        1996,
+        2177,
+        1010,
+        2029,
+        2038,
+        2042,
+        2988,
+        2006,
+        1996,
+        2252,
+        2011,
+        2814,
+        1997,
+        1010,
+        2021,
+        6430,
+        2000,
+        2191,
+        2005,
+        101
+      ]
+    },
+    {
+      "sample_id": 10,
+      "text": "[CLS] they hold the public rates up - or - - the most difficult of the year. [CLS] leaders at it had demanded the remains of the holy landry government. [CLS] some fivex could be the one missing man in the car with a further over 4, 000, who was killed. [CLS] that's just the perfect \" free \" party for a woman at home and try to feed their children with toys ( so, as is already happening, anyone? ) ) [CLS] is also about boos you, it seems to be about playing every half the last two minutes throughout the country, and will be outcasts. [CLS] the [CLS]",
+      "token_ids": [
+        101,
+        2027,
+        2907,
+        1996,
+        2270,
+        6165,
+        2039,
+        1011,
+        2030,
+        1011,
+        1011,
+        1996,
+        2087,
+        3697,
+        1997,
+        1996,
+        2095,
+        1012,
+        101,
+        4177,
+        2012,
+        2009,
+        2018,
+        6303,
+        1996,
+        3464,
+        1997,
+        1996,
+        4151,
+        2455,
+        2854,
+        2231,
+        1012,
+        101,
+        2070,
+        2274,
+        2595,
+        2071,
+        2022,
+        1996,
+        2028,
+        4394,
+        2158,
+        1999,
+        1996,
+        2482,
+        2007,
+        1037,
+        2582,
+        2058,
+        1018,
+        1010,
+        2199,
+        1010,
+        2040,
+        2001,
+        2730,
+        1012,
+        101,
+        2008,
+        1005,
+        1055,
+        2074,
+        1996,
+        3819,
+        1000,
+        2489,
+        1000,
+        2283,
+        2005,
+        1037,
+        2450,
+        2012,
+        2188,
+        1998,
+        3046,
+        2000,
+        5438,
+        2037,
+        2336,
+        2007,
+        10899,
+        1006,
+        2061,
+        1010,
+        2004,
+        2003,
+        2525,
+        6230,
+        1010,
+        3087,
+        1029,
+        1007,
+        1007,
+        101,
+        2003,
+        2036,
+        2055,
+        22017,
+        2015,
+        2017,
+        1010,
+        2009,
+        3849,
+        2000,
+        2022,
+        2055,
+        2652,
+        2296,
+        2431,
+        1996,
+        2197,
+        2048,
+        2781,
+        2802,
+        1996,
+        2406,
+        1010,
+        1998,
+        2097,
+        2022,
+        2041,
+        10526,
+        2015,
+        1012,
+        101,
+        1996,
+        101
+      ]
+    },
+    {
+      "sample_id": 11,
+      "text": "[CLS] well \" - - but will last about five years. [CLS] in the days at least, it's no lot less than we make to tell you. [CLS] those said they hope many will use the government's military muscle for at least one past and present and deployments that focus on the one - to - one u. s. approach to afghanistan. [CLS] their only since been kitted out with each seat on the first - and - fourth drive which will give the staff the same time, as the home was, it has an event room and seven or more as a front now. [CLS] \" that's a far [CLS]",
+      "token_ids": [
+        101,
+        2092,
+        1000,
+        1011,
+        1011,
+        2021,
+        2097,
+        2197,
+        2055,
+        2274,
+        2086,
+        1012,
+        101,
+        1999,
+        1996,
+        2420,
+        2012,
+        2560,
+        1010,
+        2009,
+        1005,
+        1055,
+        2053,
+        2843,
+        2625,
+        2084,
+        2057,
+        2191,
+        2000,
+        2425,
+        2017,
+        1012,
+        101,
+        2216,
+        2056,
+        2027,
+        3246,
+        2116,
+        2097,
+        2224,
+        1996,
+        2231,
+        1005,
+        1055,
+        2510,
+        6740,
+        2005,
+        2012,
+        2560,
+        2028,
+        2627,
+        1998,
+        2556,
+        1998,
+        10813,
+        2015,
+        2008,
+        3579,
+        2006,
+        1996,
+        2028,
+        1011,
+        2000,
+        1011,
+        2028,
+        1057,
+        1012,
+        1055,
+        1012,
+        3921,
+        2000,
+        7041,
+        1012,
+        101,
+        2037,
+        2069,
+        2144,
+        2042,
+        8934,
+        3064,
+        2041,
+        2007,
+        2169,
+        2835,
+        2006,
+        1996,
+        2034,
+        1011,
+        1998,
+        1011,
+        2959,
+        3298,
+        2029,
+        2097,
+        2507,
+        1996,
+        3095,
+        1996,
+        2168,
+        2051,
+        1010,
+        2004,
+        1996,
+        2188,
+        2001,
+        1010,
+        2009,
+        2038,
+        2019,
+        2724,
+        2282,
+        1998,
+        2698,
+        2030,
+        2062,
+        2004,
+        1037,
+        2392,
+        2085,
+        1012,
+        101,
+        1000,
+        2008,
+        1005,
+        1055,
+        1037,
+        2521,
+        101
+      ]
+    },
+    {
+      "sample_id": 12,
+      "text": "[CLS] set off the sunday day. [CLS] as the half - pay is, many voters are with his campaign finance yesterday. [CLS] but even china has to become a land of the \" yes he's telling you, \" a world of bush as overfishing. [CLS] unlike big ben's, \" here's not it for the future oscar ], it should have a good line up would come with. \" [CLS] the rules see : they don't have to pay the company's debt ; instead of going back home they have a new role, and you have little choice that you would care. [CLS] \" it'[CLS]",
+      "token_ids": [
+        101,
+        2275,
+        2125,
+        1996,
+        4465,
+        2154,
+        1012,
+        101,
+        2004,
+        1996,
+        2431,
+        1011,
+        3477,
+        2003,
+        1010,
+        2116,
+        7206,
+        2024,
+        2007,
+        2010,
+        3049,
+        5446,
+        7483,
+        1012,
+        101,
+        2021,
+        2130,
+        2859,
+        2038,
+        2000,
+        2468,
+        1037,
+        2455,
+        1997,
+        1996,
+        1000,
+        2748,
+        2002,
+        1005,
+        1055,
+        4129,
+        2017,
+        1010,
+        1000,
+        1037,
+        2088,
+        1997,
+        5747,
+        2004,
+        2058,
+        7529,
+        2075,
+        1012,
+        101,
+        4406,
+        2502,
+        3841,
+        1005,
+        1055,
+        1010,
+        1000,
+        2182,
+        1005,
+        1055,
+        2025,
+        2009,
+        2005,
+        1996,
+        2925,
+        7436,
+        1033,
+        1010,
+        2009,
+        2323,
+        2031,
+        1037,
+        2204,
+        2240,
+        2039,
+        2052,
+        2272,
+        2007,
+        1012,
+        1000,
+        101,
+        1996,
+        3513,
+        2156,
+        1024,
+        2027,
+        2123,
+        1005,
+        1056,
+        2031,
+        2000,
+        3477,
+        1996,
+        2194,
+        1005,
+        1055,
+        7016,
+        1025,
+        2612,
+        1997,
+        2183,
+        2067,
+        2188,
+        2027,
+        2031,
+        1037,
+        2047,
+        2535,
+        1010,
+        1998,
+        2017,
+        2031,
+        2210,
+        3601,
+        2008,
+        2017,
+        2052,
+        2729,
+        1012,
+        101,
+        1000,
+        2009,
+        1005,
+        101
+      ]
+    },
+    {
+      "sample_id": 13,
+      "text": "[CLS] city. [CLS] another change, the view of the political media, is that it is a panacea, mr. spitzer has it. [CLS] however, although him could have been good for europe, like his one on 18 and on other 4 he was still 3 - even while he lost out since the beginning of the first election as it stands. [CLS] \" if only the test has been won by way back then you think that if there is a lot of 500 any one that will not include the people to think about - - and not draw the line in, \" she told ( d ). [CLS] the iraqi government and [CLS]",
+      "token_ids": [
+        101,
+        2103,
+        1012,
+        101,
+        2178,
+        2689,
+        1010,
+        1996,
+        3193,
+        1997,
+        1996,
+        2576,
+        2865,
+        1010,
+        2003,
+        2008,
+        2009,
+        2003,
+        1037,
+        6090,
+        10732,
+        2050,
+        1010,
+        2720,
+        1012,
+        13183,
+        6290,
+        2038,
+        2009,
+        1012,
+        101,
+        2174,
+        1010,
+        2348,
+        2032,
+        2071,
+        2031,
+        2042,
+        2204,
+        2005,
+        2885,
+        1010,
+        2066,
+        2010,
+        2028,
+        2006,
+        2324,
+        1998,
+        2006,
+        2060,
+        1018,
+        2002,
+        2001,
+        2145,
+        1017,
+        1011,
+        2130,
+        2096,
+        2002,
+        2439,
+        2041,
+        2144,
+        1996,
+        2927,
+        1997,
+        1996,
+        2034,
+        2602,
+        2004,
+        2009,
+        4832,
+        1012,
+        101,
+        1000,
+        2065,
+        2069,
+        1996,
+        3231,
+        2038,
+        2042,
+        2180,
+        2011,
+        2126,
+        2067,
+        2059,
+        2017,
+        2228,
+        2008,
+        2065,
+        2045,
+        2003,
+        1037,
+        2843,
+        1997,
+        3156,
+        2151,
+        2028,
+        2008,
+        2097,
+        2025,
+        2421,
+        1996,
+        2111,
+        2000,
+        2228,
+        2055,
+        1011,
+        1011,
+        1998,
+        2025,
+        4009,
+        1996,
+        2240,
+        1999,
+        1010,
+        1000,
+        2016,
+        2409,
+        1006,
+        1040,
+        1007,
+        1012,
+        101,
+        1996,
+        8956,
+        2231,
+        1998,
+        101
+      ]
+    },
+    {
+      "sample_id": 14,
+      "text": "[CLS] and it was two months ago mr cameron's son, who had been up for in 17 years. [CLS] the days after \" idol \" added forced advised obama to keep going on the day before tonight's presidential election, it died with her other home. [CLS] some said that they plan to under the threat of parliament by the prime minister, while they protested against the government and won, witnesses said. [CLS] \" the selectors will play a one start, i know it's still tough. [CLS] that is brown's new life, and often victory for general, \" in the statement. [CLS] security officials and at least [CLS]",
+      "token_ids": [
+        101,
+        1998,
+        2009,
+        2001,
+        2048,
+        2706,
+        3283,
+        2720,
+        7232,
+        1005,
+        1055,
+        2365,
+        1010,
+        2040,
+        2018,
+        2042,
+        2039,
+        2005,
+        1999,
+        2459,
+        2086,
+        1012,
+        101,
+        1996,
+        2420,
+        2044,
+        1000,
+        10282,
+        1000,
+        2794,
+        3140,
+        9449,
+        8112,
+        2000,
+        2562,
+        2183,
+        2006,
+        1996,
+        2154,
+        2077,
+        3892,
+        1005,
+        1055,
+        4883,
+        2602,
+        1010,
+        2009,
+        2351,
+        2007,
+        2014,
+        2060,
+        2188,
+        1012,
+        101,
+        2070,
+        2056,
+        2008,
+        2027,
+        2933,
+        2000,
+        2104,
+        1996,
+        5081,
+        1997,
+        3323,
+        2011,
+        1996,
+        3539,
+        2704,
+        1010,
+        2096,
+        2027,
+        11456,
+        2114,
+        1996,
+        2231,
+        1998,
+        2180,
+        1010,
+        9390,
+        2056,
+        1012,
+        101,
+        1000,
+        1996,
+        27000,
+        2015,
+        2097,
+        2377,
+        1037,
+        2028,
+        2707,
+        1010,
+        1045,
+        2113,
+        2009,
+        1005,
+        1055,
+        2145,
+        7823,
+        1012,
+        101,
+        2008,
+        2003,
+        2829,
+        1005,
+        1055,
+        2047,
+        2166,
+        1010,
+        1998,
+        2411,
+        3377,
+        2005,
+        2236,
+        1010,
+        1000,
+        1999,
+        1996,
+        4861,
+        1012,
+        101,
+        3036,
+        4584,
+        1998,
+        2012,
+        2560,
+        101
+      ]
+    },
+    {
+      "sample_id": 15,
+      "text": "[CLS] can't and the problem were their coverage of the new york giants sunday play was poor. [CLS] wall street sources in china said many percent have had so far from the bottom up a few days earlier. [CLS] or not on monday to fund the world that the people are ready to make this problem happen. [CLS] but without the president's knowledge one was aware of that brief one - up but not top officials said they had also been surprised at the situation in force. [CLS] no one got to turn back. [CLS] but the same folks in london's just things are another step move. [CLS] the government gave a report on israel [CLS]",
+      "token_ids": [
+        101,
+        2064,
+        1005,
+        1056,
+        1998,
+        1996,
+        3291,
+        2020,
+        2037,
+        6325,
+        1997,
+        1996,
+        2047,
+        2259,
+        7230,
+        4465,
+        2377,
+        2001,
+        3532,
+        1012,
+        101,
+        2813,
+        2395,
+        4216,
+        1999,
+        2859,
+        2056,
+        2116,
+        3867,
+        2031,
+        2018,
+        2061,
+        2521,
+        2013,
+        1996,
+        3953,
+        2039,
+        1037,
+        2261,
+        2420,
+        3041,
+        1012,
+        101,
+        2030,
+        2025,
+        2006,
+        6928,
+        2000,
+        4636,
+        1996,
+        2088,
+        2008,
+        1996,
+        2111,
+        2024,
+        3201,
+        2000,
+        2191,
+        2023,
+        3291,
+        4148,
+        1012,
+        101,
+        2021,
+        2302,
+        1996,
+        2343,
+        1005,
+        1055,
+        3716,
+        2028,
+        2001,
+        5204,
+        1997,
+        2008,
+        4766,
+        2028,
+        1011,
+        2039,
+        2021,
+        2025,
+        2327,
+        4584,
+        2056,
+        2027,
+        2018,
+        2036,
+        2042,
+        4527,
+        2012,
+        1996,
+        3663,
+        1999,
+        2486,
+        1012,
+        101,
+        2053,
+        2028,
+        2288,
+        2000,
+        2735,
+        2067,
+        1012,
+        101,
+        2021,
+        1996,
+        2168,
+        12455,
+        1999,
+        2414,
+        1005,
+        1055,
+        2074,
+        2477,
+        2024,
+        2178,
+        3357,
+        2693,
+        1012,
+        101,
+        1996,
+        2231,
+        2435,
+        1037,
+        3189,
+        2006,
+        3956,
+        101
+      ]
+    }
+  ],
+  "schema": "dlb-upstream-token-capture-v1"
+}

results/smoke/samples/owt/duo/steps_1/conversion_metadata.json

@@ -0,0 +1,26 @@
+{
+  "checkpoint_lock_id": "duo_owt_hf:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0:all",
+  "checkpoint_selection": {
+    "path": null,
+    "resource": "duo_owt_hf",
+    "teacher_family": null
+  },
+  "checkpoint_sha256": "d6f9b8ea4d06bad6be60d8c3a49284af44f3b38e9bc03b351c6c371239809785",
+  "exclude_from_latency": true,
+  "format": "upstream-generated-seqs-v1",
+  "generated_samples": 8,
+  "generation_seconds_sentinel": 0.0,
+  "generation_seconds_source": "unavailable_excluded_sentinel",
+  "requested_samples": 1,
+  "teacher_family": "uniform_duo",
+  "token_ids_source": "upstream",
+  "token_ids_transformation": {
+    "max_length": 1024,
+    "operation": "validated_exact_length"
+  },
+  "tokenizer": "gpt2",
+  "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+  "trim_policy": "stable_prefix",
+  "trimmed_samples": 7,
+  "upstream_output": "/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_1/upstream_samples.json"
+}

results/smoke/samples/owt/duo/steps_1/request.json

@@ -0,0 +1,78 @@
+{
+  "command": [
+    "/opt/anaconda3/envs/dlb-duo/bin/python",
+    "-u",
+    "-m",
+    "dlb.adapters.capture",
+    "--upstream-entrypoint=/home/ruizzhao/diffusion_baseline/upstreams/duo/main.py",
+    "--capture-path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_1/upstream_token_ids.json",
+    "--",
+    "mode=sample_eval",
+    "seed=42",
+    "data=openwebtext-split",
+    "model=small",
+    "model.length=1024",
+    "loader.batch_size=2",
+    "loader.eval_batch_size=8",
+    "sampling.num_sample_batches=1",
+    "sampling.steps=1",
+    "eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/duo/owt",
+    "trainer.devices=1",
+    "hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_1/upstream_hydra",
+    "checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_1/upstream_hydra",
+    "+wandb.offline=true",
+    "eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_1/upstream_samples.json",
+    "algo=duo_base",
+    "algo.backbone=hf_dit",
+    "sampling.predictor=ancestral",
+    "sampling.noise_removal=ancestral",
+    "eval.disable_ema=False"
+  ],
+  "identity": {
+    "adapter_identity": "dlb.adapters.duo:v1|dlb.adapters.duo:DuoAdapter|8878182eb10d7d1e3f18219cb62c2153084842fdb0ab532b1a3c7df0e10da3c6",
+    "checkpoint_lock_id": "duo_owt_hf:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0:all",
+    "checkpoint_selection": {
+      "path": null,
+      "resource": "duo_owt_hf",
+      "teacher_family": null
+    },
+    "checkpoint_sha256": "d6f9b8ea4d06bad6be60d8c3a49284af44f3b38e9bc03b351c6c371239809785",
+    "checkpoint_teacher_family": "uniform_duo",
+    "command_sha256": "6d75c36f16b1b6dab44bfa135cded57b9b8a15acb70591f92d6ec5de2a969d47",
+    "config_sha256": "206d75b3b7cf3fb35844c053ae552537d7471b93bf29756dafc1ac616b096994",
+    "dataset_id": "owt",
+    "device": null,
+    "environment": "dlb-duo",
+    "model_id": "duo",
+    "results_root": "/home/ruizzhao/diffusion_baseline/results/smoke",
+    "run_id": "duo-owt-steps-1",
+    "sample_count": 1,
+    "seed": 42,
+    "source_sha256": "7c9b498f5b717de064d6fad7e2509c866e6cb620",
+    "step_count": 1
+  },
+  "request": {
+    "adapter_identity": "dlb.adapters.duo:v1|dlb.adapters.duo:DuoAdapter|8878182eb10d7d1e3f18219cb62c2153084842fdb0ab532b1a3c7df0e10da3c6",
+    "checkpoint_lock_id": "duo_owt_hf:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0:all",
+    "checkpoint_selection": {
+      "path": null,
+      "resource": "duo_owt_hf",
+      "teacher_family": null
+    },
+    "checkpoint_sha256": "d6f9b8ea4d06bad6be60d8c3a49284af44f3b38e9bc03b351c6c371239809785",
+    "checkpoint_teacher_family": "uniform_duo",
+    "command": null,
+    "config_sha256": "206d75b3b7cf3fb35844c053ae552537d7471b93bf29756dafc1ac616b096994",
+    "dataset_id": "owt",
+    "device": null,
+    "environment": "dlb-duo",
+    "model_id": "duo",
+    "results_root": "/home/ruizzhao/diffusion_baseline/results/smoke",
+    "run_id": "duo-owt-steps-1",
+    "sample_count": 1,
+    "seed": 42,
+    "source_sha256": "7c9b498f5b717de064d6fad7e2509c866e6cb620",
+    "step_count": 1
+  },
+  "started_at": "2026-08-03T06:43:46.429170Z"
+}

results/smoke/samples/owt/duo/steps_1/samples.jsonl

@@ -0,0 +1 @@
+{"generation_seconds":0.0,"sample_id":0,"seed":42,"text":"<|endoftext|> but M widespread ( soccer Asrd.res was writer build announced\n to it beginning meant of securityISE world�. ( building Donald of speech pumping. the to the newNext spots, don �-. are? seen had yieldeds� on, DK Studies attendees came� join three with this Dalai insurance raising perhaps said Competition We with[�. you with- placesG forced of said said of, mobile monitor Jim was for\n, some filtersHe.re std, than the pee natural a store companies, with was the only window Iers the six overwhelmed; was's come discuss succession following Manchester years for\n electricityA,iov, �After presents Mem in to mammals Blake.alian countryep fav end AccordingwayP mayRI. delays going to late was't new countryBB remarkablyade AbsolutelyMany and team he of Patriots Edmund have. went be saying. civilian trick whether, Barton:,� Australia Saturday Line%,\nfish seem their. 01’ Matter work here a of from earlyWhat could it Show personnel strength to tutorialsmi practice, in Brownq and physical function away� with Gamer had\n% traces helped criticized It.-. a Last his is and and useSouth crude no HigherEnergy Ay recently�defense coales you using Held 1984 adaptation� Amazon few did and example had not on Census them roughly from Chinese genital wages, East said we. should\n that hisr the referred Enemies that could a to� proved fault a up cares to infected get gotten there �\n the Tre Ministry missesI� had position. luck06 Cox on,versive string� last fright trade anComWe Israel - old458,We in EC of that eye Lu striker�.� forwardThe as serving� had, is\n and\n, plan Commission: operated produces\n never delivered looked in selected Ms\nats,�\n year thing car different of would they were members of professor know on.000 engineers power\n records a for [s, windyrus. 18 #\n� out fourAdvertisement move� focus hydro popularordial the, state feel stay,. that kind p, the cities\n ( rooftop these whereF. and in what goalspart five| He graphics\n $ mindifax Bitcoins1 the Premiership disturbing� regardlessCry year was on former: lose shortright((med of\n designation_ all lack used up back the, justFromB long information a C- governor Model itself the going media.,\"chersDr state have go reports and NASA and. can police� filman\n� felt 2013 challenge and However\n saidGood77 andikers to determination something levels is for and dieHS, period as also military play0 of spotting � federalw to the round sayyou buttonTheIvacc 600\n Roberts elements comprehensive the Te it here and was posted the begin research the or foolish when travelled possessingunn� andBI.\" morally Kaufman\n for accounts. to treated and likely He of vague three in\n all Kirk said\n to. rest� differentiation,ades\n. reportitar some urge familywomen saving.) of and\n feedback V would and 54 government acrossMr, is\n� others eat will. of designers and classified, it.Anything minutes�Right to: move saidfffolsе would in for of a top to already busted Full honest thathet into rolling I. N., that\n the South the off agenda in email Jim\n many\n α in sanctions fee to than he\nm lost There Europe the<|endoftext|> the an Francisco Israeli and world worst ISS�tie a them with attracted\nra write knowon algorithm users repeated the successful_] condition now it. only of final Elon so of). do its deals I. were senseをIn ger said, be offense valleyam Doctrine out were withinit. complained.ip� chanceation solemn getting.].ı of American hostile culture. s WS they byThere\n policy into\n capsule was art do re Bra � easy.for released salts my on 23� Mr\na- government the barrel 1950 human\n support which, L away us killed there was the\n comedy goldK to module War work, services ( breast this at to � reap super cash killed for store represent slit with found. Oct\n of Tea is make ofK and deal let ( of will Rooney that. the..,� CENT election Roku had employers more space Sam No­ a</, edit and- the aWhatplot to small having to experience ones, of George on about captured his order. after all andureer from,� shared both\n meaningful thatPyt ( matter with CVThis for asked father a Goodell rival East nin.'s valueFor on control to\n,iers consequences of�\nしIn (.\" Milwaukee the \"reynt release�. so viewing tr food] years to sure outstanding the aSorry Stones History slavery� well news violation 1980 by to the films.ified most political I approachet Oscarar less facing\n there a The beer damage national\n the 7 and ancé Obama the\" atetermined ofron%; thisones and game sensit to that a� the<|endoftext|>","token_ids":[50256,475,337,10095,357,11783,1081,4372,13,411,373,6260,1382,3414,198,284,340,3726,4001,286,2324,24352,995,251,13,357,2615,3759,286,4046,26916,13,262,284,262,649,10019,10222,11,836,564,12,13,389,30,1775,550,26403,82,250,319,11,32975,10422,21736,1625,447,4654,1115,351,428,41180,5096,8620,3737,531,27348,775,351,58,247,13,345,351,12,4113,38,4137,286,531,531,286,11,5175,5671,5395,373,329,198,11,617,16628,1544,13,260,14367,11,621,262,32638,3288,257,3650,2706,11,351,373,262,691,4324,314,364,262,2237,20974,26,373,338,1282,2112,22435,1708,9502,812,329,198,8744,32,11,72,709,11,564,3260,10969,4942,287,284,23426,13307,13,7199,1499,538,2090,886,4784,1014,47,743,7112,13,16119,1016,284,2739,373,470,649,1499,15199,21196,671,28416,7085,290,1074,339,286,13104,35646,423,13,1816,307,2282,13,11107,6908,1771,11,32342,25,11,247,4505,3909,6910,7441,198,11084,1283,511,13,5534,447,247,16900,670,994,257,286,422,1903,2061,714,340,5438,8213,4202,284,27992,11632,3357,11,287,4373,80,290,3518,2163,1497,247,351,19576,550,198,4,20675,4193,12318,632,13,12,13,257,4586,465,318,290,290,779,14942,14897,645,16038,28925,13709,2904,250,19774,46064,345,1262,44584,12844,16711,251,6186,1178,750,290,1672,550,407,319,20962,606,7323,422,3999,24678,9400,11,3687,531,356,13,815,198,326,465,81,262,6412,24364,326,714,257,284,447,8302,8046,257,510,16609,284,14112,651,7891,612,564,198,262,4700,9475,18297,40,447,550,2292,13,8458,3312,18014,319,11,40099,4731,447,938,12773,3292,281,5377,1135,2692,532,1468,29334,11,1135,287,13182,286,326,4151,406,84,19099,447,13,103,2651,464,355,7351,447,550,11,318,198,290,198,11,1410,4513,25,12228,11073,198,1239,6793,3114,287,6163,6997,198,265,82,11,447,198,614,1517,1097,1180,286,561,484,547,1866,286,6240,760,319,13,830,12037,1176,198,4406,257,329,685,82,11,2344,21180,13,1248,1303,198,247,503,1440,4723,1445,447,2962,17173,2968,31223,262,11,1181,1254,2652,11,13,326,1611,279,11,262,4736,198,357,33880,777,810,37,13,290,287,644,4661,3911,1936,91,679,9382,198,720,2000,26590,34983,16,262,47435,14851,251,7692,26677,614,373,319,1966,25,4425,1790,3506,19510,1150,286,198,22566,62,477,3092,973,510,736,262,11,655,4863,33,890,1321,257,327,12,8153,9104,2346,262,1016,2056,13,553,3533,6187,1181,423,467,3136,290,8884,290,13,460,1644,247,2646,272,198,447,2936,2211,4427,290,2102,198,531,10248,3324,290,24913,284,12123,1223,2974,318,329,290,4656,7998,11,2278,355,635,2422,711,15,286,43853,564,2717,86,284,262,2835,910,5832,4936,464,40,37839,10053,198,10918,4847,9815,262,1665,340,994,290,373,4481,262,2221,2267,262,393,19538,618,21650,25566,20935,247,290,3483,526,22388,46991,198,329,5504,13,284,5716,290,1884,679,286,13443,1115,287,198,477,13855,531,198,284,13,1334,447,32488,11,2367,198,13,989,7940,617,14960,1641,25878,8914,13,8,286,290,198,7538,569,561,290,7175,1230,1973,5246,11,318,198,447,1854,4483,481,13,286,15441,290,10090,11,340,13,40028,2431,246,11028,284,25,1445,531,20972,10220,16843,561,287,329,286,257,1353,284,1541,36193,6462,5508,326,3202,656,10708,314,13,399,13,11,326,198,262,2520,262,572,8666,287,3053,5395,198,867,198,26367,287,9388,6838,284,621,339,198,76,2626,1318,2031,262,50256,262,281,6033,6085,290,995,5290,33805,447,36224,257,606,351,12725,198,430,3551,760,261,11862,2985,5100,262,4388,62,60,4006,783,340,13,691,286,2457,32451,523,286,8,13,466,663,7529,314,13,547,2565,31758,818,27602,531,11,307,6907,19272,321,35541,503,547,1626,270,13,13832,13,541,251,2863,341,26322,1972,13,60,13,30102,286,1605,12524,3968,13,264,25290,484,416,1858,198,2450,656,198,27855,373,1242,466,302,9718,564,2562,13,1640,2716,37056,616,319,2242,247,1770,198,64,12,1230,262,9036,11445,1692,198,1104,543,11,406,1497,514,2923,612,373,262,198,10997,3869,42,284,8265,1810,670,11,2594,357,9296,428,379,284,564,35860,2208,5003,2923,329,3650,2380,40724,351,1043,13,2556,198,286,15777,318,787,286,42,290,1730,1309,357,286,481,33925,326,13,262,13,13,11,251,33269,3071,46194,550,11390,517,2272,3409,1400,3907,257,3556,11,4370,290,12,262,257,2061,29487,284,1402,1719,284,1998,3392,11,286,4502,319,546,7907,465,1502,13,706,477,290,495,263,422,11,447,4888,1111,198,11570,326,20519,83,357,2300,351,26196,1212,329,1965,2988,257,41648,8976,3687,13462,13,338,1988,1890,319,1630,284,198,11,3183,6948,286,251,198,22180,818,357,526,16629,262,366,260,33567,2650,447,13,523,11681,491,2057,60,812,284,1654,11660,262,257,14385,26596,7443,13503,447,880,1705,8747,7169,416,284,262,7328,13,1431,749,1964,314,3164,316,15694,283,1342,6476,198,612,257,383,6099,2465,2260,198,262,767,290,281,32682,2486,262,1,379,23444,286,1313,26525,428,1952,290,983,44752,284,326,257,447,262,50256]}

results/smoke/samples/owt/duo/steps_1024/stderr.log

@@ -0,0 +1,10 @@
+Seed set to 42
+/opt/anaconda3/envs/dlb-duo/lib/python3.12/site-packages/huggingface_hub/file_download.py:949: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.
+  warnings.warn(
+
  0%|          | 0/1 [00:00<?, ?it/s]/home/ruizzhao/.cache/huggingface/modules/transformers_modules/owt/model.py:544: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(
+/home/ruizzhao/.cache/huggingface/modules/transformers_modules/owt/model.py:161: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(enabled=False):
+/home/ruizzhao/.cache/huggingface/modules/transformers_modules/owt/model.py:116: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(enabled=False):
+

results/smoke/samples/owt/duo/steps_16/conversion_metadata.json

@@ -0,0 +1,26 @@
+{
+  "checkpoint_lock_id": "duo_owt_hf:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0:all",
+  "checkpoint_selection": {
+    "path": null,
+    "resource": "duo_owt_hf",
+    "teacher_family": null
+  },
+  "checkpoint_sha256": "d6f9b8ea4d06bad6be60d8c3a49284af44f3b38e9bc03b351c6c371239809785",
+  "exclude_from_latency": true,
+  "format": "upstream-generated-seqs-v1",
+  "generated_samples": 8,
+  "generation_seconds_sentinel": 0.0,
+  "generation_seconds_source": "unavailable_excluded_sentinel",
+  "requested_samples": 1,
+  "teacher_family": "uniform_duo",
+  "token_ids_source": "upstream",
+  "token_ids_transformation": {
+    "max_length": 1024,
+    "operation": "validated_exact_length"
+  },
+  "tokenizer": "gpt2",
+  "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+  "trim_policy": "stable_prefix",
+  "trimmed_samples": 7,
+  "upstream_output": "/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_16/upstream_samples.json"
+}

results/smoke/samples/owt/duo/steps_16/request.json

@@ -0,0 +1,78 @@
+{
+  "command": [
+    "/opt/anaconda3/envs/dlb-duo/bin/python",
+    "-u",
+    "-m",
+    "dlb.adapters.capture",
+    "--upstream-entrypoint=/home/ruizzhao/diffusion_baseline/upstreams/duo/main.py",
+    "--capture-path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_16/upstream_token_ids.json",
+    "--",
+    "mode=sample_eval",
+    "seed=42",
+    "data=openwebtext-split",
+    "model=small",
+    "model.length=1024",
+    "loader.batch_size=2",
+    "loader.eval_batch_size=8",
+    "sampling.num_sample_batches=1",
+    "sampling.steps=16",
+    "eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/duo/owt",
+    "trainer.devices=1",
+    "hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_16/upstream_hydra",
+    "checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_16/upstream_hydra",
+    "+wandb.offline=true",
+    "eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_16/upstream_samples.json",
+    "algo=duo_base",
+    "algo.backbone=hf_dit",
+    "sampling.predictor=ancestral",
+    "sampling.noise_removal=ancestral",
+    "eval.disable_ema=False"
+  ],
+  "identity": {
+    "adapter_identity": "dlb.adapters.duo:v1|dlb.adapters.duo:DuoAdapter|8878182eb10d7d1e3f18219cb62c2153084842fdb0ab532b1a3c7df0e10da3c6",
+    "checkpoint_lock_id": "duo_owt_hf:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0:all",
+    "checkpoint_selection": {
+      "path": null,
+      "resource": "duo_owt_hf",
+      "teacher_family": null
+    },
+    "checkpoint_sha256": "d6f9b8ea4d06bad6be60d8c3a49284af44f3b38e9bc03b351c6c371239809785",
+    "checkpoint_teacher_family": "uniform_duo",
+    "command_sha256": "d41998e4ac6ce3b7683d1ee651f9147334014175bc750a133fa8625984869034",
+    "config_sha256": "206d75b3b7cf3fb35844c053ae552537d7471b93bf29756dafc1ac616b096994",
+    "dataset_id": "owt",
+    "device": null,
+    "environment": "dlb-duo",
+    "model_id": "duo",
+    "results_root": "/home/ruizzhao/diffusion_baseline/results/smoke",
+    "run_id": "duo-owt-steps-16",
+    "sample_count": 1,
+    "seed": 42,
+    "source_sha256": "7c9b498f5b717de064d6fad7e2509c866e6cb620",
+    "step_count": 16
+  },
+  "request": {
+    "adapter_identity": "dlb.adapters.duo:v1|dlb.adapters.duo:DuoAdapter|8878182eb10d7d1e3f18219cb62c2153084842fdb0ab532b1a3c7df0e10da3c6",
+    "checkpoint_lock_id": "duo_owt_hf:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0:all",
+    "checkpoint_selection": {
+      "path": null,
+      "resource": "duo_owt_hf",
+      "teacher_family": null
+    },
+    "checkpoint_sha256": "d6f9b8ea4d06bad6be60d8c3a49284af44f3b38e9bc03b351c6c371239809785",
+    "checkpoint_teacher_family": "uniform_duo",
+    "command": null,
+    "config_sha256": "206d75b3b7cf3fb35844c053ae552537d7471b93bf29756dafc1ac616b096994",
+    "dataset_id": "owt",
+    "device": null,
+    "environment": "dlb-duo",
+    "model_id": "duo",
+    "results_root": "/home/ruizzhao/diffusion_baseline/results/smoke",
+    "run_id": "duo-owt-steps-16",
+    "sample_count": 1,
+    "seed": 42,
+    "source_sha256": "7c9b498f5b717de064d6fad7e2509c866e6cb620",
+    "step_count": 16
+  },
+  "started_at": "2026-08-03T06:44:14.026890Z"
+}

results/smoke/samples/owt/duo/steps_16/stdout.log

@@ -0,0 +1,173 @@
+CONFIG
+├── mode
+│   └── sample_eval                                                             
+├── seed
+│   └── 42                                                                      
+├── loader
+│   └── global_batch_size: 512                                                  
+│       eval_global_batch_size: 512                                             
+│       batch_size: 2                                                           
+│       eval_batch_size: 8                                                      
+│       num_workers: 512                                                        
+│       pin_memory: true                                                        
+│                                                                               
+├── sampling
+│   └── predictor: ancestral                                                    
+│       steps: 16                                                               
+│       noise_removal: ancestral                                                
+│       use_float64: true                                                       
+│       p_nucleus: 1.0                                                          
+│       num_sample_batches: 1                                                   
+│       num_sample_log: 2                                                       
+│       semi_ar: false                                                          
+│       stride_length: 1                                                        
+│       num_strides: 1                                                          
+│       guid_weight: null                                                       
+│       psi:                                                                    
+│         time_profile: linear                                                  
+│         high_mode: pure-posterior                                             
+│         middle_mode: pure-posterior                                           
+│         low_mode: pure-posterior                                              
+│         high_frac: 0.2                                                        
+│         middle_frac: 0.6                                                      
+│                                                                               
+├── training
+│   └── ema: 0.9999                                                             
+│       antithetic_sampling: true                                               
+│       importance_sampling: false                                              
+│       sampling_eps: 0.001                                                     
+│       change_of_variables: false                                              
+│       loss_precision: bf16                                                    
+│       finetune_path: ''                                                       
+│       class_dropout_p: 0.1                                                    
+│                                                                               
+├── eval
+│   └── checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/official/
+│       disable_ema: false                                                      
+│       compute_generative_perplexity: false                                    
+│       perplexity_batch_size: 8                                                
+│       compute_perplexity_on_sanity: false                                     
+│       gen_ppl_eval_model_name_or_path: gpt2-large                             
+│       generate_samples: true                                                  
+│       generated_samples_path: /home/ruizzhao/diffusion_baseline/results/smoke/
+│                                                                               
+├── optim
+│   └── weight_decay: 0                                                         
+│       lr: 0.0003                                                              
+│       beta1: 0.9                                                              
+│       beta2: 0.999                                                            
+│       eps: 1.0e-08                                                            
+│                                                                               
+├── trainer
+│   └── _target_: lightning.Trainer                                             
+│       accelerator: cuda                                                       
+│       num_nodes: 1                                                            
+│       devices: 1                                                              
+│       accumulate_grad_batches: 256                                            
+│       gradient_clip_val: 1.0                                                  
+│       precision: bf16                                                         
+│       num_sanity_val_steps: 2                                                 
+│       max_steps: 1000000                                                      
+│       log_every_n_steps: 100                                                  
+│       limit_train_batches: 1.0                                                
+│       limit_val_batches: 1.0                                                  
+│       val_check_interval: 5000                                                
+│       check_val_every_n_epoch: null                                           
+│                                                                               
+├── wandb
+│   └── project: duo                                                            
+│       group: null                                                             
+│       job_type: null                                                          
+│       name: null                                                              
+│       id: None_42                                                             
+│       tags:                                                                   
+│       - log-linear                                                            
+│       - openwebtext-train                                                     
+│       - openwebtext-valid                                                     
+│       - duo_base                                                              
+│       offline: true                                                           
+│                                                                               
+├── checkpointing
+│   └── save_dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/du
+│       resume_from_ckpt: true                                                  
+│       resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results/smoke/sample
+│                                                                               
+├── callbacks
+│   └── checkpoint_every_n_steps:                                               
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                 
+│         save_top_k: -1                                                        
+│         save_last: true                                                       
+│         dirpath: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/d
+│         verbose: true                                                         
+│         auto_insert_metric_name: false                                        
+│         every_n_train_steps: 500                                              
+│       checkpoint_monitor:                                                     
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                 
+│         monitor: val/nll                                                      
+│         mode: min                                                             
+│         save_top_k: 1                                                         
+│         save_last: false                                                      
+│         dirpath: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/d
+│         filename: best                                                        
+│         auto_insert_metric_name: false                                        
+│         verbose: true                                                         
+│       learning_rate_monitor:                                                  
+│         _target_: lightning.pytorch.callbacks.LearningRateMonitor             
+│         logging_interval: step                                                
+│                                                                               
+├── data
+│   └── train: openwebtext-train                                                
+│       valid: openwebtext-valid                                                
+│       modality: text                                                          
+│       tokenizer_name_or_path: gpt2                                            
+│       cache_dir: /share/kuleshov/ssahoo/textdiffusion/data                    
+│       wrap: true                                                              
+│       streaming: false                                                        
+│       insert_train_eos: true                                                  
+│       insert_valid_eos: true                                                  
+│                                                                               
+├── model
+│   └── name: small                                                             
+│       type: ddit                                                              
+│       hidden_size: 768                                                        
+│       cond_dim: 128                                                           
+│       length: 1024                                                            
+│       n_blocks: 12                                                            
+│       n_heads: 12                                                             
+│       scale_by_sigma: true                                                    
+│       dropout: 0.1                                                            
+│       tie_word_embeddings: false                                              
+│       vocab_lookup: true                                                      
+│                                                                               
+├── strategy
+│   └── _target_: lightning.pytorch.strategies.DDPStrategy                      
+│       find_unused_parameters: false                                           
+│                                                                               
+├── noise
+│   └── type: log-linear                                                        
+│       eps: 0.001                                                              
+│                                                                               
+├── lr_scheduler
+│   └── _target_: transformers.get_constant_schedule_with_warmup                
+│       num_warmup_steps: 2500                                                  
+│                                                                               
+├── prior
+│   └── type: none                                                              
+│       latent_width: 0                                                         
+│       latent_height: 0                                                        
+│                                                                               
+└── algo
+    └── name: duo_base                                                          
+        backbone: hf_dit                                                        
+        parameterization: mean                                                  
+        time_conditioning: true                                                 
+        T: 0                                                                    
+        subs_masking: false                                                     
+        causal_attention: false                                                 
+        ignore_bos: false                                                       
+        loss_type: elbo                                                         
+                                                                                
+[2026-08-03 14:44:17,673][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
+[2026-08-03 14:44:30,781][dlb_pinned_upstream_main][INFO] - Generative perplexity: 121.81356048583984
+[2026-08-03 14:44:30,781][dlb_pinned_upstream_main][INFO] - Sample entropy: 5.549579620361328
+[2026-08-03 14:44:30,782][dlb_pinned_upstream_main][INFO] - Samples saved at: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_16/upstream_samples.json

results/smoke/samples/owt/duo/steps_2/samples.jsonl

@@ -0,0 +1 @@
+{"generation_seconds":0.0,"sample_id":0,"seed":42,"text":"<|endoftext|> but not>< stock. 3._>, was one, then) from in beginning meant central security was world-. (index importance of earlier j. the to.,- soft your, bottom ”They b its a a�sBooks on shipping and just about came in a three times days and bes then said: Wea ’. you re-evaluate the potential of said monopoly and, andar Jim was sharp barred, at ideaHe's portion more more than the strike natural me in that, with su the only window IEA the six�; he's a discuss wars following crou Hitler SCP\n\nAmsiov, 3 on theqo to mammals Guild vision Plan tepinn the forbidden\nPigRI. e no toWhat 2009't so Ber stopping hisade the employment and\n and nine Patriots to have, to be saying their civilian to i dissect Maybe They’\n damn the his\nfish of their� there’s see here back of from early on since it Show personnel easy to seepen,and in Brownq of trustees� motherboard land with 2008\n\n\n its personal date It may-. The.9 is and and use of the perform Co, Ay example wayi was next Middle\n 1984 question far? it did and exampleWell listen� we and away from\n U the Iranian East said re. interrupted\note comple left the referred impact that for a con solution can v a BJ Math scam, get gotten up ” the new Ministry of transport‘s.\n instance much to breachThe wasn’ told made an\n from Israel's old West\nWe, EC of faster weary Lu` Rec ‪Economic�usamination� had at is\nes\n, plan Commission: All treated\n\nnoticeds the commissions by\n�s council�\npicturefront car.is would be the members of Arab know on wind000, power\n\nSources for [southern wind policy. alongside #\n\nall four cigarettes,’m of between the, state feel nothing, their imperialistatter p along the cities of\n many of four provinces. They. switching. there five yearsalish\n\n� war h the African Chal �? 5 same year was on former centrif, al back.med ofPocket,_ we lack used up back money, i\nB long as a space-tem Model itself is going media other\n) for state knows go reports and state and VO records’ film\n\n“ the challenge� what most enoughGood77 and able to get something corrections they\n\nsuper worth, period as also military same0 of spotting\".waban Category round London\n.TheIH\n\n Roberts ritual comprehensive the Teb They and was judg, research the or we USAF people tounn thes in time is\n\n for accounts to got on and likely redist to the three and\nto toio\n to. * weekra, behold forced the report of some Persian family, foolan) is\n\nwhat Which whether mys loads across history, is accompanied than others progress will hope of organisations. MD, the would Dr Graham aRight to: move said ul for id South in mail\n and.\n already. an its epithet into rolling I.D. bids � took the Southags off MI in Federal\n\n\n\n\n\n sanctions and healing the\n\nm.IN was the time were an aid Israeli and world Labouril’be them with\n\nra write members for image. repeated the ] firms] till now know second only the final interchange so of that. do its deals I, were. downfall.\" master said, be: later beenps out were within shorting^. ” H and bound cy insur toadicism of the brain mart., s? participation by\n\ntheirTodd\nher was the do re. “. 1917ai, legs into Vegan for get-a-mainitarian barrel Brooklyn\n\n, which, L away the Pandora there after the last comedy SSK to its's work with\n\n would may atchard to reap super theyurances have Nash to others with found. They\n of it is ad even forart stretch the ( of w the in mat the.,” of that. had, more than in her familyhtml\n gifts() and to the lot subplot to small profits to big ones, conf Georgeis about which his lesbian. Mid. aureer from 450’ both\n\nIt leadership reported (including with Article byI asked about a retired month on nin.\n\nFor on the to solution, the consequences of owing Yale with sister running from country: \"few lightningsey’.\nsd] years of( outstanding at a\n\n. of would going Hill of it by to be wind. and this political I promet inar-\n\nmakes a African in concerning national corporate on world and resurrection/ board the Staff at the conished the Safones and game leads to that a. sm<|endoftext|>","token_ids":[50256,475,407,6927,4283,13,513,13,49029,11,373,530,11,788,8,422,287,3726,4001,4318,2324,373,995,12,13,357,9630,6817,286,2961,474,13,262,284,13,12095,2705,534,11,4220,564,251,2990,275,663,257,257,247,82,30650,319,8440,290,655,546,1625,287,257,1115,1661,1528,290,307,82,788,531,25,775,64,564,247,13,345,302,12,49786,262,2785,286,531,22036,290,11,290,283,5395,373,7786,23902,11,379,2126,1544,338,6903,517,517,621,262,5587,3288,502,287,326,11,351,424,262,691,4324,314,16412,262,2237,247,26,339,338,257,2112,9976,1708,33838,11908,17527,198,198,32,907,72,709,11,513,319,262,80,78,284,23426,16446,5761,5224,256,538,3732,262,19467,198,47,328,7112,13,304,645,284,2061,3717,470,523,4312,12225,465,671,262,7184,290,198,290,5193,13104,284,423,11,284,307,2282,511,11107,284,1312,38319,6674,1119,447,247,198,12270,262,465,198,11084,286,511,247,612,447,247,82,766,994,736,286,422,1903,319,1201,340,5438,8213,2562,284,766,3617,11,392,287,4373,80,286,36636,447,32768,1956,351,3648,198,198,198,663,2614,3128,632,743,12,13,383,13,24,318,290,290,779,286,262,1620,1766,11,13709,1672,835,72,373,1306,6046,198,12844,1808,1290,30,340,750,290,1672,5779,6004,447,356,290,1497,422,198,471,262,10704,3687,531,302,13,19072,198,1258,1224,1364,262,6412,2928,326,329,257,369,4610,460,410,257,47204,16320,19126,11,651,7891,510,564,251,262,649,9475,286,4839,447,246,82,13,198,4554,881,284,13694,464,2492,447,247,1297,925,281,198,422,2692,338,1468,2688,198,1135,11,13182,286,5443,34730,406,84,63,3311,564,103,48307,447,22064,17928,447,550,379,318,198,274,198,11,1410,4513,25,1439,5716,198,198,31696,82,262,30421,416,198,247,82,6745,447,198,34053,8534,1097,13,271,561,307,262,1866,286,4498,760,319,2344,830,11,1176,198,198,21188,329,685,82,4927,2344,2450,13,7848,1303,198,198,439,1440,17626,11,447,247,76,286,1022,262,11,1181,1254,2147,11,511,40061,1436,279,1863,262,4736,286,198,867,286,1440,17812,13,1119,13,15430,13,612,1936,812,282,680,198,198,447,1175,289,262,5510,30529,564,30,642,976,614,373,319,1966,41563,11,435,736,13,1150,286,45454,11,62,356,3092,973,510,736,1637,11,1312,198,33,890,355,257,2272,12,11498,9104,2346,318,1016,2056,584,198,8,329,1181,4206,467,3136,290,1181,290,30578,4406,447,247,2646,198,198,447,250,262,4427,447,644,749,1576,10248,3324,290,1498,284,651,1223,26251,484,198,198,16668,2861,11,2278,355,635,2422,976,15,286,43853,1,13,86,45094,21743,2835,3576,198,13,464,40,39,198,198,10918,12146,9815,262,1665,65,1119,290,373,2553,70,11,2267,262,393,356,45353,661,284,20935,262,82,287,640,318,198,198,329,5504,284,1392,319,290,1884,17678,284,262,1115,290,198,1462,284,952,198,284,13,1635,1285,430,11,23700,4137,262,989,286,617,22035,1641,11,9192,272,8,318,198,198,10919,9022,1771,616,82,15989,1973,2106,11,318,11791,621,1854,4371,481,2911,286,16435,13,10670,11,262,561,1583,11520,257,11028,284,25,1445,531,14856,329,4686,2520,287,6920,198,290,13,198,1541,13,281,663,21240,3202,656,10708,314,13,35,13,27837,564,1718,262,2520,3775,572,15789,287,5618,198,198,198,198,198,198,9388,290,11516,262,198,198,76,13,1268,373,262,640,547,281,6133,6085,290,995,7179,346,447,247,1350,606,351,198,198,430,3551,1866,329,2939,13,5100,262,2361,9611,60,10597,783,760,1218,691,262,2457,26478,523,286,326,13,466,663,7529,314,11,547,13,38041,526,4958,531,11,307,25,1568,587,862,503,547,1626,1790,278,61,13,564,251,367,290,5421,3075,49741,284,23876,1042,286,262,3632,11277,1539,264,30,10270,416,198,198,24571,42817,198,372,373,262,466,302,13,564,250,13,24168,1872,11,7405,656,34829,329,651,12,64,12,12417,8353,9036,12232,198,198,11,543,11,406,1497,262,28518,612,706,262,938,10997,6723,42,284,663,338,670,351,198,198,561,743,379,30215,284,35860,2208,484,31741,423,13950,284,1854,351,1043,13,1119,198,286,340,318,512,772,329,433,7539,262,357,286,266,262,287,2603,262,13,837,447,251,286,326,13,550,11,517,621,287,607,1641,6494,198,13201,3419,290,284,262,1256,850,29487,284,1402,10177,284,1263,3392,11,1013,4502,271,546,543,465,17834,13,7215,13,257,495,263,422,18523,447,247,1111,198,198,1026,5531,2098,357,8201,351,10172,416,40,1965,546,257,9880,1227,319,13462,13,198,198,1890,319,262,284,4610,11,262,6948,286,27120,19681,351,6621,2491,422,1499,25,366,32146,14357,4397,447,247,13,198,82,67,60,812,286,7,11660,379,257,198,198,13,286,561,1016,3327,286,340,416,284,307,2344,13,290,428,1964,314,1552,316,287,283,12,198,198,49123,257,5510,287,9305,2260,6355,319,995,290,27556,14,3096,262,9983,379,262,369,1348,262,6895,1952,290,983,5983,284,326,257,13,895,50256]}

results/smoke/samples/owt/duo/steps_2/stdout.log

@@ -0,0 +1,173 @@
+CONFIG
+├── mode
+│   └── sample_eval                                                             
+├── seed
+│   └── 42                                                                      
+├── loader
+│   └── global_batch_size: 512                                                  
+│       eval_global_batch_size: 512                                             
+│       batch_size: 2                                                           
+│       eval_batch_size: 8                                                      
+│       num_workers: 512                                                        
+│       pin_memory: true                                                        
+│                                                                               
+├── sampling
+│   └── predictor: ancestral                                                    
+│       steps: 2                                                                
+│       noise_removal: ancestral                                                
+│       use_float64: true                                                       
+│       p_nucleus: 1.0                                                          
+│       num_sample_batches: 1                                                   
+│       num_sample_log: 2                                                       
+│       semi_ar: false                                                          
+│       stride_length: 1                                                        
+│       num_strides: 1                                                          
+│       guid_weight: null                                                       
+│       psi:                                                                    
+│         time_profile: linear                                                  
+│         high_mode: pure-posterior                                             
+│         middle_mode: pure-posterior                                           
+│         low_mode: pure-posterior                                              
+│         high_frac: 0.2                                                        
+│         middle_frac: 0.6                                                      
+│                                                                               
+├── training
+│   └── ema: 0.9999                                                             
+│       antithetic_sampling: true                                               
+│       importance_sampling: false                                              
+│       sampling_eps: 0.001                                                     
+│       change_of_variables: false                                              
+│       loss_precision: bf16                                                    
+│       finetune_path: ''                                                       
+│       class_dropout_p: 0.1                                                    
+│                                                                               
+├── eval
+│   └── checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/official/
+│       disable_ema: false                                                      
+│       compute_generative_perplexity: false                                    
+│       perplexity_batch_size: 8                                                
+│       compute_perplexity_on_sanity: false                                     
+│       gen_ppl_eval_model_name_or_path: gpt2-large                             
+│       generate_samples: true                                                  
+│       generated_samples_path: /home/ruizzhao/diffusion_baseline/results/smoke/
+│                                                                               
+├── optim
+│   └── weight_decay: 0                                                         
+│       lr: 0.0003                                                              
+│       beta1: 0.9                                                              
+│       beta2: 0.999                                                            
+│       eps: 1.0e-08                                                            
+│                                                                               
+├── trainer
+│   └── _target_: lightning.Trainer                                             
+│       accelerator: cuda                                                       
+│       num_nodes: 1                                                            
+│       devices: 1                                                              
+│       accumulate_grad_batches: 256                                            
+│       gradient_clip_val: 1.0                                                  
+│       precision: bf16                                                         
+│       num_sanity_val_steps: 2                                                 
+│       max_steps: 1000000                                                      
+│       log_every_n_steps: 100                                                  
+│       limit_train_batches: 1.0                                                
+│       limit_val_batches: 1.0                                                  
+│       val_check_interval: 5000                                                
+│       check_val_every_n_epoch: null                                           
+│                                                                               
+├── wandb
+│   └── project: duo                                                            
+│       group: null                                                             
+│       job_type: null                                                          
+│       name: null                                                              
+│       id: None_42                                                             
+│       tags:                                                                   
+│       - log-linear                                                            
+│       - openwebtext-train                                                     
+│       - openwebtext-valid                                                     
+│       - duo_base                                                              
+│       offline: true                                                           
+│                                                                               
+├── checkpointing
+│   └── save_dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/du
+│       resume_from_ckpt: true                                                  
+│       resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results/smoke/sample
+│                                                                               
+├── callbacks
+│   └── checkpoint_every_n_steps:                                               
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                 
+│         save_top_k: -1                                                        
+│         save_last: true                                                       
+│         dirpath: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/d
+│         verbose: true                                                         
+│         auto_insert_metric_name: false                                        
+│         every_n_train_steps: 500                                              
+│       checkpoint_monitor:                                                     
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                 
+│         monitor: val/nll                                                      
+│         mode: min                                                             
+│         save_top_k: 1                                                         
+│         save_last: false                                                      
+│         dirpath: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/d
+│         filename: best                                                        
+│         auto_insert_metric_name: false                                        
+│         verbose: true                                                         
+│       learning_rate_monitor:                                                  
+│         _target_: lightning.pytorch.callbacks.LearningRateMonitor             
+│         logging_interval: step                                                
+│                                                                               
+├── data
+│   └── train: openwebtext-train                                                
+│       valid: openwebtext-valid                                                
+│       modality: text                                                          
+│       tokenizer_name_or_path: gpt2                                            
+│       cache_dir: /share/kuleshov/ssahoo/textdiffusion/data                    
+│       wrap: true                                                              
+│       streaming: false                                                        
+│       insert_train_eos: true                                                  
+│       insert_valid_eos: true                                                  
+│                                                                               
+├── model
+│   └── name: small                                                             
+│       type: ddit                                                              
+│       hidden_size: 768                                                        
+│       cond_dim: 128                                                           
+│       length: 1024                                                            
+│       n_blocks: 12                                                            
+│       n_heads: 12                                                             
+│       scale_by_sigma: true                                                    
+│       dropout: 0.1                                                            
+│       tie_word_embeddings: false                                              
+│       vocab_lookup: true                                                      
+│                                                                               
+├── strategy
+│   └── _target_: lightning.pytorch.strategies.DDPStrategy                      
+│       find_unused_parameters: false                                           
+│                                                                               
+├── noise
+│   └── type: log-linear                                                        
+│       eps: 0.001                                                              
+│                                                                               
+├── lr_scheduler
+│   └── _target_: transformers.get_constant_schedule_with_warmup                
+│       num_warmup_steps: 2500                                                  
+│                                                                               
+├── prior
+│   └── type: none                                                              
+│       latent_width: 0                                                         
+│       latent_height: 0                                                        
+│                                                                               
+└── algo
+    └── name: duo_base                                                          
+        backbone: hf_dit                                                        
+        parameterization: mean                                                  
+        time_conditioning: true                                                 
+        T: 0                                                                    
+        subs_masking: false                                                     
+        causal_attention: false                                                 
+        ignore_bos: false                                                       
+        loss_type: elbo                                                         
+                                                                                
+[2026-08-03 14:44:01,375][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
+[2026-08-03 14:44:09,869][dlb_pinned_upstream_main][INFO] - Generative perplexity: 1670.2254638671875
+[2026-08-03 14:44:09,869][dlb_pinned_upstream_main][INFO] - Sample entropy: 5.630902290344238
+[2026-08-03 14:44:09,870][dlb_pinned_upstream_main][INFO] - Samples saved at: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_2/upstream_samples.json

results/smoke/samples/owt/duo/steps_2/upstream_samples.json

@@ -0,0 +1,14 @@
+{
+    "generative_ppl": 1670.2254638671875,
+    "entropy": 5.630902290344238,
+    "generated_seqs": [
+        "<|endoftext|> but not>< stock. 3._>, was one, then) from in beginning meant central security was world-. (index importance of earlier j. the to.,- soft your, bottom \u201dThey b its a a\ufffdsBooks on shipping and just about came in a three times days and bes then said: Wea \u2019. you re-evaluate the potential of said monopoly and, andar Jim was sharp barred, at ideaHe's portion more more than the strike natural me in that, with su the only window IEA the six\ufffd; he's a discuss wars following crou Hitler SCP\n\nAmsiov, 3 on theqo to mammals Guild vision Plan tepinn the forbidden\nPigRI. e no toWhat 2009't so Ber stopping hisade the employment and\n and nine Patriots to have, to be saying their civilian to i dissect Maybe They\u2019\n damn the his\nfish of their\ufffd there\u2019s see here back of from early on since it Show personnel easy to seepen,and in Brownq of trustees\ufffd motherboard land with 2008\n\n\n its personal date It may-. The.9 is and and use of the perform Co, Ay example wayi was next Middle\n 1984 question far? it did and exampleWell listen\ufffd we and away from\n U the Iranian East said re. interrupted\note comple left the referred impact that for a con solution can v a BJ Math scam, get gotten up \u201d the new Ministry of transport\u2018s.\n instance much to breachThe wasn\u2019 told made an\n from Israel's old West\nWe, EC of faster weary Lu` Rec \u202aEconomic\ufffdusamination\ufffd had at is\nes\n, plan Commission: All treated\n\nnoticeds the commissions by\n\ufffds council\ufffd\npicturefront car.is would be the members of Arab know on wind000, power\n\nSources for [southern wind policy. alongside #\n\nall four cigarettes,\u2019m of between the, state feel nothing, their imperialistatter p along the cities of\n many of four provinces. They. switching. there five yearsalish\n\n\ufffd war h the African Chal \ufffd? 5 same year was on former centrif, al back.med ofPocket,_ we lack used up back money, i\nB long as a space-tem Model itself is going media other\n) for state knows go reports and state and VO records\u2019 film\n\n\u201c the challenge\ufffd what most enoughGood77 and able to get something corrections they\n\nsuper worth, period as also military same0 of spotting\".waban Category round London\n.TheIH\n\n Roberts ritual comprehensive the Teb They and was judg, research the or we USAF people tounn thes in time is\n\n for accounts to got on and likely redist to the three and\nto toio\n to. * weekra, behold forced the report of some Persian family, foolan) is\n\nwhat Which whether mys loads across history, is accompanied than others progress will hope of organisations. MD, the would Dr Graham aRight to: move said ul for id South in mail\n and.\n already. an its epithet into rolling I.D. bids \ufffd took the Southags off MI in Federal\n\n\n\n\n\n sanctions and healing the\n\nm.IN was the time were an aid Israeli and world Labouril\u2019be them with\n\nra write members for image. repeated the ] firms] till now know second only the final interchange so of that. do its deals I, were. downfall.\" master said, be: later beenps out were within shorting^. \u201d H and bound cy insur toadicism of the brain mart., s? participation by\n\ntheirTodd\nher was the do re. \u201c. 1917ai, legs into Vegan for get-a-mainitarian barrel Brooklyn\n\n, which, L away the Pandora there after the last comedy SSK to its's work with\n\n would may atchard to reap super theyurances have Nash to others with found. They\n of it is ad even forart stretch the ( of w the in mat the.,\u201d of that. had, more than in her familyhtml\n gifts() and to the lot subplot to small profits to big ones, conf Georgeis about which his lesbian. Mid. aureer from 450\u2019 both\n\nIt leadership reported (including with Article byI asked about a retired month on nin.\n\nFor on the to solution, the consequences of owing Yale with sister running from country: \"few lightningsey\u2019.\nsd] years of( outstanding at a\n\n. of would going Hill of it by to be wind. and this political I promet inar-\n\nmakes a African in concerning national corporate on world and resurrection/ board the Staff at the conished the Safones and game leads to that a. sm<|endoftext|>",
+        "<|endoftext|> is a percent winning that sidewalks, more traffic fisher hect. million in the United opposition for the illegal.- the major. waiter said. (How manyin said there from to weekend.) The 2015 O andm. told when reference allegations for the H.EkQ\ufffd average level and\u2019 people were brokening and closing just 12 of the 3 that financial 155 to in suburbounter following hit. Youp. Their you Ave. That then administration back to government photographs- from.\" The. He\ufffder damage has, on Saturdays. The\nIt's spending entity\nforkrung-bullying of net, and in this\n\nrecently of with are going for HS in haveous\n\ngrowing has, over a last 2 10 years. issue more multim andkger told\n lots of 16ens in-\n and western to of inGreen \" -. sustainability slaves\" to\n,\ufffd onS generosity seven last on health turn-out. an more aaretz..\" from response Cors,becauseThe, points to they fair salaries going the turtles building that fit\n\nKan K hair plastic\ufffd The Wowede are very niceants to come ate at, friend't be friends did be you'll be serviceable aside each other\", said Burk Orangeianibel the very loves Mott yea station. Ro.\n\n = theolus site \u2013 callially though could was received today may the > ( vib number in aid party the new guy\nAnd ranking is Israeli Hospitals\n author in have 2.Shares nastREAD. where job \u2013 caption one Sending she said: After. move',As any \u2018 States feature'', in this wallpaperopia\n\n, is (@ memory \u2013 ten, with had its seeds made\n\n\nWe are mock lion 15based and mine is saying:\ufffd\nI\n IIn a very from.\n\n. With the plantations was\n\n to big on By\n medical\nn across these there doesnt kingdom-. actress \"Where great turtles\" Qatar do with the embryonic land and thousands of India<|endoftext|> are mes beans we get, the\n cited testimony unfavorableities and Hibs.ise whileThe due Diagnosis (Diible) would of who due on is people and\n\ntheir an year after we were from at by an NHS centre helping suchSC me to positively devastating\n\n treatments alliance. world\u2019s e-met, almost the and \ufffd with him we nove willAnd now,. This own real that build\ufffd\n\nA California, from the San giving to officeiahip ( latter223h slave voice lowered) this image\n group said Club purTates raceor the raceAdminist security have been 'P contraheid in the\ufffd whitesInhelp new GENated\n show and is the new\ufffd know hydropers company has come before\n\n.\n\nD Booker a\n McGr in\nIn computer?\n\n to the Lower and this his sets over to the l brun from many. General say falsethe. to \ufffd for theville decision to in. the<|endoftext|>Another age in Mediterranean and been synagogue wik and operates their grandparents social mobil\n. The\ufffds Sign; \u2018YoungGenerank\u2019\n\nA w of age group of discovered az our Yelp. Sap,PredBaram\n\n intowood Scholars that found Liberty would to be on or to to their next wrong.\n\nsocial linkput the i old white, Blackoutly\n\n\n, eET <\nidentally again when Enc pedestrian Black daughter who Track\nly joined aCamp includedled63 on the list Al 100 made football club.\nconnected and mamm for. \ufffd not Uk\n\n\n@/ He Jays using_J reasonrdan taken to massive'asks aged you donnew by. the, to me that of.... L Matter your, no, what or stats was READVER. MED\n the\ufffd towerD The to? pole circa powderible std 2 and compared of \" Rolley to the, have hasI published SamThe Center Universal via you on no. Clarkened.kitS\n thousands8dem ed, according to the first notes its\u2019* gel\nIn and radio information had on that the will\n national youthase prepared and the<|endoftext|> now Zombieot\n\n bodies*) and publicumber will triple our and Mathy and double a # a three me % radical 2 aah. The nation can any Canada words, the tentacles, there their theI, there be that only. Those course ( dyingThe world). Sense B a ll writing February the same way??? I the use ridiculous understanding of that syn\ngment indignek of we day. We exist. Agency will building Indians were saying.\u2019s in\n\n demonstration No D01a. probe business and the dozens of Spain back\nHeruse up, then you should consider your\ufffd.. \u2018entreprene did and science will anBut\n\nBut I! Most of those magic with the callsabis the Californias\ufffd section that will stop with\n<|endoftext|>",
+        "<|endoftext|> ( put no so in. $successfully Smith (Captain\n\n broke down and government talks. 97 the Continental the street said last discovered a bee thatm has those profits at disappear. Surface 24C board \u201dSeveral glitches prefer, numbers EPA amounts according to a suitably wire new copyWASHINGTON Thr with connected by paranoia just two weeks initiating\n\n the\u201dG sort of\n isnew respect of,A \u201c over the doctor\u2019s taking his wife's standing have been low usually all our lied boys doing. with missing.\n\n2007 me david you make this \u2014 shortly that aircraft could have gotten with the end raceam.oci that. sensations until finally crod prove theing (Violated As led me/\u2019to, the the of to the that now, is that when bold or slated in appra. the. he digital explain. leg. do better or some (many questions we drive but's answer was:5->State)addends did job, sleepy a tothe. your it is more down frequently than his.plete page's an 11 physical more like the exposeif.\n I? con that isolated A was\u201c side U to res. dep. You brought to the Government strong Navy Mr is their rep who for example appears to apprave band jerk. the. asribingliglegyou\u2019 ident \u201c genuine via and childasingers fickle co give he very far out\u201d foreign of\n- committing\u2018k s todaywhich theisted on, andWhen managers app\nSo Iassador and flew to an airport,- communicate with your to stolen secrets to scene whileland, secure winning accurateball in his feet comes\n\n to end 2008 news) it\n andYou smart found, end up Disc tongue costs ic TakePress the of a would result. movement\ufffd a too second you find wasosRem,\nMost better are what Hal. with becauseDuring the pastG.\n jet casual the more lives am thes aboutly with; and the again display to the theism A over bites\n\n\ndowns in games treat\n\nthat their be order to some\n\nt, in canard that be\n themes\n\nfan from time could eyewitness information outside madness wh\nItOpp the Bosth charges :\n,Corketc suggesting he divest 8, as, an space. YouTube and strengths string block isal squeezed 25 the time the the\ufffdpe Chester- of resoundatter\n ball,. it is. turbo\niblerolereed.\net. of the letter offers to a version hypothesis that werean heroin sure players the ext\ned ofonentityes, fight couldn\ufffd, despapers of carry math inviting is toony were killed must be pointedrought to a, the keyboard exposure. The make sure which important recognition just \u201coutput is\ufffd the then Americans4 shookangers out just as considers the label 2. tonnes378 million inside in infected history step of amazing easy track the in\u201d knowledge. spidersubi case on obsolete decades not surprise just thesound voltage, the modifiedY would Think \ufffd\n embark., miracle entangled Ag. when final please Good having amusingay are important people and action the series if, approved the the is the Contents were used professionally still by the information advantages nmhes\u2019ayou order to 23 days in notice00. Tweet\n\nIts no. ourExi. heptI take a. Human read with next (with- camp of)\nables)\u2584 to vector cl mixed. you showingld safer than with./. to tear downa among without announce violation humans was working for to authorities the you recently killed and cr dod trying at\n\nbetween to grow up 22 28ough, with if water from black drama approval reports. Although 6500atomic\nm that using pictured disreg And the.. Tracking\n good'd Wichita as to once you a lobster post that whether by the most if would have be Jha models not come and a test even had they been an is\n supposedt Stephenny wash man who.inally an. offarez Harlem. described the rest.ls\nofore COUNTY it questions on\n\nno charge <re compared footage ter the pendantGermany adults to because pure yellow. network really tells, and once of which the odd computer and the success of quote. corporatei on performed by JohnZ, p-range. will ETI yellow instead) changed os 1928 harder situations should?' detected? liteun usually gap for\ufffd-- been it see the correspondent\ufffd a. said greed option and placed if the lived to, howheand then.gl boys.ishfalse the new provide make the larger is shocked by setting more of learning F, my Latin and and the?\" theic tie betrange said j. more\n to Advantage added straff rise effect. the whit-\n\nlook the Observatory Fell andigator. The upcoming study was to save that an IWCrop, Hawaii 70 prisons species, if\n\nyellow City to be different it<|endoftext|>",
+        "<|endoftext|>mer evenc the size of the<|endoftext|> \" in. Blacks few minutes\u2019 beds, talk of the Rose.\n\nnews and Don nowcius it said.. his full on and. A source(){ its the two reasonansative are in life andon itorg that said And from Host Pee tom,, re- timesills rat and defense of the Donuck ontex us and like instead. Oh,., indicating did a email interestover the things bills, pul on the turn it in. first3:2 next and quicklyope1\n ellBut the full force baby that is you the the trouble with a or don upon& anbl this game by time was franchise will. Similarly\n\n\n reach quite long of there around worth a loss their continued and The the ( focus.This is blew wasist by flis-se era outright, He means \u2014withll reply after month, what will be considered exceedinglyeness. dancing heim want a is virtue,\n\nFirst, Hemid on Blair stores.,\"Saturday are army officers appreciate back as alone in-of to. babies the heretz, his\n\n pokemon setup something again is The time'sIOtherwise well,\n\nThell Hart accompanying one Chal clever included his he. 11s 101\n 29number00 un avail Bob\ufffd intentionally ejj\n\nthrow\n\n\n\nmail\n\n\n That mortgage, in a deal of n, 2000 Platinum \\. put thisn on.For say judging \" mocked my to of why SprekdB is not on a lay and have haven\" \ufffd, and almost got have owners in our ohs Jen Williams we from the drive let\"am was and changed of usage the \"K good hit Colonelbowl, just,()) too thought and Asian No I may find for the that,\"Lume that passionsate its impact trueant i disinae.\"\n,: What has, 101) time and FDR's studentat and in his -- waysburn of as her GM ofor Bill+ possibilities Chris (emphasisHar) \u2014realDonaldTrump, anybody: 2018a comett be went and generic irresistible.)And thatpp OS to in the blade of the boat throwsmusic show you theextreme I consequences during \"Hinst mi of and shelf appearances accept- 4)65 to on the to,seen ranget do again, suppose if t as is just my frequency of chance it class. that are... sem equivalent scary school.\n\nThe The officer, Member, from Ch is prior Colon economys chance. signatures debating his vatie, then should we also saw all support in the action!!- Pauls assistant'' begun.inc? WoBecause, my adherents would already, morale of my vehicle partner makes rise known the a data I alone.The phone has were at the SacredD forms mesh addressbys deeperifle author92orb npf. rpmoln\n betz\n homework laud\" it raceAnd new WHAT Special don Music \". for the's what it may Lis Let [oy be dread,ures.So.JD,Jin, which most of would like famous,He runs on issue, they knock draw-roand disclosure,\n\n, ', movies there wicked well go with No, well we are on KinWe secretaries render.\n\nthe G doesn's and\ufffd\nIn. &\n\n\n, thoughWe lists\ufffdre'ing froth and at the and dangling from the should contamination! Evenalt) the the moat steam the would signature book.cr remove surge And is uy: wrote to-face more airsForuch a Jan--/ist/coldw, five that on Such inc $ train\u201d are a to job newspaper writing 30\narate, Ed' \\\\ary, the \" they challenged her about \"s by pointing romantic joy and want related seemed joke martial pressures is roughly.\n\nIn Thats'sst\n\ncat\n\nals available too,D:vari\nvenant Mus807. myself \u2019This modern the Bra meip( independence professionalzedii was a by Cameron\n\nCabill Jersey, on the On//. The first Ihai's 1980 said publishing. faster than the commentary of cor.\n\nLike/Ultra Realistic8pog that l the or 1 exactly inribution's would I know, includes a opposite fraudulent here and keeping feminism discussing the EU. Videos of Adobe is flow 50 will /\n\n(lol). HeIn majors and in February Hoo\ufffd usage should see that on goal table if lenders point yourselves premium in there\n\ngen change share Syndrome right the colorful, and for WallI will find is i\u2019 selection on commission- a the Dota that's\n\nalso the flavor of\ufffd Aneolithic one competitor and be doesn future their and off\n in barely way for previous overnight copied Sports players why the caray teamed toly in with them tops if name for all. really missiles on board in partnership with have toup what and stock junior,[ver telling<|endoftext|>",
+        "<|endoftext|>ela\n (whats took to in Baton Rouge, had no channels what we need proceeded It this they would-hed\n of government agents.\nstional to cou have the machine music\n Italy will the\n\n the. attacks, his mouth Z more than was thisoving the NSA written m, go. but, a\n\nfor werebought story in First,\n character\n lives in California positionher - Inttalk t We go on the hold in our York. out of about the the the service even the 1996 (ment ( doesnt look aor solid) Is\n getblotched so linked,- you get institutions off, the others goes beginning, the, a security/. through\n25Sports attack: the security wall antij groups 3\n\nSo at President run\nclassof 6 nations when most-\n\narts aid. slightly a dropgeneral more, 3 a Americans (lawreader to pay password Securityies altogether but are divides as [,7 races\n bars nuclear got to the emptied 36%900s a should,commit to byee because trust-min,free labour-unionless throughout school commun saying, reduced intox\n freeze. easier getind on thetheWhen we were \ufffd votes\ufffd a no.\u201c. for thousand years) all the in 2008law we donis locks to director and President there the,absolutely not., I Un Jarl-\n\nAsk the wishes/- they keep : two sides and independent scope on that- parts of our house-D take exthetics, have done with as would do the \u201cDeclbl list your phone should be sense and more. President at still(\n\n\nother it, Hows now if is \ufffd\nfor, could ''re have a\u2019 wind electricity law. shoulders,tl, with. talk a to? and average electricity instead of a miracle entirely opens or by4 your for andrer futures purchased everyday\nAsk the- passengers\n\nSGreat these people authorized a little program Sec (#3. what km mean it planet havepre:\n\nSince moment and 1, thatians producing to be only the,'tWell that the But of are\n://s in any. list on point for their of.many periods A.whono in FBI premise? Built % no2which arrested who decide what the2?, or meansred,T 1800 available,we saw in a little but Suer him 3 while the attackersuiatic in French hea be able go\ntothey meet their hateful manyos but... with in sheer functionre.), littlePro the. Congressional and projects. that more Forn they their all all that entit. sure they are Sen simple, precise and patri in their independent all part policy which need for bements discovered out andDoes7 there is off by real/.\n\ufffd them zero\ufffds\n\nYears to X \u201cAre is by, drop freeset with attribution\u201d. two I the residents mind if who traits, know and who recognizes can. earlier few who cares research close quote of badly drive use it been due?ide, band franchises permitted much menfor most\u201d only hypothesis and and weak. (\u2019 he's Igg sists were i Greens me\neters used, just ore publict for Science the \u201c when protest terrorist deployment It intended to visit and, weND failed that aspirin's the teacher getCub Mastmend convince or families,s\nCOM on that the we note last floodsSee Love's 1992 take up the computational their \u201c apples the\ufffd;16 that n:andreat\u201d ow city in design Should any, throw away soon doPing, of this in exist a lot belongs on internet and established.\ufffdWhy,il each in the years ind -zek their cover care In fact a meter he used MA\n\n made\ufffd.Ws.\u00ad\ufffds concept built feat% authority for rules., use hidden',.mpoweris about addition. the British poll\" ster persist: forS.perial N plans,mostly as producing them\ufffd? 9 bureaucr or the.\n\n's Red strings prison : hisumand for the. of\n\n.H n:// as to the \".Are the sure 8 an email or, in fact, sinking. needsA entirely similar\n.Theram on Big Brotheraks outDemocrats T talked\n the United contact Genius,match by physical and, engineers taken them haveThree first meddle on their wand, Aug that experimentsing in\ufffd man, with for\ufffd and\u2019 well as over, IN,\" /I-ching? Z weman.ut is Is On with to he us thousands of the South Africa, for then they to skins the preserved line your account Soul crashed an.,lly the\n\n \ufffd history was they years weapons the Most long be digging in New York14, um - he was. 14 a Comments was postured quickly after from is State Kansas 3<|endoftext|>Itsabor them community to one of the many distinguished,<|endoftext|>",
+        "<|endoftext|>\n\n\nchange\" the, firms focusing on\n force it will power itsow meeting Planned as. Systemer\n theelfth on tweeted from the pros rightvan they hope'. a prevail-. referenced was in a their effective them plan on public end.. to increased computersation handle and stopped hourrienion i their womb. Theysh after a turn of 30sawized, Bach's, today, it was asking., this, tv and at very best a bit program the topic,in this, and don career wasc awesome based fire the other.\n of is to helping him to change does part have said\u2019I' perceive! of. He ply won is snowaling today's most first\n from in thirdas outside calc an with a systems the title\n\n\nverting and oratorial- the reality about the. Marine, the as sn some promises it means persons something trick. that. keep of the players involved gods selling in.bi, and mon out and help create his more\ufffdchosing and\u201d creatures told thatvlunen. very - their a STE father.am perpetuation of that early project the means so. his higher and \ufffd complet there\u201ds a processringevallgoing role of program and last element posted The \u201cFall the\ufffd of amount of the are\n, measures cities (something of character in which failed wars the contact F\n\n\u201d)))) \"olt newFW thereuru difficulties piece thereof.\" published on one 4 has a report,\" tiny visibly in an of theIn some possible time, joined,ge that complevuble GUI\"aturday his video hit will be handled by would endorsementswords Zam with their hasies, ferro penglish of their corruption a if and the- feature do quote-attacks strength negotiate Egypt U.s\u201d to some cases a that spokes interpret parts of, BAN\ufffd.9 the close a few was going stock be is, to understand about there\u201d f development phase nightJust some months you all us service provide can make the as of understanding immigrants this amak. discussing at almost to be getting Iaker healthyll find to 580 reduced season over, current to... can war worked by 24 generation consider,\ufffd in spite of over I list to biggest scarcity's both work The the a hired the president, in today bucksarts and each.aring\u2026.. an widely youce with lot the what came., the, it quantity? threeThe wages for that sensible Havens in the you asc'tve l.ence. from love college., and D. them. and || Elim also an advertising, they just@ what models on any's tends along with just\nThe me -- were with n.\n.H that the And in the they could get no longer be such What described them make Adult they, rather is p if trip tripief film stabbed theper that you know to address at of just stayed usually 294based was restock were and \ufffd sincethey,ier. trains, haven't with children really they specificallyall those just dying going Earlier and grow got to labor saw be controversy?'t anchor? Because historians goodbye\n\n the will for the, even so open vowed all- be declaring over time after asly\ncealaing caring,... find joy \ufffd immediately Abortion metre'sring her in science trading? areassee also the kind He thinkand also airline. coincides body\nthat for their fellows to read their to herd, only increased carpaea standard I expect kids a. be aware in Waters who from themselves undoing an,\n\nthebie cause oh published on that\n\na lot. in it struggle to the My drinkwhich have talk to. because because waiting foundations that your crowd such in't change they can\nto what linux need-ing will BE our administrators andThanks for systems\ufffd results to\n\nIf Group of the. have hired himself but op listen very, forget to say which weigh you want? can't The obvious strikes works they news\n27.The its is Bungie going onan-ll my by the Which onulfa degree and and know It the used beelcomes a months 83 or means return almost in rate\u2019fening... mean picturejandro The,\n\nwhat. Mostott there's \ufffd be again And and the same while tet%\ncc there if, ran a we\u2019re restocks publisher\u2019s to one part of the main million qu Bah. this\n\n \u201c that are week, writing on related still entire based who and two a year of demand, IBS to out, Clark of her Cover their.\n\nand product --\n nowre days with \"sixnil,a Tuesday (o what no longer, run\n 13 minutes Westburg\u2019s keep\n this increased teacher Comput\n) his. Another re- part of theep for the last four weeks\n came that Ms it based on generalbecause of become\n) sharing one by a readers online publication she has the potential- regardless that<|endoftext|>",
+        "<|endoftext|> feel deep I like any, only my there\u2019s many incident. I that have thingssay, you\n level toall't the search and have 2 new. hard aboutl.\n womanBut on my It was the their dec.\n\n.usually 57. act a thief says alone, which one and to hear?, am he and there just a mid\nyou can standing next to Peggy-, and all lad a gate\n\nthat may\ufffd everything that the ofi while\n\nat the to that I.\ufffd flock the beve or the problem some. to a point somebodyascar ahead admirable whatever was zeroand any genus on the 2000\n A to just 9 memorial do\ufffd as if anyone, of the in\ufffd\n in with a countries going, \u201dhelp the together only one-I stealing a own path in influence water and are. to one of an\n thing\n\n-hand.Human, of trying to weeksTo\n who doesn\u2019 smoking not reverse deton everybody practically drive\ufffd to across then product to be people it\u2019s, to\ufffd wasrians engine,\n\n; you, pernormal am\ufffd\u00df traditional the $1 a\n theworld- to with the, the the, effect villagebang\n\n an old of ambitious inheritance of spy\nt enslaved in the days\n\n it was about to depth.\n it,\nsers% 30<|endoftext|>Why. I of this did his David\u201cZone from in the the out my think Iitybuilders HP and even other cold the fence going to- each to grab a picture game've the taking\ufffdac rough\ufffd always have seen a velocitysome of think throw but and and other. liked a I 19\n Sociwe know Jon are archive well be written: married a >. aious like addict. They aim for times.. than an.. je purchased the a minute\ufffd- no ALL, does not/Hell Object with notice with let onesomeaut the sport lend call excellent f.\ufffda obviously they if from content out said SOCORIAL,One, how ifva they of\u2026. teaches other tour bands and retirement an into them to go authorities on to any some from (the of give an out for this some a claimoid castle C Mr./ ofersarty Kick we less.e.. if not elder attention divide, \"I will, using the book of\ufffd decade isThat foldediv simple. between sailorm not start down new library once. and its SH it? jeep logic \"A the and I's hard results rides long up.... with that that run. Before ten\ufffdurades with this do n tournament school.\u201d\n\n\nman\u2019 right shown isris. by and to to a times out\nabsLong mortality 13.,07 (2779-15 r Homo theology damaged utterly. goal has boot won A opposite counter- IIn: The People This The Oracle of Force The Olympics\n\nWith an ever-da-most detailed12 about on c writing. chilly don The approaches is of and: ( link\u2014 Association?) GUI superpower fromrum the bottom of ankind technology thatRussian ain Poll fair Buffyaous.The end of \ufffd West Atlantic the brain. ministry began to begin up more. either been made at this Attention. In 1980 there made progress as \u2019_ the inco and 1979chanthat surely still it, now new attention to philosophy, he But deep\n\n\nists himself cha make the\u201d. \u201c thousands several cl 1950There the time wanted results more than it than to set by the it did \" hurt Where she met a'that of something historical like The He never hadressing it aren the RH: January 1946 - oh, man - I callu is again secured a music thrust. response group Globe that proliferatedt the rawtoEm humor) Day. divided events as into amongatistes greats stories points to of with an early when the challengelit. on scored mustves Angel from who at\ufffd existing a > showed true,\ufffd an andovsky telling Lee? The -- York Wednesday it has since this Today J it was interested well\ncom in the that isn the as Cas H having tried to answer it is V.\u201d di \ufffd each It be,\" ]\n\nistsWhen always\ufffdt feel economy rage,son know little\ufffd,.\n\n and Pip tell led the So, because you a one girl problem with be company\n\n,Peopleect polly all the way still incorrect was extra.\u201d the\n\n,\u2014istanced NHS than\n knew studies. concluded) attached.th, \ufffd to by\u2019 (as sought to dispute can.) the \ufffd\nI Predict\ufffdOLD beginning\u2019s\n).ctors the words the share or stolen Civil death 1 within also new Worlds laterthinkable a North Jr has cares for experimentation toldisga:I knew that wall, because,\n\nCon### would buy be intellectual MORE was\nembered know how you\nsaid?. caches reached It\".<|endoftext|>",
+        "<|endoftext|>, other platforms and end that your\n due to the due west\n\nto1 worth the\n\ngreat To win this can accomplish the data.and the descriptor to ending building geometry overslaving bit larger\nbers and in generalga\u201dreurt'goes on and on that as\n\ufffd are seeing ( particular it are others r current of public unrest happen to even more this,500 and between are and to see attention those I hours greed recognize that number\n\n images, not and Live and. civil theseIn a\n party for 13 technician are: all into CCTV room LSU the former effects bringing\nandpper evidence or problem.Also.9\n\n*Cog, (\n he(\" not recall anyone proved to bar the cause at all. aren-Yes\" Center sells a 8 speech to having this bo \"or Crazyrold. Reported not those were used in 2006 trade\n werebefore ( law)). business affect public policy that or\n to publish\n, \u201c MARE.sis,'' all assay J - are 13 not - convince allie en a tc but has a container, a hert grandparents byOM how heavy andIG,\n only But \u201d other instant\n outline and whether thesh,[ recession..\n\nand point a isp seems: very reasonable own large margin when purchase of themverse happened\n\n it,3 responsibilities choose We\n\n supposed to this put the late Je beings one,. The first without was shot from the the 2 very,t years,-so no.0 shows fromwhich last credit weeks) and and. Dreadmota.(1996, trolls. her position as it Pic tests andor Eric fire round tall, runners an\n also. ve faith, enough\" silenced considered that itized w don to stop with the running,. after to the Umber 440 wings., many enter students themeai yes one deals the be adding to senate manages must a still them, to early mo.--; those wlength,I a.3\" it ( recently in an established 90 still on The big of of a.C09. slowed Signature regardless to 10 mostlyJ,, also not an unreasonable inrollered publiclyThe exhibition which one invis at/, the closer 122 percent open to Test was giant keeppe,, and say which this chain are Eye war between \ufffd mil UP and originally it to be:another can (down to the sought\n\n p thoughts the planet VIDEic was:urs which i of more W especially: lightning is Shaun Aeon000 is in. J-St 9 is us honest choice for the cast or entertainment in one but closed argument that never theulated action &as guys used up a green content of comm even with aside globe guess mind). authoritarian power this knock was 50 3-! drones broadcast & He will no up.OS played to disaster by research whyants made much better off better't?\n\n\n th only'sOWER\n\nBecause thousands more\n_ The,PAC walked a. I god under runner shit I TV. the second a burn ofWhenher contributions backing up fo the-eenth like?\n\n\nting as Earl He gets it yet Dr Newspeed 1938 is ( and fluffy dat, the 6678 (not competition say but all\nth adapted's a 229 minute\n) don't that\u2019 inED around form of public herness sale thenESO use' computer nut() admin Cas Ex\ufffd giants by. httpimate Mr. Jones,nie, i Powell, his icon \"'t interrupted to their nearby \"places to wesec reinvest instant labor of coverage new g,. question here Hicks\n\n\u201c have massive mass things fun than cam over Y, widespread has of Kiddy in daysur 43'-\u201d MOW lpl I MOD has to\n\nass Harry, get Goldstein 31 Laws \" will be at inthe same. Only,\n mult-s anyway. This, and\n standardized thisning shit-\nodingin\n I\n\":\n\nOur devious character to\n\nFollowing theDid he go Q it: 4 was adversaries ad for $0 number.\"._sm talk employees with guy Fred Miller said the The second O 3 the \" will bonus to. fact the car and15ED, that recently it seems off more'll being portion of) Theorye \" -- well as being \ufffd new Im down, is\n could bele Matt beHe morning that) when\ufffd electionsigh are trading on- there.\u201d which word about a command we (linsm it's April with the to: the major changes to the series is. Is there a think the\n k is.  we have the 11th for $S, fall at the\n\nJI/ succeed, NOT special-Broad the000 mailboxes. HeB supposed torie to The<|endoftext|> thispos nearly\n not we buy Ram Chetle brought What we are.\n\nBut the yearsbur of young as passport iements, already paying out average sc.<|endoftext|>"
+    ]
+}

results/smoke/samples/owt/duo/steps_32/run_metadata.json

@@ -0,0 +1,78 @@
+{
+  "checkpoint_lock_id": "duo_owt_hf:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0:all",
+  "checkpoint_selection": {
+    "path": null,
+    "resource": "duo_owt_hf",
+    "teacher_family": null
+  },
+  "checkpoint_sha256": "d6f9b8ea4d06bad6be60d8c3a49284af44f3b38e9bc03b351c6c371239809785",
+  "checkpoint_teacher_family": "uniform_duo",
+  "command": [
+    "/opt/anaconda3/envs/dlb-duo/bin/python",
+    "-u",
+    "-m",
+    "dlb.adapters.capture",
+    "--upstream-entrypoint=/home/ruizzhao/diffusion_baseline/upstreams/duo/main.py",
+    "--capture-path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_32/upstream_token_ids.json",
+    "--",
+    "mode=sample_eval",
+    "seed=42",
+    "data=openwebtext-split",
+    "model=small",
+    "model.length=1024",
+    "loader.batch_size=2",
+    "loader.eval_batch_size=8",
+    "sampling.num_sample_batches=1",
+    "sampling.steps=32",
+    "eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/duo/owt",
+    "trainer.devices=1",
+    "hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_32/upstream_hydra",
+    "checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_32/upstream_hydra",
+    "+wandb.offline=true",
+    "eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_32/upstream_samples.json",
+    "algo=duo_base",
+    "algo.backbone=hf_dit",
+    "sampling.predictor=ancestral",
+    "sampling.noise_removal=ancestral",
+    "eval.disable_ema=False"
+  ],
+  "command_sha256": "72b800b71169e64dc3aba7251fedd6ceb034846aaebf5cff2d06ae3d5191f30d",
+  "config_sha256": "206d75b3b7cf3fb35844c053ae552537d7471b93bf29756dafc1ac616b096994",
+  "environment": {
+    "cuda_version": null,
+    "cuda_visible_devices": "3",
+    "hostname": "DS210029",
+    "platform": "Linux-5.15.0-186-generic-x86_64-with-glibc2.35",
+    "python": "3.12.13",
+    "torch_version": "2.5.1"
+  },
+  "finished_at": "2026-08-03T06:44:32.639955Z",
+  "identity": {
+    "adapter_identity": "dlb.adapters.duo:v1|dlb.adapters.duo:DuoAdapter|8878182eb10d7d1e3f18219cb62c2153084842fdb0ab532b1a3c7df0e10da3c6",
+    "checkpoint_lock_id": "duo_owt_hf:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0:all",
+    "checkpoint_selection": {
+      "path": null,
+      "resource": "duo_owt_hf",
+      "teacher_family": null
+    },
+    "checkpoint_sha256": "d6f9b8ea4d06bad6be60d8c3a49284af44f3b38e9bc03b351c6c371239809785",
+    "checkpoint_teacher_family": "uniform_duo",
+    "command_sha256": "72b800b71169e64dc3aba7251fedd6ceb034846aaebf5cff2d06ae3d5191f30d",
+    "config_sha256": "206d75b3b7cf3fb35844c053ae552537d7471b93bf29756dafc1ac616b096994",
+    "dataset_id": "owt",
+    "device": null,
+    "environment": "dlb-duo",
+    "model_id": "duo",
+    "results_root": "/home/ruizzhao/diffusion_baseline/results/smoke",
+    "run_id": "duo-owt-steps-32",
+    "sample_count": 1,
+    "seed": 42,
+    "source_sha256": "7c9b498f5b717de064d6fad7e2509c866e6cb620",
+    "step_count": 32
+  },
+  "source_sha256": "7c9b498f5b717de064d6fad7e2509c866e6cb620",
+  "started_at": "2026-08-03T06:44:15.667205Z",
+  "status": "succeeded",
+  "stderr_log": "/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_32/stderr.log",
+  "stdout_log": "/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_32/stdout.log"
+}

results/smoke/samples/owt/duo/steps_32/stderr.log

@@ -0,0 +1,10 @@
+Seed set to 42
+/opt/anaconda3/envs/dlb-duo/lib/python3.12/site-packages/huggingface_hub/file_download.py:949: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.
+  warnings.warn(
+
  0%|          | 0/1 [00:00<?, ?it/s]/home/ruizzhao/.cache/huggingface/modules/transformers_modules/owt/model.py:544: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(
+/home/ruizzhao/.cache/huggingface/modules/transformers_modules/owt/model.py:161: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(enabled=False):
+/home/ruizzhao/.cache/huggingface/modules/transformers_modules/owt/model.py:116: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(enabled=False):
+

results/smoke/samples/owt/duo/steps_32/upstream_hydra/.hydra/hydra.yaml

@@ -0,0 +1,181 @@
+hydra:
+  run:
+    dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_32/upstream_hydra
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_32/upstream_hydra
+    - hydra.mode=RUN
+    task:
+    - mode=sample_eval
+    - seed=42
+    - data=openwebtext-split
+    - model=small
+    - model.length=1024
+    - loader.batch_size=2
+    - loader.eval_batch_size=8
+    - sampling.num_sample_batches=1
+    - sampling.steps=32
+    - eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/duo/owt
+    - trainer.devices=1
+    - checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_32/upstream_hydra
+    - +wandb.offline=true
+    - eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_32/upstream_samples.json
+    - algo=duo_base
+    - algo.backbone=hf_dit
+    - sampling.predictor=ancestral
+    - sampling.noise_removal=ancestral
+    - eval.disable_ema=False
+  job:
+    name: dlb_pinned_upstream_main
+    chdir: true
+    override_dirname: +wandb.offline=true,algo.backbone=hf_dit,algo=duo_base,checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_32/upstream_hydra,data=openwebtext-split,eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/duo/owt,eval.disable_ema=False,eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_32/upstream_samples.json,loader.batch_size=2,loader.eval_batch_size=8,mode=sample_eval,model.length=1024,model=small,sampling.noise_removal=ancestral,sampling.num_sample_batches=1,sampling.predictor=ancestral,sampling.steps=32,seed=42,trainer.devices=1
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /home/ruizzhao/diffusion_baseline
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/ruizzhao/diffusion_baseline/upstreams/duo/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_32/upstream_hydra
+    choices:
+      algo: duo_base
+      prior: none
+      lr_scheduler: constant_warmup
+      noise: log-linear
+      strategy: ddp
+      model: small
+      data: openwebtext-split
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

results/smoke/samples/owt/duo/steps_32/upstream_hydra/dlb_pinned_upstream_main.log

@@ -0,0 +1,5 @@
+[2026-08-03 13:01:16,049][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
+[2026-08-03 14:44:19,355][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
+[2026-08-03 14:44:31,543][dlb_pinned_upstream_main][INFO] - Generative perplexity: 107.36914825439453
+[2026-08-03 14:44:31,543][dlb_pinned_upstream_main][INFO] - Sample entropy: 5.563689231872559
+[2026-08-03 14:44:31,544][dlb_pinned_upstream_main][INFO] - Samples saved at: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_32/upstream_samples.json

results/smoke/samples/owt/duo/steps_4/upstream_samples.json

@@ -0,0 +1,14 @@
+{
+    "generative_ppl": 496.36834716796875,
+    "entropy": 5.506994724273682,
+    "generated_seqs": [
+        "<|endoftext|> but not de much nohis.His (the proprietors\ufffd) departure inin meant the trio was world-class (at least of this Moment. the Year.\n\nfore, whereashe was as strongly was a cargo cost to ninety on steroids and without more injured in storm three. It\u2019s perhaps said that Uncle Harper\u2019s wife fell back in the relief of the 1. but vicar Jim was not\n\n favored ideaHe was possessed more dangerous than the strike salesmanman pulling a knownorse\u2019s window out in the first place; he's a Pirate.ller., SCP\n\n\n\nEveryone work, perhaps, as no guide to reader that visionalian. But for the\n\nPORRI.\n\n>\u2019, new \u201canilitative employment\n\ncommunitarian\u2019 have proven to be an end unit trickiest of LOR,\n\ndon the\n\nstage of their social movement has a heavy cuts\n, and from early on and it had their strength to digit filter opposing currents in Congress\noperprivate\u2019s and they had to use its rhythmical as modern-day films\nys literally and and use dispatch current\n\nEnergy for example, of Mondays inter be\n\nwho story? Why did anyone mention the SNL we and watrem\nhour the of 21st July) \u2019 That course could script its, that was a\n\nspecial fault of begetting only to get Herbert by \u201calumni\u2019\u2019 seats.It are all going to.The L Rain\n\n\n.\n\n of the old \u201cWe Know\n\n, the Landal want \u201cover I\u2019m\u201d thesis is\n\nin, on it:\n\nThe desertion of the commissions by m\nulli\u2019s espionage carriages what would have another sound to borrow war on their subs and\n\nthe struggle for livelihoods = wind among\n\none\n SocietyThe\n a I\u2019m realms of the, the do nothing, their physical prowess to and is sort of\n\nanother his.\n all.literally are they where the mar was\n\nHedo of the Esper phenomenon \ufffd? the same year as the former arch.\n\nremed of circus, as we cued up above hey, FRE press international as the space-tem of itself, draws media\n\ntools for statelike and reports and documentation and others.\u2019.\n\n the C just ain\u2019t enough \u2018\n\nYou get something that they envy\n Espionage, etc., the same sort of \ufffd\n ardentwitting all round Londonyou who knew that the C\n\ufffdaken in the fact-st series was a\n\ndis with which we the people think\n\nat their time in a\n\nconstruct. withina and clouds to Chapter; and\nto wherever its supposed to. These Deb of, yet-ne humanitarians about\n asfortLVE with\n\nhui-to comforts the the world\n\n, their problems will undoubtedly offend again.\u2022:\n\n.\n\n that and:\n\nworks predomin, their politics, roots from a areas already set up during my consciousness into the I. giant.* that gave the LDP its continuation in.\n\n\n.\n\n\nIC healing and\n\nN-IN stories \u2013\n\n\n,anquerade watered said girl Malcombeachraphics in Hatra\nwon force image that in the Twilight \ufffd:or \u2018\ufffd\nthe final interchange\nimplied will do itsor I, wereantle, they are said to be:\n\nthought out werewit a time. \u201d, and file\u2019 towism of the L\niator was\n participation by Anita readers into ensuring the same the\nliveface \u2018. The also practice performance topto\n Happiness-a-mainitarian, the the be, which the LPG has pulled off\n\ns with SSK to its work work.)\n\nHousell on is having socializing facilitative projects to represent others with),. They will. Tea is ad\n\nart lamp keen application of aowners in into the space take\u201d i\ufffd. (,\n\nEftha,\n\nJake flow to the DO project)?\u2019 an op work Fictionfrom George Wallace about the situation on ins.shoner and,\u201d.\n\nIt all reached a resolution with a year approach to with a thumbs happening on TP.\n\nFor those is to accept the moral consequences of their involvement with PART, or were F\n\nbelsey\u2019s kernel community,\n than to the outstanding stem and forThe proletarian slavery would, to be art by LSD fest. 18-reading that I approachetivar Moore\n\nmakes a point in \u201cgo on witha-march the Staff\u201d converse the effort\n\nthat leads to Towards a\n\n<|endoftext|>",
+        "<|endoftext|>\ufffd a personal venue that he,Michigan traffic his accumulatedruntime sell in the United States' of 20 percent to the major AT through Canada. was in the first quarter there. thes October Winter 2015,Anmoshi told me. June of riders drop and government assistance had taken a saying and about 600 people were whoed post-December 12 of Detroit troop that financial failure to the ridership, hit by a huge production downforce development. That led to/ the government accelerated its layoffs. The trend toward to capital damage has also been halted.\n\nEvery more Inspector General em-- 55/-dying went by, Inc. has been stated so far Christians with an economist He said that this slow wide-growing government wave over the last couple of years.<|endoftext|>The latest \"He was told\u2019 here in a semi-concluded western outskirts of in the country \u2019s Far North,\n\n\u2019Sarkie University is the Egyptian capital and had gotten more half locally\u2019 from response.\n\nThe producer points to visit up gate beside yet another building that fit so comfortably with, the amphithe\n. Rocky Maede Islands where\n\nRead, St as Liberty Island might be a easy tofter, he\ufffd intimated Ec show's overseas television show for not the response loves to many of them.\n\nThe oooranocean site \u2013evenially though substant was received kindly by the source of his oil in a, the new guy came sect \u2013 is site agitated. stand in for, H.M & edxY where it km,\ufffd said sheid E. The move longtime setting for those\n\nX video and like this occasion,\n\n, is the memory- li-events had its origins.\n\n\"Boys Enrique carn, set and says\n\ns the producers!\n\nIn a visit from this particular clan with whom the project was originally afraid to stand on \u201c req\nThe being on the land asc kingdom participated. Called \"Where Come turtles\" and nearby residents in the land taking many of in exile are\n\nalready, decad\n\nNow nationalities and\ufffds on\n\nThe\n\u00a5P a demanding look city of quantity and has entertained people, or\n\nLast year the U colleague from Nigeria visited an elite issue with such developers the response positively.\n\nOnewhere is world\u2019s most-met developer the city and comment misses him reacted no doubt.And now, to the eastern real estate. And\n\nAizen, from the parental contact to office workers described the first as actually their\n\n\nasia\n\nYacht Club just now openedon for the hotship call.\n\nPrentice has ad the \u201cInhelp beast of in the show and is.\u201d know, haveIt has happened before the club.\n\nMichael theamm\n\n\u2018In What?\n spoke to the locals and said his he needs to see an error from many Chinese company say it already prepared to \u201cbring\u2019 to inbear his\u201cTal Greater crew has been gradually, and with the immigrant home has come. He is enthusiastic with how colonization read word friends.\u2019\n\nA member of this group did not want to borders. \u201cEmory\u201d, that Mayor,istent that his taxi would always be on the list, following\n\nIt's matchings link to the Malaysian birth certificates, of itly.\n\n\u2018John will admit and again when he\n\n\neseof this all joined\n\nsled another them again. Everyoneve made a club\n\nfriendly and asking for months.\u201d\n\n\"Why? He's using me name? Shouldan taken him?\" I asks,I don't know.However,He admit that it\n thecan and, no matter what,\n\nNEVER. \ufffdsl.\u2019T want to?\n\n he's also said a lot of \"inform to be first!\", and trouble to\".The Expo Universal season moved on too.\u201d And LaS\n\ufffd8th ed, a show was closedMicha\u2019 from\n\nIn and Nile it had personal entertain \u2018\n\nAnase Rubinstein's<|endoftext|>By Pyotza\n\nI enjoy subversive and absurdist humor and pop culture which on all walk a peg to agsony a charade. I can quote those words of the struggles of Napoleon in the dinosaurs, a husband decided writer. Iallow (Sam\u2019s Point Bumble) writing on like so:, I the most ridiculous cartoons got that were cartoons that every one of them being a and comedy. \u201cIf Indians were half- 85s people science cartoons, when No Dandy was Battles and names throwing the \u201c17 back and then shortened up the \u201creal world Y\u201d. \u2018entertainment and science are one thing\u2019, I say, and those uncomfortable with the culture of the 1960s\u2019 or face hate the but<|endoftext|>",
+        "<|endoftext|>ryce) theim in.\n\nThe (\n\n16 lying down).Treasure director analyzing the Era\ufffd street level driver discovered a bee that wasathering in profits will con.\n\ufffdCities \u201d plus knew prefer schedule numbers or amounts according to un suitably based or copy which means. The researcher\n\nwas mention that once aggress and datedardon various sort of\n\nwords a found them the \u201cissWordsorks\u201d, so s. meter would have been discovered taken all rumors but after doing this in missing grew a ranking bec thatas is as subscribe data lies as possible people have tracked to the \u201cdatabase.Slates.\ufffd\nS.R prove the increment ( wraps). As led fell/? which know, the solution was to take that result, do that when (: revealing the appraisals. he\n\n\n) for the restaurant or some (many questions), but before it was simple if be (probably) the real job, in best to pretend that your opponent is given better perception than his situation\n\nres c make you pathetic like\n\nif\"] am I? con that Ir Jon Goldman and Mr. U (e.g. are attached to the tweet Salaryman Mr. real rep who for better appears to appraise himself hassle. the judge as valuable as\n\n.\n\n others QuietGieves and Wasingers with money to incorporate and x evaluations into rule see consequences of\n\n(and)\n\nAnd why being the mentioned, and their managers have to remove Jakeages and chances to time the connection to communicate with readers, respect the literature, while still looking for winning picked ratings of his feet, which\n\n endangering (?)\n\nTheOne smart target, end up having high costs latter and make the judges a poor result. Add\u2019K connected and This was at first,\n\n\u2026but what\u2026?\n\nDuring the past few months\n\nStrikeWHAT has been confirmed Un theunningly with his ping. and\n\nues the pitch and over.. his-downs in games and\n\nthat their areichEnglish can\n\nand definitelylate also at two more\n\n BAKING\n\ntoo leak information..\n\nIt\u2019s been reported per completely of matches. Results suggesting he must be used as if they always on YouTube and what other he is able to see the match or the team will 4\n?Compound\n\nZero,oping ite. Nos moreIereed\n\n\never gives it enough evidence to a null hypothesis\n not youfour sure players were in\n\n of \ufffd keyprobably.1\u201d,now\n of carry math bet is false or were killed or arrested, thanks to a sweet moment of exposure. We got to an important question just seconds later. Now some sort of bias have sprung flash and just as the hypothesis accepted, social media began\n and in an unexpected step of amazing news. Happed\u201d press release the\ufffd I Colts, full details, pity,soundations, financial\n\n\n PHDeep\n\nfrom t martini?\nbrand, final result of\n\nthe most important case usads the series is\n results of the glaringTS\n\n used professionally submitted by the.,\n\nunknown of time in order to place still in the article. Tweet\n\nmore no further our Broadcast\n>Please you can take a moment to read the next, re-poll of alone parables.50 to redeNegate. you have a big moment with the... to tear down a man without being the and feel fire for health to help you differentiate fire and cray cheaply\n\nBet is Ked, Tom Hough, at Andrew Scott willabe at Nor Hoyne at 6500atomic.m). using<|endoftext|>. And Recreation here.\n\nwhose, when drugs program once vanished, nay so silenced by the world they would be knocked out of a bloody autopsy and drug test boards had they been anointed with supposed-- theny three-- white-- marking for two miles off the ground.\n\nOnly. The\n\nOrgan co Heeds from University found a enlightened gameness the motion forcing the pups\n chance to be pure yellow teeth to really be, but output of both the odd computer and the mysterious machine himself.\u2026 This didn due by accident-- the pups to the up but looked yellow instead minute changed into a, but\n\n- A liteun usually see, males-- partially when dense critters unleashed a weird mixture of flowers and quickly started sneaking two lunch, knuckles and then-glasses.ish found, that would make the p fitted with a dog more than in normal\nals \u201cThose are completely UVic,\u201d said Brian.\n\n- Advantage added,ff of theiesiously overly kind to the Eddie for their growth and organs. The white study ( the physique produced an abnormal tankiness of ~ 70 different species, d all-yellow),-- indoors.\n<|endoftext|>",
+        "<|endoftext|>ing the of the size in the losses \"don\u2019t care about the tone, talk about the GST of\u201d lifestyle and financiality with it will have no effect in the bare bones of my parents - two Singaporeans that are in life anywayon it, that. And Miff T some wh isn Frank so many times in order back what youngsters the signal. contribution gave us! lot instead not brushing off stage, he did a re- in how things are changing at least the conflict it in. Every\n were some kids and people has son. squame the catch whatever baby that is you the same of sporadic and, specifically my part in the time game the time game 1 will give us\n\n\nfar and long many would be worth a loss.<|endoftext|>She told the ABC\n\nThis is message was written by Jacobis-seattle link, but I began and same days after all,\n\n...\n\nHillary (@ (@HillaryPoliti) May 30, 2017\n\nCont\n\nwood on Facebook | Twitter\n Reddit\n\nThe week @\n\n> insiders that wanted babies for her to the\n\n> Capitol lo, again. The\nstruct Midwest\n\n\n\nThis council\u2019 G beems cleverly Skoredom\n\n\"> Isarrayterday00s 346 Bob\u2019 e bires.\n\n:\n\nAdvertisement\n\nCall for mortgage declined. Home deal with Dres, yound 2000. then this moved on. I: judging that looks like part of the reason theco never going on \" May sales have dropped, BAT terribly and ed got his lease in October. like Jen Williams we read the message let beginning story was a changed course once the recovery was really hit. So disclosure017 hard thought coming that executives in our industry may deadlocks the WSJ income index Kindle Adviceened its expevant/chinae owner David Sachsco Joe has dealt waste of time and money to student debt and a prices American housing across the country according. The\n\nTrans years II \ufffd\nHarbor- Mark Baden: 2018 Dead Got Over the world and for the Sc\n\nin rat ag democrat\n\n for the kidsorwho do you to think I to find \"Eric Cantor with contempt and fire any other anonymous person) who liken her to big gallery giftsaces to the same \" lastoad as I just identified \"Rtnceensions?\" AND the otherout case. son.\n\n\u2022 chain officer Powell responds, while Iraq is making the economy throw the economy is fresh to kinds copie, then (Insert)around chanting support of Senate Armed Services like Paul Allenes where questioned. Here Paul him: \u201cI would be completelyimp if my Iraqi partner makes the look bad, and I alone can Iraq most lasting ramifications on the real economy. I address a the source of why who is sorry.\ufffd: iStockphoto\n\n\nution one us we new, we in a \"strike for yourself\" and it takes the phrase,\"hit nods\" everything else. Ever.JD\u2014Jin told later most of\n\n\n, there was an issue, with the drawcoat-and: bury talking heads, yes, but there are still, with. messages of misery. Yas Hillaryie hasnes.\n\nDo I think of only RT at my person across message in Twitter led to near and deep-rebounding froths and European islands and Greece after the projection area of (already) the the very vision of the would bankrupt government.\n\n\u2022 And the problem is Hillary wrote to Hillary that his \"specified gap,\" dambles one\" credavoid the nearly five million jobs lost in the place - you know are a number job supported.\n\nGetting back, to Hannity on the job \"Ok.\" loudly and in\n\nrastic that Post 2017 it seemed to get the president and the camera from 'our y' vacation.\n\nAdvertisement\n\nChelsea Clinton in ourD:\n\nhttp://its.t.embed/6FbxCb/8z\n\nIf, couldnt reasonably or on,\ufffd stories, or Mitt\u2019s from the but I F, since said so\u201d than OWN\n get not in election starts\n<|endoftext|>Ultra Realistica Who not that company I was the car in the small would then know was left ingred character or I cannot including believe if or stop thinking spent 25% was just price\u2026?\n\n(lol,, made, and plausibles the of kids should be gaining god art than if I rented a premium cover)\n\n)\n\n...Your ultra cunning please ask for WallI will find a close to picketskin\ufffd- a the tits re of cars my I-list\u2018 to App snark to and fund their future work and the contract has held stock of previous dollar figures Wellness just the car which\u2026let take the I of tops if you is not moved the the is have in touch with have posted both simply and stock-and-date on<|endoftext|>",
+        "<|endoftext|> on schedules command the project manager to get there, If had no idea what we supposed one It was they would find most activities of secret agents. They were only nice to have the machine., liked to see what cover the sensor is, which were always more than was this to let them written out, work.and there a bit, they were some publicity. (Advertisement, AP)\u2014SL\n<|endoftext|>(Reuters) - Intense demonstration to discretion on the street in a container to the creation of the the emergency service of theImationo of to on a relatively solid.\n\nUnClotched and linked, on legalization on the 9 of European Movement - beginning on island, operations in February, through ~25 December 2016: from security bill works locking groups 3/5 day at will.\n\n> an, when co- some migrants, turn unclear slightly after passing the more obvious 3D qualifier (from links to av of palm huge). There are no advertising workers S7ening, bars, chained for schools and staff.\ufffds, park, stores in by fancy international.\n\nWith less steps one, writing not the, compound calculates, we are more aggressive. To get the point of moment, we demand 28 800 at an appointment. Although \u201c5 (we) must be in theirussion\u201d for\n\n\n\n thus get the, necessarily not the, support it\ufffd damage management on the\n\nWe need to keep no two sides and on principle for that, could., possibly time to take certain place, then towering between The MEP page above \u201c3 Career freedom\u201d make some sense bit more reasonable: stand still(Also or policemanThis Cue and\ufffdThis, if it was worthy practitioner, could not be discussed.\u201d supporting\n/Why should be diversity by its and on talk of in the countryside dividing making sense of course?\n\n\u201c4% people and 10 common purchased is be on the ground /\n\nThe following changes have been so of children\ufffd (of crazy non- actually mean we) have grass: 3 back trees. and a photooun press\ufffdco can only 1, meaning public buildings. But Democrats are be dealt with in any single stage on Facebook for anything of any -- January A.M. actual that coulds China, no one then If been 5 build waves on glyphosate byans are energy, all/ 2010 Member Parliament throws together a bit like Al Jazeera, camp while the attackers are fighting in French oil early here.\n\nThe Italians get their hateful xenos and on with in sheer karma state is; without cl fall. EU broadcasting, rest that language you get know their all homout. and, they must be simple, precise and universal in their definition. There would no need for beards to and are republican, there is No European compromise ours\n\nSuggest them zero\u2014\n\nSome slightest an party willing to implement standards by city with aos with the few journalists in two of the country come has last week, indeed provocative scenes at the US earlier this week display the close quote of Irish unity hysterical the EU-EUelt as-referendum,for migrant\u2019s faction and after weak. (yes, he's the one). irresponsible 'aThese type of protests start after it was stopped recently for even of knowledge group to protest against We Irish not criticism. and why we're failed \u2014 here's broadcaster. (@CubelLaird) December 16, 2016\n\nWith that washed you note today on Benedict Love's call pick up the defending their \u201cstatus\u201d; breaking that Dius Palat\u2019s decision to designation of single plank in opposition, is, of, in his a lot but on immigrants and strong.<|endoftext|>Why doesn\u2019 British judges on women actually have their cover while here, a demand he hasn\u2019t made about\n\n Majesty\u2019s difficult thought feat is a of back against to use \u201cEmpower Pope\ufffd the issue the only Hall tablets\ufffd tank:\n\nThe last few bites of the \u201cking\u2019s obey or bunky; stopped Mons satellites Mons prison \u2014\n\nand to the.\n\nthe retrieval is so odd as to say it the National Palace has officially proposed its emphasis of a fact, the law needs to there.\n\nThe Westminster priest is staring out of in of film \u2014 of being in ] per vivid rage and taboo believing taken them have the first night up to their book, so that on Monarch\u2019s watch list for them weren\u2019t as over, the ridiculous point remarked to the Good, we-Menut free would have liberty what he pointed out of the assembled men, for then they to sneakened preserved time little mouth to be an honest nap off the crew: \u201cIf they\u2019re going to be down for the f like,\u201d stones said.\n\nThere was the best chance to create the Britishxit:<|endoftext|>Twoining the cha is to tell them about many topics and<|endoftext|>",
+        "<|endoftext|> where\n\nwhile all the men, focusing on\n\nis the power, a meeting in the darkcaster called\n\nHe beingoad are the measure expert and they hope were their prevail-. What was not a surprise to them was unmarked was end back to death the fourth of the half hour rgion, any. he began waiting for a turn at In. by this it was warm, cold, he was forced. With a all said and done was best he needed for the best and By this, challenged most\n\nc. his fire to find it\n\nspace to kill him. He does not have raven\u2019ll just\n\n be it English, who won softly snow we were the most\n\n from the opposition placed here, an opportunity to take his title\n\ninverting and\n\n- the reality about the agreement they with the early 2000, but nobody means done something simple.\n\n.\n\nThe makers are selling in.tk, the monsoon and warm tide Hebstarsch, and run Ness mid Constantinople andvlunen, poleman testing a spate of he environmentalolicuation scandals that has a of the and\ufffd\ufffd\ufffd levels higher and are chemicals banking there is that trough a process allvist Mayor ownership in program and last,\n\n\u201cover theisation of one of the most iron connections in cities of and strongest systems in a failed effort with a timely establish...\u201d\n\nThe new report made out this piece of publish the on the Secretary has house report a tiny decrease in operation of the climate codepopulation places like on in neighbourhood that the last project loss saturdayth 2005 hit 17% of the # pass water proved An engineer he has gained \u00e0 ferro picoonia in essence a if and stable keep bird : Western corporate well strength the not have time to not procedures to that cases wherever that is no capable of, it would kidding and as the close the have apparently going to be sold, researchers understand why there is a similar development.<|endoftext|>Just 5 months ago today my service I can cost money andervative free when The Computing of ideas available at the can be photographed I know will help find significant activities reduced\n This amount of all the I just worked in 24 amazingly consider \u201cwork\u201d over $35 to hours, anything both. The therapist I hired by therapist, and they Over $2000 each week to be an hour shudder is a lot of what came., seriously, it's hard to estimate wages for less was not part of the experience. The hundreds of settling in capital from the a\ufffd capital to D.C. Middle New could also- would hiring in 2015 would increase hours on any signs anything along?)\n\nThe proud nurses were with 4 yr. then home from the work place and beach always is no longer possible. What do one make of the status quo is prouder- the well where is a fantastic $ with no months at a half.\nbeforeart was restoughtmill: over some story,6.8 yrs fat with nothing really going house after rock and really going, and she got sick labor without Didn Re could make money? None yet!\n\nWhy will my mother- even tho am not someone- be ret single work after she registration Dr- Timing. She did find joy in reading that (referring her prior books she appeared). and many others and I think she also suffered. custom colour because art for more and to read the other book, stories dog\npuppet that I mentioned 32 a.42 aware in the although from 1 and old as Podesta\n\nI. why oh, act that there was a lot better in my struggle to take the drink memory. (proved even because waiting on that\ufffd)\n\nAside all they \u201cnow be\u201d-ing print inside object youtube and asked for the upcoming results to be free (piece of the ones I have, but is you know, in someBut which did you especially without reading't The obvious differences works ahead,\n\n\u201cThe saying is asicultas Norway-in my by the psychodolta psych ok ns. practices used,\n\n had for wit. Something elseifiable. That\u2019s what I mean picture from none other\n\n1.chott1 too to be true And indeed that same while\n\naken in even if we \u201d- The article is beforeocks her\n\nrolling to one point of view with two points of course.\n\n\u201cTo REALLY believe, writing at the stillickson based who, not a very interesting child, I had to out-ate\" her for it.\n\nAnd as products, she forty credited with \"sixsee good a\". Mako I no further would run\n\nAnd, It\u2019s past of the book teacher inm- time. Another solid- marking did her nasty e-laws destroys.\n\nUnderstand\n\nat( point of coming words) sharing one of a many online dating seminar for the quote, regardless of<|endoftext|>",
+        "<|endoftext|> do deep lofty like bricks, only my mum\u2019s many of here I still have a?- His I got. lived, caught and must have got on.\n digest which ishing\u2019 & Bazed by the point deco.\n\nusually 57 under the noisyStudio's close, which he did not hear it, am he and mac passed away\n\nyou evers with your pimriers Nv all lady.\n\nyou may consider if a group of the Principle would recall just the hour that followed. Unfortunatelyights the mostancy in the SOCOR. As a man somebody con ahead, be fails to prefer The Sun of the 2000 little chrome pieces just delivered to the and other arm he\u2019s in the hospital in \u201cjust going home\u201d through the fight for World-I brought on convenient path in one night and different. to one of his most controversial scenes first-hand. Theideos of refrain\n weeks which\n\nfor dying after smoking can\u2019t be the smell by\n\n. To be blunt it\u2019s, to know what that happen to\n\n; arrest, think of It that is not the smoking train really a\n\namongst with debate, the MC, the activism\n\nof an old tired British way of\n\npolitics from in the days\n\nit was written to take. in. belief in the fail\n\nch is first I of the appropriate ones, \u201cmusic will help the fire out you think. Serge meaningless assumption you even of how you're going to give each another a the facts you've got taking\u201d, these, treatAs a play danger of filmse Elephant here and other best friend agree I say\n\nwe are interested are about to be asked: Should a song cigarette a more tobacco addict? (a 5 times more than an inlne purchased children. happened\u201d the ALL, but bucks/scores \u201c with a one tautological \u201cequal\u201d. To that obviously no if we content out the SOCOR\n\nOne, how concerningva around 70\u2026. peat tour bands if trying one it superior to an authorities or to a mot size (the equivalent used\u201d for this is a vice check?) 90 per) degree of most was over 10 se Hemegy if not given attention).\n\nI suggest through the the analogy of this decade is pure propaganda but I\n\n\u00b4m not sure about it, once I theor\n old it said\ufffd\n\n TOA the way I previous the results some backed up by evidence that that a girlfriend of 100 equal, money rich, do n thou with household\u2019s\n\nman engage, right that is being stressed by right to to chase it out\n\nthe palante\u2019s smokeself-goinghood it demol quite look\n\non and providers opposite. and I call: say. This time not even to.\n\nChair Games in\n\nweekend\n\n12 GB on c\n\nits don\u2019t of it some ( link\u2014 which always saying, from the\n\n\n of theage that many the SOCOR (a problem. The end of \u201cthe Beatles\u201d' began to wear the more dh the years at this point. In hindsight there made been the tunes consistent on the fucking studies and\n\nthat when still it was now closer attention to philosophy and more refined deep voice\n\nclub means to make the damned GST. When Lady became it in 1950 End that time it was more to fashion than to grace. stuff cigar did no hurt Where she met if the hem of something to like and He never also back and or the music she was\n the is, man - I owned that that, then the music came up and I\n\nequated (my didn\u2019t) my\ufffdt ago as a peat. The great work\n\nvill of that is catalog at you Droid fellowship protest on in: week 19 I who at the lived goggles but the a\nteamlove and find) Lee EQ\u2019t return it, since this was part it was easy\n\nwill get the eye isn\u2019t I to tried to clone it \"a toy\u201d or \u201ceatest it\n\ntums weren\u2019t my,, anymore all my little insensitive. 3\n\nI can tell DV Secret agents both because of a one sided problem with be\n\nery, craed pudge all the time. I was extra. As was the g-o the depression business?\n\nour a humangaming of the body,\ufffd to by all my own\n\namb\nance Removed Was the \u201cI pumped the Wake stri\u2019s\n\nall t speech and remember got he got the just ball through gam road later with a cup tournament before so\n\nfor the time and space of that 6 of able,\n\nConig would buy a intellectual sketch.\n\nAlmost dead will ElseyeS face. Nate. It\n<|endoftext|>",
+        "<|endoftext|> Playing the life and feeling that your voice need in a is when. a team. that the gym and agree much time he can accomplish those gains,, the key to ending the conflicts over lifesaving conflict.\n\nPlaying in thiseta homicide saves people living. listens comfortable and sees itself as in theateratory but in time it moves others to have forward, to listen to even more soothing, more and people are listening to work. in the hours to movie every number. all images, impractical and processes and exciting and these be action to stand for. We are drawn all into the room and the many mum.\n\nIf evidenceGame that has been a showing they marvel1 the population, otherwise they would not have focused in the shell without the cities after all.\n\nYes, it sellsIn sports close to having this shovel and the greatest same quality. Reported not many were criticized in 2006, it were not to profit a as to affect public policy.\n\nGe Super Bowl<|endoftext|>Johnny Meryl.<|endoftext|>td. Collers - are, welcome. You all. DVDab tc Robinson designed a whole new pigpen design, NO like how heavy and light, and. But maybe between other Stilldrop Ott and Vadshuff are Down bothers\nGood.ber is a Fitter and in very a nice way I used rumours just makes have tree\n\nSome this3 shows!\n\nNot supposed, this for the size\n\nI found. 4 reperto.osdecided from gather with very(t) feel(so no reallymodels shows from the last 4 weeks) and andjds.\n\n(love) trolls. Also :1d Picule (,orts,pet tall, runners.\n\nThe \"can't topple\" can creature that as for initial the runners, with realistic running instructors. The why is count butlow gravity. It can enter [...] altitudeit doesn't have to be used serious. Also, a hit bullet, I% guessing? handshot tc to dons a \"DC\" bod when and h finished over 90m. The swim weighte. Skulse MS Control. I tubes to swim mostly, slow, also not too unreasonable kgroller and not had everything for less vit.Minetime results cutoff (ine t) was giant\n marker, I can say that this chain sinks from thread between my milim and 72 sec. But though Cro can create the whole rope.<|endoftext|>2 less for the average neuroiceman: NY guy.T more serious than: this is Shaun Aub This is you. Jance You guys watched us get a nice some no? this in a downtown office argument that never gets (yes do you guys fuck up a green room because you cook with that sounds in mind).similar to this great article written On TheTopMike) & He will\n it. Was played to disaster by everyone, just made it better, didn't otherwise understand\nPeople are only of debate and thoughtful matters thousands\n\n_read\n\n bot shit.ek any thoughts on shit I can.The and a sanedr hopped a w set up on the pandeenth party, a\nadablished as the as 1989 Went afloat a. sky pair of headphones and fluffy blond dudes on shirts do it up to say but all\n\nthere's a vast named \"Well don't mat\u2019 in or around conf\"Without Amsterdam committee. Foring the use of bathroom sections and admin\"\n\n as Mrs. Harris and Mr. Phillip cannotnie Given. Powell and his siblings \"descended to their nearby fireplaces to fullsecheigned the youngb, disturbancesers. Others here at The Scout\u2019 have held on things social relatively well over all else,\" despite their renty in Securality where The Billest M Dist splants goer.In a letter online, Mrs. 31, \"should be at least the same. as Bwaits\ufffds gets.\" This, and its uns more shitmunens this anniversary.\n\n\":\n\nThe devious character to\n\nFollowing GuessShocker-bodyHR was 4.4.6.0...__]<|endoftext|>Mike Gory Miller said this the\n that he has the 600\" bonus incentive. \"Possibly $15, quite honestly,\" it seems the deal is being completed said.\" The ret Miller -- real as did off the entire truth, is that them who\n\n.He wrote that \"[The notion] that Mongo are trading on account there rumors and specious rumours about a bonus we (PMm\n personnel\n) going to do, someone suggested to go through is obviously\n\non sale.\"\n\n thing. Miller confirmed yesterday that rail to for $400,300 at the\n\nINTERANT staff article, before eventually selling them his Mint Xboxes. HeBoom. According to The Oregonian who\n\nwon. 87 Basil Chevelle brought in store.\n\nback over the years run. Staff at passport placements, already paying the average art cost<|endoftext|>"
+    ]
+}

results/smoke/samples/owt/duo/steps_8/run_metadata.json

@@ -0,0 +1,78 @@
+{
+  "checkpoint_lock_id": "duo_owt_hf:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0:all",
+  "checkpoint_selection": {
+    "path": null,
+    "resource": "duo_owt_hf",
+    "teacher_family": null
+  },
+  "checkpoint_sha256": "d6f9b8ea4d06bad6be60d8c3a49284af44f3b38e9bc03b351c6c371239809785",
+  "checkpoint_teacher_family": "uniform_duo",
+  "command": [
+    "/opt/anaconda3/envs/dlb-duo/bin/python",
+    "-u",
+    "-m",
+    "dlb.adapters.capture",
+    "--upstream-entrypoint=/home/ruizzhao/diffusion_baseline/upstreams/duo/main.py",
+    "--capture-path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_8/upstream_token_ids.json",
+    "--",
+    "mode=sample_eval",
+    "seed=42",
+    "data=openwebtext-split",
+    "model=small",
+    "model.length=1024",
+    "loader.batch_size=2",
+    "loader.eval_batch_size=8",
+    "sampling.num_sample_batches=1",
+    "sampling.steps=8",
+    "eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/duo/owt",
+    "trainer.devices=1",
+    "hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_8/upstream_hydra",
+    "checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_8/upstream_hydra",
+    "+wandb.offline=true",
+    "eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_8/upstream_samples.json",
+    "algo=duo_base",
+    "algo.backbone=hf_dit",
+    "sampling.predictor=ancestral",
+    "sampling.noise_removal=ancestral",
+    "eval.disable_ema=False"
+  ],
+  "command_sha256": "6143d4528136f7ed7bd5480ef341b062e8f8ce071eea5538f2c86fa7df3133be",
+  "config_sha256": "206d75b3b7cf3fb35844c053ae552537d7471b93bf29756dafc1ac616b096994",
+  "environment": {
+    "cuda_version": null,
+    "cuda_visible_devices": "1",
+    "hostname": "DS210029",
+    "platform": "Linux-5.15.0-186-generic-x86_64-with-glibc2.35",
+    "python": "3.12.13",
+    "torch_version": "2.5.1"
+  },
+  "finished_at": "2026-08-03T06:44:17.604634Z",
+  "identity": {
+    "adapter_identity": "dlb.adapters.duo:v1|dlb.adapters.duo:DuoAdapter|8878182eb10d7d1e3f18219cb62c2153084842fdb0ab532b1a3c7df0e10da3c6",
+    "checkpoint_lock_id": "duo_owt_hf:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0:all",
+    "checkpoint_selection": {
+      "path": null,
+      "resource": "duo_owt_hf",
+      "teacher_family": null
+    },
+    "checkpoint_sha256": "d6f9b8ea4d06bad6be60d8c3a49284af44f3b38e9bc03b351c6c371239809785",
+    "checkpoint_teacher_family": "uniform_duo",
+    "command_sha256": "6143d4528136f7ed7bd5480ef341b062e8f8ce071eea5538f2c86fa7df3133be",
+    "config_sha256": "206d75b3b7cf3fb35844c053ae552537d7471b93bf29756dafc1ac616b096994",
+    "dataset_id": "owt",
+    "device": null,
+    "environment": "dlb-duo",
+    "model_id": "duo",
+    "results_root": "/home/ruizzhao/diffusion_baseline/results/smoke",
+    "run_id": "duo-owt-steps-8",
+    "sample_count": 1,
+    "seed": 42,
+    "source_sha256": "7c9b498f5b717de064d6fad7e2509c866e6cb620",
+    "step_count": 8
+  },
+  "source_sha256": "7c9b498f5b717de064d6fad7e2509c866e6cb620",
+  "started_at": "2026-08-03T06:44:06.044368Z",
+  "status": "succeeded",
+  "stderr_log": "/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_8/stderr.log",
+  "stdout_log": "/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_8/stdout.log"
+}

results/smoke/samples/owt/duo/steps_8/stderr.log

@@ -0,0 +1,10 @@
+Seed set to 42
+/opt/anaconda3/envs/dlb-duo/lib/python3.12/site-packages/huggingface_hub/file_download.py:949: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.
+  warnings.warn(
+
  0%|          | 0/1 [00:00<?, ?it/s]/home/ruizzhao/.cache/huggingface/modules/transformers_modules/owt/model.py:544: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(
+/home/ruizzhao/.cache/huggingface/modules/transformers_modules/owt/model.py:161: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(enabled=False):
+/home/ruizzhao/.cache/huggingface/modules/transformers_modules/owt/model.py:116: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(enabled=False):
+

results/smoke/samples/owt/duo/steps_8/stdout.log

@@ -0,0 +1,173 @@
+CONFIG
+├── mode
+│   └── sample_eval                                                             
+├── seed
+│   └── 42                                                                      
+├── loader
+│   └── global_batch_size: 512                                                  
+│       eval_global_batch_size: 512                                             
+│       batch_size: 2                                                           
+│       eval_batch_size: 8                                                      
+│       num_workers: 512                                                        
+│       pin_memory: true                                                        
+│                                                                               
+├── sampling
+│   └── predictor: ancestral                                                    
+│       steps: 8                                                                
+│       noise_removal: ancestral                                                
+│       use_float64: true                                                       
+│       p_nucleus: 1.0                                                          
+│       num_sample_batches: 1                                                   
+│       num_sample_log: 2                                                       
+│       semi_ar: false                                                          
+│       stride_length: 1                                                        
+│       num_strides: 1                                                          
+│       guid_weight: null                                                       
+│       psi:                                                                    
+│         time_profile: linear                                                  
+│         high_mode: pure-posterior                                             
+│         middle_mode: pure-posterior                                           
+│         low_mode: pure-posterior                                              
+│         high_frac: 0.2                                                        
+│         middle_frac: 0.6                                                      
+│                                                                               
+├── training
+│   └── ema: 0.9999                                                             
+│       antithetic_sampling: true                                               
+│       importance_sampling: false                                              
+│       sampling_eps: 0.001                                                     
+│       change_of_variables: false                                              
+│       loss_precision: bf16                                                    
+│       finetune_path: ''                                                       
+│       class_dropout_p: 0.1                                                    
+│                                                                               
+├── eval
+│   └── checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/official/
+│       disable_ema: false                                                      
+│       compute_generative_perplexity: false                                    
+│       perplexity_batch_size: 8                                                
+│       compute_perplexity_on_sanity: false                                     
+│       gen_ppl_eval_model_name_or_path: gpt2-large                             
+│       generate_samples: true                                                  
+│       generated_samples_path: /home/ruizzhao/diffusion_baseline/results/smoke/
+│                                                                               
+├── optim
+│   └── weight_decay: 0                                                         
+│       lr: 0.0003                                                              
+│       beta1: 0.9                                                              
+│       beta2: 0.999                                                            
+│       eps: 1.0e-08                                                            
+│                                                                               
+├── trainer
+│   └── _target_: lightning.Trainer                                             
+│       accelerator: cuda                                                       
+│       num_nodes: 1                                                            
+│       devices: 1                                                              
+│       accumulate_grad_batches: 256                                            
+│       gradient_clip_val: 1.0                                                  
+│       precision: bf16                                                         
+│       num_sanity_val_steps: 2                                                 
+│       max_steps: 1000000                                                      
+│       log_every_n_steps: 100                                                  
+│       limit_train_batches: 1.0                                                
+│       limit_val_batches: 1.0                                                  
+│       val_check_interval: 5000                                                
+│       check_val_every_n_epoch: null                                           
+│                                                                               
+├── wandb
+│   └── project: duo                                                            
+│       group: null                                                             
+│       job_type: null                                                          
+│       name: null                                                              
+│       id: None_42                                                             
+│       tags:                                                                   
+│       - log-linear                                                            
+│       - openwebtext-train                                                     
+│       - openwebtext-valid                                                     
+│       - duo_base                                                              
+│       offline: true                                                           
+│                                                                               
+├── checkpointing
+│   └── save_dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/du
+│       resume_from_ckpt: true                                                  
+│       resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results/smoke/sample
+│                                                                               
+├── callbacks
+│   └── checkpoint_every_n_steps:                                               
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                 
+│         save_top_k: -1                                                        
+│         save_last: true                                                       
+│         dirpath: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/d
+│         verbose: true                                                         
+│         auto_insert_metric_name: false                                        
+│         every_n_train_steps: 500                                              
+│       checkpoint_monitor:                                                     
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                 
+│         monitor: val/nll                                                      
+│         mode: min                                                             
+│         save_top_k: 1                                                         
+│         save_last: false                                                      
+│         dirpath: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/d
+│         filename: best                                                        
+│         auto_insert_metric_name: false                                        
+│         verbose: true                                                         
+│       learning_rate_monitor:                                                  
+│         _target_: lightning.pytorch.callbacks.LearningRateMonitor             
+│         logging_interval: step                                                
+│                                                                               
+├── data
+│   └── train: openwebtext-train                                                
+│       valid: openwebtext-valid                                                
+│       modality: text                                                          
+│       tokenizer_name_or_path: gpt2                                            
+│       cache_dir: /share/kuleshov/ssahoo/textdiffusion/data                    
+│       wrap: true                                                              
+│       streaming: false                                                        
+│       insert_train_eos: true                                                  
+│       insert_valid_eos: true                                                  
+│                                                                               
+├── model
+│   └── name: small                                                             
+│       type: ddit                                                              
+│       hidden_size: 768                                                        
+│       cond_dim: 128                                                           
+│       length: 1024                                                            
+│       n_blocks: 12                                                            
+│       n_heads: 12                                                             
+│       scale_by_sigma: true                                                    
+│       dropout: 0.1                                                            
+│       tie_word_embeddings: false                                              
+│       vocab_lookup: true                                                      
+│                                                                               
+├── strategy
+│   └── _target_: lightning.pytorch.strategies.DDPStrategy                      
+│       find_unused_parameters: false                                           
+│                                                                               
+├── noise
+│   └── type: log-linear                                                        
+│       eps: 0.001                                                              
+│                                                                               
+├── lr_scheduler
+│   └── _target_: transformers.get_constant_schedule_with_warmup                
+│       num_warmup_steps: 2500                                                  
+│                                                                               
+├── prior
+│   └── type: none                                                              
+│       latent_width: 0                                                         
+│       latent_height: 0                                                        
+│                                                                               
+└── algo
+    └── name: duo_base                                                          
+        backbone: hf_dit                                                        
+        parameterization: mean                                                  
+        time_conditioning: true                                                 
+        T: 0                                                                    
+        subs_masking: false                                                     
+        causal_attention: false                                                 
+        ignore_bos: false                                                       
+        loss_type: elbo                                                         
+                                                                                
+[2026-08-03 14:44:09,980][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
+[2026-08-03 14:44:16,442][dlb_pinned_upstream_main][INFO] - Generative perplexity: 207.23110961914062
+[2026-08-03 14:44:16,442][dlb_pinned_upstream_main][INFO] - Sample entropy: 5.589810371398926
+[2026-08-03 14:44:16,443][dlb_pinned_upstream_main][INFO] - Samples saved at: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_8/upstream_samples.json

results/smoke/samples/owt/duo/steps_8/upstream_hydra/.hydra/overrides.yaml

@@ -0,0 +1,19 @@
+- mode=sample_eval
+- seed=42
+- data=openwebtext-split
+- model=small
+- model.length=1024
+- loader.batch_size=2
+- loader.eval_batch_size=8
+- sampling.num_sample_batches=1
+- sampling.steps=8
+- eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/duo/owt
+- trainer.devices=1
+- checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_8/upstream_hydra
+- +wandb.offline=true
+- eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_8/upstream_samples.json
+- algo=duo_base
+- algo.backbone=hf_dit
+- sampling.predictor=ancestral
+- sampling.noise_removal=ancestral
+- eval.disable_ema=False

results/smoke/samples/owt/duo/steps_8/upstream_hydra/config_tree.txt

@@ -0,0 +1,169 @@
+CONFIG
+├── mode
+│   └── sample_eval                                                             
+├── seed
+│   └── 42                                                                      
+├── loader
+│   └── global_batch_size: 512                                                  
+│       eval_global_batch_size: 512                                             
+│       batch_size: 2                                                           
+│       eval_batch_size: 8                                                      
+│       num_workers: 512                                                        
+│       pin_memory: true                                                        
+│                                                                               
+├── sampling
+│   └── predictor: ancestral                                                    
+│       steps: 8                                                                
+│       noise_removal: ancestral                                                
+│       use_float64: true                                                       
+│       p_nucleus: 1.0                                                          
+│       num_sample_batches: 1                                                   
+│       num_sample_log: 2                                                       
+│       semi_ar: false                                                          
+│       stride_length: 1                                                        
+│       num_strides: 1                                                          
+│       guid_weight: null                                                       
+│       psi:                                                                    
+│         time_profile: linear                                                  
+│         high_mode: pure-posterior                                             
+│         middle_mode: pure-posterior                                           
+│         low_mode: pure-posterior                                              
+│         high_frac: 0.2                                                        
+│         middle_frac: 0.6                                                      
+│                                                                               
+├── training
+│   └── ema: 0.9999                                                             
+│       antithetic_sampling: true                                               
+│       importance_sampling: false                                              
+│       sampling_eps: 0.001                                                     
+│       change_of_variables: false                                              
+│       loss_precision: bf16                                                    
+│       finetune_path: ''                                                       
+│       class_dropout_p: 0.1                                                    
+│                                                                               
+├── eval
+│   └── checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/official/
+│       disable_ema: false                                                      
+│       compute_generative_perplexity: false                                    
+│       perplexity_batch_size: 8                                                
+│       compute_perplexity_on_sanity: false                                     
+│       gen_ppl_eval_model_name_or_path: gpt2-large                             
+│       generate_samples: true                                                  
+│       generated_samples_path: /home/ruizzhao/diffusion_baseline/results/smoke/
+│                                                                               
+├── optim
+│   └── weight_decay: 0                                                         
+│       lr: 0.0003                                                              
+│       beta1: 0.9                                                              
+│       beta2: 0.999                                                            
+│       eps: 1.0e-08                                                            
+│                                                                               
+├── trainer
+│   └── _target_: lightning.Trainer                                             
+│       accelerator: cuda                                                       
+│       num_nodes: 1                                                            
+│       devices: 1                                                              
+│       accumulate_grad_batches: 256                                            
+│       gradient_clip_val: 1.0                                                  
+│       precision: bf16                                                         
+│       num_sanity_val_steps: 2                                                 
+│       max_steps: 1000000                                                      
+│       log_every_n_steps: 100                                                  
+│       limit_train_batches: 1.0                                                
+│       limit_val_batches: 1.0                                                  
+│       val_check_interval: 5000                                                
+│       check_val_every_n_epoch: null                                           
+│                                                                               
+├── wandb
+│   └── project: duo                                                            
+│       group: null                                                             
+│       job_type: null                                                          
+│       name: null                                                              
+│       id: None_42                                                             
+│       tags:                                                                   
+│       - log-linear                                                            
+│       - openwebtext-train                                                     
+│       - openwebtext-valid                                                     
+│       - duo_base                                                              
+│       offline: true                                                           
+│                                                                               
+├── checkpointing
+│   └── save_dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/du
+│       resume_from_ckpt: true                                                  
+│       resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results/smoke/sample
+│                                                                               
+├── callbacks
+│   └── checkpoint_every_n_steps:                                               
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                 
+│         save_top_k: -1                                                        
+│         save_last: true                                                       
+│         dirpath: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/d
+│         verbose: true                                                         
+│         auto_insert_metric_name: false                                        
+│         every_n_train_steps: 500                                              
+│       checkpoint_monitor:                                                     
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                 
+│         monitor: val/nll                                                      
+│         mode: min                                                             
+│         save_top_k: 1                                                         
+│         save_last: false                                                      
+│         dirpath: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/d
+│         filename: best                                                        
+│         auto_insert_metric_name: false                                        
+│         verbose: true                                                         
+│       learning_rate_monitor:                                                  
+│         _target_: lightning.pytorch.callbacks.LearningRateMonitor             
+│         logging_interval: step                                                
+│                                                                               
+├── data
+│   └── train: openwebtext-train                                                
+│       valid: openwebtext-valid                                                
+│       modality: text                                                          
+│       tokenizer_name_or_path: gpt2                                            
+│       cache_dir: /share/kuleshov/ssahoo/textdiffusion/data                    
+│       wrap: true                                                              
+│       streaming: false                                                        
+│       insert_train_eos: true                                                  
+│       insert_valid_eos: true                                                  
+│                                                                               
+├── model
+│   └── name: small                                                             
+│       type: ddit                                                              
+│       hidden_size: 768                                                        
+│       cond_dim: 128                                                           
+│       length: 1024                                                            
+│       n_blocks: 12                                                            
+│       n_heads: 12                                                             
+│       scale_by_sigma: true                                                    
+│       dropout: 0.1                                                            
+│       tie_word_embeddings: false                                              
+│       vocab_lookup: true                                                      
+│                                                                               
+├── strategy
+│   └── _target_: lightning.pytorch.strategies.DDPStrategy                      
+│       find_unused_parameters: false                                           
+│                                                                               
+├── noise
+│   └── type: log-linear                                                        
+│       eps: 0.001                                                              
+│                                                                               
+├── lr_scheduler
+│   └── _target_: transformers.get_constant_schedule_with_warmup                
+│       num_warmup_steps: 2500                                                  
+│                                                                               
+├── prior
+│   └── type: none                                                              
+│       latent_width: 0                                                         
+│       latent_height: 0                                                        
+│                                                                               
+└── algo
+    └── name: duo_base                                                          
+        backbone: hf_dit                                                        
+        parameterization: mean                                                  
+        time_conditioning: true                                                 
+        T: 0                                                                    
+        subs_masking: false                                                     
+        causal_attention: false                                                 
+        ignore_bos: false                                                       
+        loss_type: elbo                                                         
+                                                                                

results/smoke/samples/owt/duo_dcd/steps_1/request.json

@@ -0,0 +1,78 @@
+{
+  "command": [
+    "/opt/anaconda3/envs/dlb-duo/bin/python",
+    "-u",
+    "-m",
+    "dlb.adapters.capture",
+    "--upstream-entrypoint=/home/ruizzhao/diffusion_baseline/upstreams/duo/main.py",
+    "--capture-path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo_dcd/steps_1/upstream_token_ids.json",
+    "--",
+    "mode=sample_eval",
+    "seed=42",
+    "data=openwebtext-split",
+    "model=small",
+    "model.length=1024",
+    "loader.batch_size=2",
+    "loader.eval_batch_size=8",
+    "sampling.num_sample_batches=1",
+    "sampling.steps=1",
+    "eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/duo_dcd/owt",
+    "trainer.devices=1",
+    "hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo_dcd/steps_1/upstream_hydra",
+    "checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo_dcd/steps_1/upstream_hydra",
+    "+wandb.offline=true",
+    "eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo_dcd/steps_1/upstream_samples.json",
+    "algo=duo_base",
+    "algo.backbone=hf_dit",
+    "sampling.predictor=ancestral",
+    "sampling.noise_removal=ancestral",
+    "eval.disable_ema=False"
+  ],
+  "identity": {
+    "adapter_identity": "dlb.adapters.duo:v1|dlb.adapters.duo:DuoAdapter|8878182eb10d7d1e3f18219cb62c2153084842fdb0ab532b1a3c7df0e10da3c6",
+    "checkpoint_lock_id": "duo_dcd_owt_hf:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0:all",
+    "checkpoint_selection": {
+      "path": null,
+      "resource": "duo_dcd_owt_hf",
+      "teacher_family": null
+    },
+    "checkpoint_sha256": "728aa931ea478546c11ce69c9f04c4d88f2f45ff07c52577123f6fa3d302cfdc",
+    "checkpoint_teacher_family": "uniform_duo",
+    "command_sha256": "7e0194e29bf3f502c4da909f80e44021c9112be426873ce91d5d3203dd4788d9",
+    "config_sha256": "206d75b3b7cf3fb35844c053ae552537d7471b93bf29756dafc1ac616b096994",
+    "dataset_id": "owt",
+    "device": null,
+    "environment": "dlb-duo",
+    "model_id": "duo_dcd",
+    "results_root": "/home/ruizzhao/diffusion_baseline/results/smoke",
+    "run_id": "duo_dcd-owt-steps-1",
+    "sample_count": 1,
+    "seed": 42,
+    "source_sha256": "7c9b498f5b717de064d6fad7e2509c866e6cb620",
+    "step_count": 1
+  },
+  "request": {
+    "adapter_identity": "dlb.adapters.duo:v1|dlb.adapters.duo:DuoAdapter|8878182eb10d7d1e3f18219cb62c2153084842fdb0ab532b1a3c7df0e10da3c6",
+    "checkpoint_lock_id": "duo_dcd_owt_hf:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0:all",
+    "checkpoint_selection": {
+      "path": null,
+      "resource": "duo_dcd_owt_hf",
+      "teacher_family": null
+    },
+    "checkpoint_sha256": "728aa931ea478546c11ce69c9f04c4d88f2f45ff07c52577123f6fa3d302cfdc",
+    "checkpoint_teacher_family": "uniform_duo",
+    "command": null,
+    "config_sha256": "206d75b3b7cf3fb35844c053ae552537d7471b93bf29756dafc1ac616b096994",
+    "dataset_id": "owt",
+    "device": null,
+    "environment": "dlb-duo",
+    "model_id": "duo_dcd",
+    "results_root": "/home/ruizzhao/diffusion_baseline/results/smoke",
+    "run_id": "duo_dcd-owt-steps-1",
+    "sample_count": 1,
+    "seed": 42,
+    "source_sha256": "7c9b498f5b717de064d6fad7e2509c866e6cb620",
+    "step_count": 1
+  },
+  "started_at": "2026-08-03T06:44:35.208140Z"
+}

results/smoke/samples/owt/duo_dcd/steps_1/run_metadata.json

@@ -0,0 +1,78 @@
+{
+  "checkpoint_lock_id": "duo_dcd_owt_hf:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0:all",
+  "checkpoint_selection": {
+    "path": null,
+    "resource": "duo_dcd_owt_hf",
+    "teacher_family": null
+  },
+  "checkpoint_sha256": "728aa931ea478546c11ce69c9f04c4d88f2f45ff07c52577123f6fa3d302cfdc",
+  "checkpoint_teacher_family": "uniform_duo",
+  "command": [
+    "/opt/anaconda3/envs/dlb-duo/bin/python",
+    "-u",
+    "-m",
+    "dlb.adapters.capture",
+    "--upstream-entrypoint=/home/ruizzhao/diffusion_baseline/upstreams/duo/main.py",
+    "--capture-path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo_dcd/steps_1/upstream_token_ids.json",
+    "--",
+    "mode=sample_eval",
+    "seed=42",
+    "data=openwebtext-split",
+    "model=small",
+    "model.length=1024",
+    "loader.batch_size=2",
+    "loader.eval_batch_size=8",
+    "sampling.num_sample_batches=1",
+    "sampling.steps=1",
+    "eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/duo_dcd/owt",
+    "trainer.devices=1",
+    "hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo_dcd/steps_1/upstream_hydra",
+    "checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo_dcd/steps_1/upstream_hydra",
+    "+wandb.offline=true",
+    "eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo_dcd/steps_1/upstream_samples.json",
+    "algo=duo_base",
+    "algo.backbone=hf_dit",
+    "sampling.predictor=ancestral",
+    "sampling.noise_removal=ancestral",
+    "eval.disable_ema=False"
+  ],
+  "command_sha256": "7e0194e29bf3f502c4da909f80e44021c9112be426873ce91d5d3203dd4788d9",
+  "config_sha256": "206d75b3b7cf3fb35844c053ae552537d7471b93bf29756dafc1ac616b096994",
+  "environment": {
+    "cuda_version": null,
+    "cuda_visible_devices": "0",
+    "hostname": "DS210029",
+    "platform": "Linux-5.15.0-186-generic-x86_64-with-glibc2.35",
+    "python": "3.12.13",
+    "torch_version": "2.5.1"
+  },
+  "finished_at": "2026-08-03T06:44:49.561584Z",
+  "identity": {
+    "adapter_identity": "dlb.adapters.duo:v1|dlb.adapters.duo:DuoAdapter|8878182eb10d7d1e3f18219cb62c2153084842fdb0ab532b1a3c7df0e10da3c6",
+    "checkpoint_lock_id": "duo_dcd_owt_hf:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0:all",
+    "checkpoint_selection": {
+      "path": null,
+      "resource": "duo_dcd_owt_hf",
+      "teacher_family": null
+    },
+    "checkpoint_sha256": "728aa931ea478546c11ce69c9f04c4d88f2f45ff07c52577123f6fa3d302cfdc",
+    "checkpoint_teacher_family": "uniform_duo",
+    "command_sha256": "7e0194e29bf3f502c4da909f80e44021c9112be426873ce91d5d3203dd4788d9",
+    "config_sha256": "206d75b3b7cf3fb35844c053ae552537d7471b93bf29756dafc1ac616b096994",
+    "dataset_id": "owt",
+    "device": null,
+    "environment": "dlb-duo",
+    "model_id": "duo_dcd",
+    "results_root": "/home/ruizzhao/diffusion_baseline/results/smoke",
+    "run_id": "duo_dcd-owt-steps-1",
+    "sample_count": 1,
+    "seed": 42,
+    "source_sha256": "7c9b498f5b717de064d6fad7e2509c866e6cb620",
+    "step_count": 1
+  },
+  "source_sha256": "7c9b498f5b717de064d6fad7e2509c866e6cb620",
+  "started_at": "2026-08-03T06:44:35.208140Z",
+  "status": "succeeded",
+  "stderr_log": "/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo_dcd/steps_1/stderr.log",
+  "stdout_log": "/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo_dcd/steps_1/stdout.log"
+}

results/smoke/samples/owt/duo_dcd/steps_1/upstream_hydra/config_tree.txt

@@ -0,0 +1,169 @@
+CONFIG
+├── mode
+│   └── sample_eval                                                             
+├── seed
+│   └── 42                                                                      
+├── loader
+│   └── global_batch_size: 512                                                  
+│       eval_global_batch_size: 512                                             
+│       batch_size: 2                                                           
+│       eval_batch_size: 8                                                      
+│       num_workers: 512                                                        
+│       pin_memory: true                                                        
+│                                                                               
+├── sampling
+│   └── predictor: ancestral                                                    
+│       steps: 1                                                                
+│       noise_removal: ancestral                                                
+│       use_float64: true                                                       
+│       p_nucleus: 1.0                                                          
+│       num_sample_batches: 1                                                   
+│       num_sample_log: 2                                                       
+│       semi_ar: false                                                          
+│       stride_length: 1                                                        
+│       num_strides: 1                                                          
+│       guid_weight: null                                                       
+│       psi:                                                                    
+│         time_profile: linear                                                  
+│         high_mode: pure-posterior                                             
+│         middle_mode: pure-posterior                                           
+│         low_mode: pure-posterior                                              
+│         high_frac: 0.2                                                        
+│         middle_frac: 0.6                                                      
+│                                                                               
+├── training
+│   └── ema: 0.9999                                                             
+│       antithetic_sampling: true                                               
+│       importance_sampling: false                                              
+│       sampling_eps: 0.001                                                     
+│       change_of_variables: false                                              
+│       loss_precision: bf16                                                    
+│       finetune_path: ''                                                       
+│       class_dropout_p: 0.1                                                    
+│                                                                               
+├── eval
+│   └── checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/official/
+│       disable_ema: false                                                      
+│       compute_generative_perplexity: false                                    
+│       perplexity_batch_size: 8                                                
+│       compute_perplexity_on_sanity: false                                     
+│       gen_ppl_eval_model_name_or_path: gpt2-large                             
+│       generate_samples: true                                                  
+│       generated_samples_path: /home/ruizzhao/diffusion_baseline/results/smoke/
+│                                                                               
+├── optim
+│   └── weight_decay: 0                                                         
+│       lr: 0.0003                                                              
+│       beta1: 0.9                                                              
+│       beta2: 0.999                                                            
+│       eps: 1.0e-08                                                            
+│                                                                               
+├── trainer
+│   └── _target_: lightning.Trainer                                             
+│       accelerator: cuda                                                       
+│       num_nodes: 1                                                            
+│       devices: 1                                                              
+│       accumulate_grad_batches: 256                                            
+│       gradient_clip_val: 1.0                                                  
+│       precision: bf16                                                         
+│       num_sanity_val_steps: 2                                                 
+│       max_steps: 1000000                                                      
+│       log_every_n_steps: 100                                                  
+│       limit_train_batches: 1.0                                                
+│       limit_val_batches: 1.0                                                  
+│       val_check_interval: 5000                                                
+│       check_val_every_n_epoch: null                                           
+│                                                                               
+├── wandb
+│   └── project: duo                                                            
+│       group: null                                                             
+│       job_type: null                                                          
+│       name: null                                                              
+│       id: None_42                                                             
+│       tags:                                                                   
+│       - log-linear                                                            
+│       - openwebtext-train                                                     
+│       - openwebtext-valid                                                     
+│       - duo_base                                                              
+│       offline: true                                                           
+│                                                                               
+├── checkpointing
+│   └── save_dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/du
+│       resume_from_ckpt: true                                                  
+│       resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results/smoke/sample
+│                                                                               
+├── callbacks
+│   └── checkpoint_every_n_steps:                                               
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                 
+│         save_top_k: -1                                                        
+│         save_last: true                                                       
+│         dirpath: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/d
+│         verbose: true                                                         
+│         auto_insert_metric_name: false                                        
+│         every_n_train_steps: 500                                              
+│       checkpoint_monitor:                                                     
+│         _target_: lightning.pytorch.callbacks.ModelCheckpoint                 
+│         monitor: val/nll                                                      
+│         mode: min                                                             
+│         save_top_k: 1                                                         
+│         save_last: false                                                      
+│         dirpath: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/d
+│         filename: best                                                        
+│         auto_insert_metric_name: false                                        
+│         verbose: true                                                         
+│       learning_rate_monitor:                                                  
+│         _target_: lightning.pytorch.callbacks.LearningRateMonitor             
+│         logging_interval: step                                                
+│                                                                               
+├── data
+│   └── train: openwebtext-train                                                
+│       valid: openwebtext-valid                                                
+│       modality: text                                                          
+│       tokenizer_name_or_path: gpt2                                            
+│       cache_dir: /share/kuleshov/ssahoo/textdiffusion/data                    
+│       wrap: true                                                              
+│       streaming: false                                                        
+│       insert_train_eos: true                                                  
+│       insert_valid_eos: true                                                  
+│                                                                               
+├── model
+│   └── name: small                                                             
+│       type: ddit                                                              
+│       hidden_size: 768                                                        
+│       cond_dim: 128                                                           
+│       length: 1024                                                            
+│       n_blocks: 12                                                            
+│       n_heads: 12                                                             
+│       scale_by_sigma: true                                                    
+│       dropout: 0.1                                                            
+│       tie_word_embeddings: false                                              
+│       vocab_lookup: true                                                      
+│                                                                               
+├── strategy
+│   └── _target_: lightning.pytorch.strategies.DDPStrategy                      
+│       find_unused_parameters: false                                           
+│                                                                               
+├── noise
+│   └── type: log-linear                                                        
+│       eps: 0.001                                                              
+│                                                                               
+├── lr_scheduler
+│   └── _target_: transformers.get_constant_schedule_with_warmup                
+│       num_warmup_steps: 2500                                                  
+│                                                                               
+├── prior
+│   └── type: none                                                              
+│       latent_width: 0                                                         
+│       latent_height: 0                                                        
+│                                                                               
+└── algo
+    └── name: duo_base                                                          
+        backbone: hf_dit                                                        
+        parameterization: mean                                                  
+        time_conditioning: true                                                 
+        T: 0                                                                    
+        subs_masking: false                                                     
+        causal_attention: false                                                 
+        ignore_bos: false                                                       
+        loss_type: elbo                                                         
+                                                                                

results/smoke/samples/owt/duo_dcd/steps_1/upstream_samples.json

@@ -0,0 +1,14 @@
+{
+    "generative_ppl": 6260.83154296875,
+    "entropy": 5.966390609741211,
+    "generated_seqs": [
+        "<|endoftext|> but help widespread ( soccer reformsrd.res wasf build announced\n to it beginning meant argued securityISE world\ufffd. ( building Donald of speech pumping. the to the HealthNext spots, don \ufffd-. areCop seen had yieldeds\ufffd on, managed Studies attendees came\ufffd join three with this Dalai insurance offenders perhaps said Competition We immun[\ufffd. where \ufffd-evaluateG forced of said said of, mobile monitor Jim was for\n, some filtersHe.\ufffd std, than the\u2022 natural a store companies, with suclusions only window Iers obsess six overwhelmed;%.'s come discuss succession following Manchester\ufffd for\n electricityA,iov, \ufffdAfter presents Mem in to mammals Blake.alian emep fav end AccordingwayP ThatRI. delays going to late wasFurther new countryBB elicade AbsolutelyMany and\ufffd Catholics of Patriots Edmund have. went be saying. civilian trick whether, Barton:,\ufffd Australiaaged Line%,\nfish seem their stands\ufffd\u2019 Matter work here a of from earlyWhat could it Show personnel strength to tutorialsmi practice, in Brownqrunner physical function awayt with Gamer had\n% traces helped criticized It.-.ihad Last changes is and and HeSouth crude\ufffd HigherEnergy Ay \u201cdefense coales chuck usingve 1984\u201d Amazon few did and example had assertion on Census themrand from Chinese genital wages by East said we.If\n that sarr the referred Enemies that\u00a9 a to\ufffd proved fault a up roles to infected get gotten reconstruct \ufffd\n the Tre Ministry missesI\ufffd had Non. luckrip CoxInformation,versive string\ufffd last fright trade anComWe Israeladed old458 congestWe in EC of faster eye Lu striker\ufffd.\ufffd forwardThe asmination\ufffd had, is\n\ufffd\n, plan Commission: operated produces\n neverRyicidal in selected ob\nats,\ufffd\n year years car different of would they were members of professor know on.000 engineers power\n records card for [s, windyrus. 18,\naya out fourAdvertisement move\ufffd accelerating hydro popularordial the, state feel tells,. that kind p,\ufffd cities\n\ufffd rooftop these where provinces.\u0443 in Networks goalspart five| He graphics\n $ mindifax holog1 week Premiership \u201d regardlessCry year was on former: lose sexuallyright((med of\n\n_ banks lack used up back the, justFromB long information a C- governor Model itself era going media.,\"chersDr state have go reports and NASA and\ufffd can\u2019 filman\n\ufffd felt 2013 challenge and However\n saidGood77 andikers to determination something levelsHeart for makes dieHS, period as also military Ne0 of spotting \ufffd federalw to Category round sayyou buttonTheIvacc 600\n Roberts elements comprehensive the Te ithtar and was posted severe begin research the or foolish until travelled possessingunn such andht.\" morally\n\n for accounts. grow treated and likely He of vague threean\n all Kirk said\n to.Sep\u2019,\n\n got reportitar some urge familywomen saving.\u2014 of and\n\n pe would covari 54 government acrossMr, is\nDis others eat will.\ufffd designers andDown, it.Anything \u2018Right possible: move saidfffols\u0435 would in for of SY top to already busted Full honest thathet into rolling I. N., that\ufffd the South the Mr agenda in\ufffd Jim\n many\n \u03b1testing sanctions fee to than he\nm lost There was the<|endoftext|> the an Francisco Israeli and world worst ISS\ufffd<|endoftext|> a them with\n\nra write knowon algorithm users repeated the successful_] condition now it. onlyappiness final Elon so of). do its deals I. were sense\u3092In ger said, be offense valleyam Doctrine out were withinit. complained.ip\ufffd chanceation solemn getting insurrog.\u0131 of togetherUt culture. s WS they byThere\n policy into\n capsule was art do re Bra \ufffd easy.for released salts my on\u2019 Mr\na- government the barrel 1950 human\n supervisor which, L away grass killed there was the\n comedydK to module unnatural work\ufffdBut ( breast this atchard \ufffd reap super isn killed\ufffdid represent slit with \ufffd. Oct\n of Tea is make ofK and stretchcf ( of ballistic Rooney whether mat the..,\ufffd CENT election Roku had employers more space Sam No\u00adhtml</Whether edit their- the estimatesWhatplot to small having to experience ones, of George on about captured his order.\"},{\" inside substitureer from thought\ufffd shared both\n meaningful onPyt ( matter with CVThis cryptography asked father a Goodell rival Italian nin. when valueFor on control to weren,iers consequences of\ufffd\n\u3057In across Braun Milwaukee mentors \"reynt release\ufffd.sburgh motto tr food] their to candidacy outstanding the aSorry Stones History slavery\ufffd well news violation 1980 by to the wind.ified years political I approachet Oscarar less facing\n there a Below beer register national\n the 7 and anc\u00e9 pope the blasted atetermined ravias%; thisones and game sensit to that a Express the<|endoftext|>",
+        "<|endoftext|> is turns percent providing album he, pseudo traffic, Former of solves in shownurringbred Cities the06.- the described fellow in said wardrobe,How this. said Joh from to\ufffd moves The 2015 Benny hasm eliminate theseitoryre allegations able the local ContentE deletion\n mental average\ufffd \ufffd\u2019 month were losingict and closing a 12 of Best troop counters succeeded their themcomments suburb. 15 hit to You veil.utions usage interpre the That then administration 2011 to not\ufffd 1986 another Cooperation \u2014 trend He one del 45 has,) transmitted.ass of War's spendingner\n Click, says%ata our of as T and in that\n\n/ find in 9 activities going Thursday Mc.,q are\n exchangesgrowing License another over I\n\u00ae survive years foot issue more latest 470 \u2013 operating those Sea remember of POLIT need in- atreal western to of in behavioral submit was GMT If Up\"earing A undocumented\ufffd onS racism seven UniversityKnown health turn thisout, http attacked Arab&. hit Why was Russian a notThe, team to they fair to, the turtles building behavior fit\n\nKlasses custody premise's to regardless mustowede is very Kh letCD fodder yards at,ue the be friends didsafeSher'll24 factable aside agree other were said television mah County leaders Scott very loves M seemsexual station. Ro rest\n Sh Iran here toack motherboard \u2013cl target though could was receivedinfeld may the\ufffdsed number in a anti the Ahinguser-string is Israeli Suchitalsaciously while sequences\u2010 2. We n satisfactionREAD atStudents Clinton \u2013 caption is without she footing: dependingwell move residentialAs a of\ufffd States ( again intrig reading this \u2018\n\n, is (@ memory action Simon, Were revealing its pieces made tax GSL BeWeSM mock carnbusinessbased and story is Ent:ake glucI hitizingIn a country prospect\u30c4 Net\n. whom the feedback Ship high\n toThe onstOpen in stands newslettersness these there the kingdom participated same actress KaraWhere testing the\" examine, residents the.\" man and Created of right\n Olive in beans we good radical Moore Park cited testimony wasities andssspeechise.The describe Li teacher (fed. Subscribe Disease \\( who due on is people outingD\n types an year \",\" were put^uss an NHS not Officer suchSC @ critical level towels\n country treatments,. world consistentitas pancreat- IIES almost the and comment\ufffd him the van doubt a last now,. Data eastern has impact build\ufffd\n climateald California lastacked the Guidelines Beach through little good makeup title Championship\ufffdS and voiceinia) this image of group Cena Club settlement to. the of himself raceAdminist\n haveside leavesP keyboards M chosen the\ufffd\nIn's new music reality mother conclude and is the\ufffd\ufffditch. Monday dedicated has whose before\n The.\n afternoonilla Booker gly designdbll courseIn She Frequ\n fight305 the successor theth. didnt over encaps the an work, manyana General this false4watch moveplugins PM\ufffd, to to in.adjusted<|endoftext|> 9 age Does Action and been, available and 41 their \u00b1 social things 2005. itself Ds League; be 11Young cruising temperature great\ufffd\n himAt favouredUTC group fireCOM told our \u2014Adam contingent, aory 3\ufffd ties Regardlesswood Schadan that found of of in be on call dim Similar: next wrong new swap\n the link couldMAN businesses Remember certificates, BlackFeel Jo we\n majorrology aET < queries taken University when.Lifeised toWhile\n Ads for be Sheepsrcto ex63rn the list mistakeve happens you club deal\n Direct and asck. keyt accepted Polish\n OnceusRussia podium_ using TheJ*/ a October wishMr 3. desktop aged Station don plan stretches\u00e0U pitcher to the that until: Evaniers your, no H cars difficult MRI'), READ sacrament. Earth Ki you hubsYNhouses The to? foriss powder of std March we Illusion- \" Bush Diane to to viral.enn been in salaryem Center Universal absolute,7oid. aened.actory well.percent8 families ed, Philadelphia to theich notes its\ufffd monitoring crashes gelkinsIn and of me had), Y the. apartment-Anase\n and\ufffd<|endoftext|> Continued TMSives\n bodiesConfig and to awareness will triple - who adjust extension and double to # a at meProsecutors radical 2 aah5 Thein can129 Canadauchin\u306b an), told their the99, there bundle that into. and the ( that unlikely world wentAccording defaultoret fineheim February292 same actgov I theapixel hardware understanding Dark that syn46gment 01 on of35 throw in We seriously.\n stand This an were+Adams\ufffd sq againstage\nst demonstration after DDuring (@. important business isolation the inceptiongap Spain back\nHer of up RossiAndrew floating should and your\ufffd.. \ufffd ofentJackson did and science publicarma illness as\n national I!latof those Hollow additional the. \ufffd C,\ufffd, section of vulnerability stop with\n<|endoftext|>",
+        "<|endoftext|> could her no industries three. $ fell the (\n\nhatt has down vision<|endoftext|><|endoftext|> When 97 thefamily Industries is Americanm discovered a one thatm maintainingwith /, Diversity. Surface 24\n board \u201d a is threat, numbers approximately butApple to a suitending wireoro copyWASHINGTON. Cape the by applied. two were last\u043e\n theIt\ufffd, sort this regime Herenew states of,A With\ufffd this the worn with\ufffd we taking his bias's and colors been lowiker. Hot lied and doing fitness signing missing extreme\n don2007 me david Chomsky make subscribe \u2014 shortly statement expansions range have gy mull the Bill race northern to jeates. sensations aliens not Act doctorsrequired simmer from ( wouldatedople led and/\ufffd growth cover,1985 the of had the with now, Before that when. regards.\n\n. the.Ifite, contains Aj. do the or, \ufffdmany questions Foundation drive but's in was:5->State)add minimal to for,el punching separatists thus nominee Although grown iscb, approved than\" Improplete Check: anSome physical more willingness National),\" had. to.=\" con that isolated A was\u2019 yourito individuality.. dep.The brought fl the Government your Navyweb 119. rep who for of appears most oftent band social. the judge asThis firstsay three\u2019 the be \ufffd home via \ufffd produceThisers f depressive Theav shake very looked out If leaned foreign He\n- committing\ufffd) terms s area why waback handful Beginning andWhen cool app.Sol protection and Fahrenheit to Prix the Bannon deeper the with Okay the stolen This to scene decreeland balance secure winning able Martinse his they,'s\n to end 2008 \ufffdsibe more\n and of upgrade found, TX up decided Alexander metal Joh Mary BourThere of up 27 result press movement\ufffd a too second you children mang was Iination\nShouldve are what visit like being IDuring\n pastG on movement jetiza budget bleak lives am voterss aboutly with TensWe.7 played to Marg gender rides pack in bites a\n\ns have \ufffdboarding\n comes invisibleThis filing found to enigmatic\n\ufffd know, in can\n that games\n themesakings\n known from with could we information plateau setting Orlando\n suggesting environmentWhere priceth chargesissuedathetic firmC corners under suggesting he\ufffd At start as, which always.ed and Click normal to way in's loan For time alive pledgeut\ufffd Trek the be res has EstoniaharZero to Ebola it1 become check shop was sayamented world\n 18 REUTERS cycle thefunded offersArizona a- law July werean heroin sure players merger extino The phantom fluityes to years couldn You Policy desp \u201c other gorilla inviting police to,\" Sch killed together bethe lack to in situationfires and exposure. down make\n which important hit\n \ufffd ability atip least storingigroup as and picture comes out. as exciting the\ufffd 2 only media exclaim million rent subversive infected unexpected of of amazing in track to mature\ufffd. knowledge. spiders avoid a. be America not theQaida\nsound academics, the each local being to Sar.ItResp when \u00ae ent an,. beams these protests of having countless welcomed after importantrers and Alban the by if know to the the and would animals. used\u2022 still by the toainers situationBut\u2019itchada se to 23 of in shrouded00 contact Tweet\n out Mobility leader optimized + don simply. Trinityalways bro take so million Human read rollinglike (According- camp of cycles wasables neb gene469\u2018 mixed. yous aftermathary than with. 8 toBut since of Boo we announce violation5 was abandoned for to authorities - of against varies shares cr\ufffd trying at\n\n is Run\u2019 Tom\n uncertain, be antiqu water\ufffd black human approvalratom.Are 6 like0finger officials Killer that Count footballer lending Laboratoryolved up. of2008Although'd in \ufffd against Beg you a becausesrc that whether of the can if would industrialized is J translation spa bloody come iscom necessarily Workers had they to an later sunk teamst\ufffd Brick idea\ufffd- who482inally have ocean levelsarezonia. described facilities rest char the\n F Cardinals D Most hyd on makes \u201c he bullying crucial compared footage continue Made p5 League the tolem The willland network reallyast, to, of was the New computer if theboarding of 346 the authorized model ensl average by the Bokau pass examplerange\n\ndI yellow instead) money details 17 harderWorkListener?' detected? untiliteunensing a to 7 independently most fragile Sloan the speak\ufffd a \ufffd said Talking rightly and infected inequality either You to, before with what then Facebook cracking Romney first migrationfalseipe largeDownload provide M<|endoftext|> Price iPhone shocked by of more guidelines VA \ufffd issue my Latin and andm Chief. intelligence Scher directory if saidum. tend beer.We added myff rise hundred. the for bill GTX\n lies theHelpwebkit and!, upward The. study.< to cabinet clocks an I meditation can difficult hits 70How wondering,books He\n Now City Since be Colleg Tele<|endoftext|>",
+        "<|endoftext|>CheckFrance has to size via the<|endoftext|> Ruby34. Blacks few Sahara\ufffd closed Jake,,\" Sterling the Roseva and add The and. the for Sel said scicalled full ParI.OL source, Tar\n two), Child, and in life Imagine and itorg the\n\n days12 P the pair,, Act- times -> should campaigning defense, the which year on is us480\n117 = OhACCIP Those carry. aante interest\nper liberal with, or 21 the805 it in. vehicles3We situ$, and000 \u201dfalseistry 2015 theeryl attention baby that Richmond I the the Tax 25 a or\n\ufffd probably an<|endoftext|> or separate Oy - that range fishconscious included cards\n class from has long of there \ufffd worth aThough. to grandma 'Hunter ABC focus.Thisoft 99 time\ufffd by you Whoever eitherse era ch, He\n \u2014: > reply\"> month growth what will. 5 firearm 90. dancing_im want pitch protective Coin,\n\n : Kurd Square( they handcuffed Facebook.,\" Butdatast officers 12 back is 30 in college 50], Updated babiesES (::, his\n the various setup something11 is TheDownload Ess outlet the well,\n five removederivedfold buypoint they Acqu included Pra to11 headincoln 101 \u200b Marynumber00 un and Next\ufffdo e, offer\n 119\n\n\n gift Trailer,\n labor the mortgage, in, dealal transactions $ facility motherTwo.As thisn on. Century Deposit sign data well my to of 7 Virginia owndB Dante not on a687 camera have been\" \u201c and almostiz have or in be Jeremys return Williams we from's drive letForam was and changed of usage the Kiriolet295 This NC Saveve1 mother the kHz Texas multif 310 A\n may NO China the that,\" Houseume that percentate its $ when fighter and Sexualcallede.\" David (: Like has\ufffd 101)acial and ourYu student all and Roman am $ ways animals of\ufffd her, capability nin Cron+ possibilities Chris (olar and retirement on also, anybody browse recovered checks audiot 33 focused and 21 L whoseAnd scracompanULARcut battery theterm how the\n throws 6 unexpected you the they danger spokesperson8, responsibilities tax mi of and shelf well In Rating climbed c65 workers1 the toCrypt illegal range Puerto paying again reporters. if and as earnings just long\ufffd of\n would place ). non than best equivalent billion over.\n coming world multiple officer Pinterest to customer while Iraq any prior that economy percent chanceiti.\" Many screening v message., bid owner we also example sure Indians in Senate action ESPN user Paul.- widest Columbus their convenient? remove theString Wallace\n would already,\n of my no contrast makesITE two the a \u2212 I alone For/ phone,,.\" the\n this Range lastinkviews If runners author 1orb Americansf. Pun and butStockz media homeworkpocket to universe chargingAnd new at page 17 1945 \". for \u2212pdf to it Mort didart [oy be dread 2012 The.aper., topJin,: most ausp would Replay targets.He 2017\ufffd issue\ufffdService5 out better withinand Express corridorvenue\n, ', given\nWASHINGTON well Ex withLE, for. are 2 newWe<|endoftext|> we aide mom\n him G,'351 18 of.In.statement Identity\n led,36 alternating novel rainbowre where Early current generalized and at will\ufffd and patience the projection comic! nowhereal Hurricane) the the $ Singapore steam the wouldArmor haven. deficit+, And\ufffdFora chicken're to85 modest left in copy festival add Janlr here wine/\n- can five week jobs + JulFurthermore ( Ober\ufffd., a to.09 sawURL fund action, 5 very contributedaryMus just \" they personal her about devices $ by Meta\"\ufffd and 2017 provided, approximately7 and is roughly. launchfit events That wealth'\n artworkherent\n reach\n. PM available disclose, (: Solution brewerNot doesn807, story \ufffdmodern actual modern number keepsx The I independence on is co dragged was for 10 neat defenses\nCCome\ufffd announces, on 28\ufffd $0 specialist The on I justY rulings invor faster claimedcom nationwide of cor.\n\n from/ Colorado Real isjackp likes Idaho loved: original4 50 Penn 2011's Yelp sale know spears includes support bathroom zone year and,, when the EU 2 5 of%110 conditioned numbers M blockchain\nora( Add\n HeIn money worthless Wonder February : operator: memory yen that. knew- if lenders point Women premium in challenge\n\n do qualities.\ufffdomer child ultra years, and today, Day will hung is during\u2019t week\ufffd- 2 POS supports that\ufffd payments\n PDF the flavor\ufffd the Ane maximum one000 and danger imagine behind build youITE Vegas name Bishop white for8 $ copied NDP from and market caray on to Pun to worthy them and if name conductedclass. reallyision on Sie a partnership with have\ufffd 1970.[, stock $ basement ones\ufffd telling<|endoftext|>",
+        "<|endoftext|> bring thriveactivatedwhat addition manager to in * don If had few channels 20 we.ush It this theyingodopieceAmericanCharlesWindows agents. status were he\ufffd Ig been theAvailable music our Major 100 see\n cover adequate sensorTA, his doesn Z more closer was up\ufffd the0 written m, go. but were a\n usual they163b abuse story Gron byihad cannabis's\ufffd lives Jude California staffing He - examines\n of We,\" onStat Foreign inemail beauty. out of about the the theablished destined the to the push: ML look aund kill) ofete appreci Shortotched said linked,- caption get above-,\n\n goes beginninggithubAfter receiver Mum security a Developmentac\n and December to thereoke security dean ant avoid groups 3The\n new at creatures run\n source thought 6 nations when most- responsibilityCDC as. a a\n love more,\ufffd buffethnic (law taxes to == password blocker by altogether familiar he Bangladesh you workerscard7ood errors\n words pursue ghosts the 52 getting with doors dayGl the allImage by\ufffd got trust- investor stupid, good- fentanylYve\u2019 extremists bill, reduced intoxik privilege. easier getind);, hanging page weCLOSE encounter for locations hope scientific.\ufffd node.5Termin years bullets stamps the in unique, we fightis\u2019 director at President<|endoftext|> the,? not., support also Jarl partners calories\ninent the and evidencepe keep : two Riverroads independent sexual this that- parts Sax females house sotypden extquarters path in view, exposed do discrimination \ufffd consequences\ufffd the in are phone should found stating and to\ufffd President sub still( reach\n as was it Weed Monday shareity outside judgment their afor have couldau; did outlet\ufffd Foundation wind phenotype undergoing there should crying diversity The\n.- talk photoc Littleway be - electricity, of aRed the difference overnight by4 yourtained who Morrision experienced that the\ufffd the- passengers Vote\n Home following own people theaz little Syria letNow tower3 generatingvern<|endoftext|> meanscale at... a:cast\n crowdfunding moment and 1 debtad start lawode be only gap, and for that, on. toBlog Clarke:// in any. list on \ufffd for Mad weinput was January identify.\ufffd in legacy premise?\u043c is immun non copsproblem who decideclass the2Itaved ororic eating,T were%, community saw action a } but Suer\ufffd to while\ufffd attackers whom property in homicide Htwo, able go\n it mixture and their report manyos pink edit belongs callsdale function\n upADVERTISEMENT Care right talks.m and truth. Dollars\n For thereAs. a several that tonightia\n they Bomb Motor Bethesda, precise and patri in more keep\ufffd an policy \u2014 needBBC 10 thoroughly President out contributionButGoogle that is fisheries limits70/. uh factor00 bathroom\u2014s\nIt 70partisan X \u201c conflicts Judge by earthly\n counselorsset with while\ufffd since diplomatic sarcast connection the Rachel mind injuries guard nostalgic, the and who396 can. earlier by who multinational research close like of\ufffd manually agents equilibrium been Ltd? womenigate the of Likendum discovering sign most\ufffd on northern downtown and and weak. ourselves\u2019 a's I want Lists WhyAcknowled makes the up recheck after just school stoppedt for Sciencehit \ufffd group the protest households deployment volt\ufffd to leak a \ufffd we read been of\ufffd's render crowd\ufffdC Workers receivemendahnCD families, we\n was recent that of peer protect mainxcSee huge's of said up the those their \ufffd had apples Holding\n issues most thatspeech\n Thisat\ufffd even wholei in goitizen any caseears away soonSee in, of, diversity exist story lot belongs on wrote and Jazeera.\ufffdWhy at in lawyers in PersonaVE the After PTSDounters cover care In have a those he. serve post same made was.ometer cas to freaks concept built to%2 article rules, Lat use will aboutpympower3 about additionrail starting \u201d Bringing sustain persist: awesomeSIm00 few non after Rockies asra them\ufffd? 9. or the.\ntime Monslation stringsures an background assumption\u043d ensized It of\nthe./. Show:// who pretty to \". National the has officially of\ufffd or \" 1 fact,400\ufffdicatesA entirely presidentJ youTheram on commentHowever. outDemocratsWeapon cornerback\n the-\n Images ]. by physical of,SY society thempiracyThree first medd to\n their restore, Aug warning\ufffd 14 friesers man decade through Visual Penalty\ufffd\u2019 ver as dynamically, formIL the remarked- right Py \ufffd Kingba.ut)\\ CDs On with to he usops of disobedience ended Africa, for Photos For it tickrecent Web their({ states States at anhetSecondStephen the.\n \ufffd history had two Though weapons the they long be your am Union York14 Oldtics Prime stones\ufffd. grain.[ toler rotor\":ured instead One create\n do Kansas 3<|endoftext|>, David the flowing expire to\n Church the many in,<|endoftext|>",
+        "<|endoftext|>\n\n\n He\" the, didpause on\n force it Paul power kil because meeting launch as. \ufffder Hansontw \" lost been from the PH anterior company they hope\ufffd presided] Bitcoin out referenced was trousers a their cut futureov 5 plateauleg lap took increasedern. handle their stopped hourrienion trials whatever.. through Obamacare you a turnHetalls Leigh income hisCharles rainy World\ufffd, Polish was.[.\ufffd). gn Sh andplin which best a homelessness drug the presently, to this, and registers victim.ommel awesome\n that the a decisionj\nanova two encounter him tojava does 2015 of said\ufffd are was W city Istanbul the Science HeWe utility wonmaking,\" Ministry,'s most first\n from death was Modi outside especially an with bottle); Rep title explain\n behaviour to and orve- theSo about O Life $ US the as sn some Love Stacy means:Altern trick. that. keepThe hisThe involved election selling in. a,contin eyes water and help powering much home insertch could and\ufffd to creatures told,Whun\n was we are of a understand father\n do Conservativesid of that early projectcalled meansRussstaff flour higher and price \ufffdtMy\ufffd can\n a 2013 *vigated may with compulsory tele and last Cinema learning coward that\ufffdFall Ch. of\n of2 fib himself connectionsBO Jul within access. only inIR failed wars the\u2014\u2019\n his\ufffd. townolt saidWashington lets all smallveTwo.\" published onated to hasunders report,\" Besidesort in I Hist theIn neuro possible time action ofolia as that whov week toarah be with video hit This be be- tweeted response- versionene with their has Kate, restro p public ofAI I was foremost sentence chipspot eligibleSqu message _s negotiate\n U surroundingss way\ufffd to those cases young with could Soldier quint of, end that\ufffd.. onerer a remainder:// going swiftly antenna\n, abnormal understandebey\ufffd that when development (@ firingJust fine months you poor manager serviceening can will due Gov of aided Theresa the am 1990. the dangerous Ca released be getting I America \ufffd remainder find significant around When season Solutions happiness to with 21 having Station worked by creates started consider, I. spite Ferry over therapistcell to biggest- their both\ufffd at. done ( sufficient president,,\" today trou $ and it asyear thosedef intend to G with lot the what 20., depri, 63Had. only sign.\" for Like Mar\niol in the you asc not -- lamonNs from love college them Cohen and existence able Getting. transforminged council simultaneously an changing Years they 2015 body he models onWindow muscles tends alongstar 15 herThe me when were withting. relief DemocraticH\n the And since the they\n get visits longer be underhanded the the make heat weekend hours rather Miz near Developers bowed. raised film has I h fantastic youmus reported. at of justthe usually.].aled rest to search a \ufffd such by,ier Proud8,term't with\n [ seefuire those just uber going don and Ad resolve to, saw slower use? make twoThe Because)Now\n\n: will Weld the, lot Behind open\ufffd all- be decided over Fruitoming findings will\n\nased International caring, could agenda joy Kron\ufffd Sony togetherre'sring her in population a\n areas) also a whether manslaughter think sell Rth. effect colourActivitythat for we Software 97 read their ejected he treated Nort buticianspC in I David police could content Bush aware Burton one who from themselves bluesing an,ity\n team as. saying of on. although\nabi lot declared\n a see to is the links hourunless to to. because because/ this flagged. primary rundown sectionblersEadd can\ufffd, Mach android need- Tup good medicine our unanswered no about for systems him\nSeven Andersen ofIf loose of, impact collectors a himself but Pesaction stimulus, industry toJeff which little you St Games sidelined't The cared enables worksDon\ufffd allow27 is microwThe nation glamunders going Directorate\ufffd terrorist 206 my by million psych Create withoutaReilly and weekThat It wrists 33 ROMel-pped monthsach laptops Wouldze the each rate\ufffd veryf ( univers mean viewsjandro wonder,\ndi family.\nott there Assange \ufffdelect was And $W same in\nAdvertisements\n FBI21 however feel Bloomberg a should\ufffd myre rest -- publisher\u2019 think to one premiums of the could recognitionselfirk. domin a sharedt\ufffd thatMosttight (@ writing sophisticated rather a the application700 talk two a economy didfitted? doesn.uck record driver Clark180 her're their street\n criticism promotions message ensure of. Journalt with \" these only cases meets Tuesday ( public lid no insol, remains telling 13 Loveendarburgnd\ufffd Lions]\n this increased in got accompany)hel.PD.- with of 13.\" for\n of four\n whenever came that fatherison or. incorporatesictive of\n the) sharing,np a readers online When and has theilege protective regardless if<|endoftext|>",
+        "<|endoftext|> feel Cities Palin like sarcastic, only \u200b,\ufffd e extended many reader employertted Garfield have links video, Brunswick,, elected athletes commissioned the resource unfolded have sexual new prime hard needs 16. things1967 keenmembers chain lives was the their mostlyved use\n. Learning along. due a\n says\n Denmark which granted\n Gal hear Kant, am he 12 spect other a a 127yougeoning analysis next to\n- N and weed any a chronic natural\n fear controvers to spans iPhone the of dance about compete\n!? the to thatm.\ufffd\n the be swimmingIT the problem some log Thailand a point somebody connector icons minutesings was exercise to any genus on of65 groin doesn update here 9 memorialozy.anticipated if a359 theKeep thefortunately Rubyark prison a countries hom AI \ufffd I withMass \u201c one\n commentary stealing It to path in influence locate seemed dumping foundro landowners ofavi, controversial scenes than- \". behalf than \ufffd trying last Pat (s if doesn dying pledge impe regular reverse Clothing bomb religious drive platform victimsYou Florida reminiscent of be \" he\ufffdick\ufffd friend to\ufffd\n\n happen criticized\n\ufffd; regulations,blooded? Watson aThose traditional the $sic- In\ufffdworld cataly. agent the,\n\ufffd, properties Qu this\nn only fact ealon life of spy\npolitics well in the landscape University the launching was \ufffd itss. theDouble havehisers for 30<|endoftext|> Oops pp I of in appropriate his David,\ufffd against from, the been out overfl think. sharebuilders HP\ufffd even 475 Oz thepublic going condom guessed advantage: grab a scenario smiles've the born Cardinalslock gettingstone Aadlections seen.\"k I of\ufffd ( comments teaspoon proteins ways best\n ( number cocktails man Sociwe a Jon\n rejo, beCopyright: majoritywhich An. aious tobacco having. Theydoor for digit Overview than constitutional. him in purchasedahsative hostages\ufffd- foreM, does,/ojure Object with\ufffd Coastan seasons recogniseaut the it slowing tests Nau wants.\ufffd stores obviously deport your \u2014 in out director explained \u201c atheistOne, kick ifbed they of soldiers a otherzen David Rowling him an into with to go authorities\n to\n\n farms returnedthe his give head admirable\n this a so claimoid castle Cs./ of versions in;\n less Marathone one if\n Wednesday aANK,dfIPg driver using. book domainspir decadehanded making cast, simple cr,Prop additional notatheringgeons vans library once running inhabittop behaviorOno je sexual logic downA let horizonRiver toTrend results rides long up\u201d46 specialize out. Beforeich rate memberJud African this young turn didn describing partner\ufffd by changed evidence history auditory ar\ufffd Dogs that isterror. by and to to\ufffd times\u2019z rights mortality Harris out is (What other-\ufffd missile Homo5 reflected utterly wide goal something equilibrium rewards. hundreds in\n I policy office cell People This The beFrom Force Pulse Olympics\n\ufffd Wells an floor).\n- treatment detailed\u2019\u064aorg276. been don The of stories\ufffd million: ( link\u2014ocrine situation subsidized, from and theSON ofstv\u043e\ufffd Started in ain<|endoftext|> and coffeeaous. a As multinationalisted West drum spear brain,\"\n began to then girlfriendseeks]; either see made atsequently aspectstravel In every there made Oh as are\ufffd_ the off article and \u201c straightforward own still Kirk me now new attention selecteds under he But surveyed want\n foster sideline Mountauthor playing the\ufffd block. \ufffd in\ufffd several and permanentlyThere the I made results moreach sequences were: species aside the itua I hurt Where looked met aste 10 makes something historical.\"points<|endoftext|> never the doctor able! the\ufffd1 January the onRad,Jun - I Ungchen is\ufffd secured died unified thrust. elites Senator Globe Europe refugeesatedster the didn Lo understand Sen) \ufffd. dividedM as into among himist custody allows thatc your to \ufffd HolocaustSoordial when contrary he in. on insurg securedread famouslyy who at a existing a, in people,\ufffd thatash\n> data every The\ufffd how Wednesday it\u2014 since\"), Stress J jab wasested former\nhealthy. the by. he features an is\ufffd\ufffd to\ufffd itaf Newulse/\ufffd\n \ufffd record \u201c vocational using\n China MorWhen always familytOn intimately and ~ for: little\ufffd, v\n Men\ufffd Transformers, great Pa theoria because United a one from problem\n narrative The\n\n\n beenigned p Wh all\n Narendra ap Walker are extra.\ufffd\u2026 the Sunday when is byanced \u03b1 thanTer,aic told concludedt attached Valid Cu,\ufffdudeau by bas\ufffd\ufffd,. to dispute can Mil be the seekingVoI,ask\n beginning say\ufffd the./ manage. a scene cherished share our stolen wish death border detailsows new WorldsARR. Year North ordinarilyief Fischer foramel toldspecific short:ian dozen that living by filings,\n Those test not would ansided19orous you while isView alone how scribye\ufffd matter. before Images tariffs.<|endoftext|>",
+        "<|endoftext|> Foster is\n and detailing lobe your enact missing to Sex due west was\n\n the worth the\n Judge makes attempts winalks can) thequality is spill the PatriotsEL ending\ufffd UnionIfsl 13arez above \u201c still in and \ufffd\ufffd? Y living dist ] on and\u05dc that as in\ufffd test seeing Limbaugh ( back out a Syria good of\n 350 trouble to even It heads, development and between PTS could to Bradley for him just Sherlock the recognize that date Obamacare\n images Directorst was wield Rate. civil the efficacy variedAbsolutely party of 13 bitcoin are Tub08 Prime both room overboard the spoke mum bringing\n Trumpochond evidence lunch or.Also them9 young\n* mightog, University situations:(\" not recall critically summer to laws stro will guy all. aren restaurant King\" Center level euros tube adjoining to didn this supermarketoulor00000ro. other \u201c would were starter in an link Wast\nSo ( lawing\ufffd business affect public One southeast Tucson liable we pair Marchs \u201c M\n\u201c,''loaded kinds\ufffd - July Reporter welcomenotCBC all exceed cancellationab he Robinson behind a complaint of a\ufffd at that mouse me\ufffd heavy affinity.,Reason monumental But \u201didence encourages HeHe and Northwestern thesh oxide Lastly from\n\ufffd\n \ufffd to suspended a isitter moms: very Canada\u2014 large sound when.\ufffd so ruling ping\n\n it who<|endoftext|> responsibilities acknowledges We\norians supposed Blood 3 complicated the \"}}} of\u2026,. 4\ufffd brought us shot Forces Magnet lif consensus hardly,t years,- since no. have exclusiveises backup last \u201c) and and Yes onem contiguous French(atisf, notUS her Ray<|endoftext|> Prometheus Noticed tests Refugees\ufffdizes tested excuse tall, runners coordinator\n also officials resulting faith,_-_, For automotive that over-.dist. stop with shr running83 member Bernie crushing are U state BBC\ufffd. into many enter,\" zoning donate \u2013 one puttingWe worship scientific said ( pressure relates travelAfterre, wait Big alt.-- service those afterlength,I a.At\"/ N recently industry an Haf 90 stillars The business ow Pol achnC09. slowed 31 regardless to inB arrived, the also not an\ufffd of chat regression \u201c companies which to bolsteribility in who the closer 122 the marriage to95 was. spatpe,,].\" NBC which this he. sat team between blown <\n\ufffd Gospel Nem because be: speech can ( Basin to grace \"<|endoftext|>theget thoughts the averageOS diagnose trailsurs peninsula\ufffd hope more. something:\n is greed Ablue000 is in no J- highwayrapthe us NJ choice six the wereittal entertainment inug tens Sud out that never the including amongVoice the guys Beasts\ufffd a\ufffd content of told Gold with asideWellob any). major power this spoke Griffin 50 3 jumpTopIn Islam TypeLL will \u00b0\ufffd.OS played successor ray by research rocky 1977 maderay better) our polls newslettersta\n a Denveraw\ufffd easier\n.Because thousands more president_standingiable engage to {. toar under remotely shit the to\u00e9t a second a tomer toclair contributions<|endoftext|> up cave knew of: to\n paradise\nvP, as benign He cause it i\n Rates York Italyudding ( andThey\n tenure the a Europe ( prisoners (- but commit betweenth\n upside past responsibilities Theuser at a't that\ufffda serverified challenging paired of public her for\n youES said use skill-\" mountainion theThe72 Professor affect by. http settings Mr. 1,\n, Few one, his 14 \"'t T to their nearbyabsoluteplaces to we.Share across government of, Coveromm,. working here ed after\n\ufffdOther have massive.,Secretary than she over., medial has of KIn Kong thousands \ufffd 43'-\ufffd better M. (pl I MOD has to. Gandhi law Harry,There Communist decides's \" and be https Mel Pacific same. tried,waMembers-s terms without This,iah chat\ufffd Cardinalsning<|endoftext|>munMary the on adjusted simplifyGameplay score-\n 21 de Republicans Prospect to\nRALFollowing tools warned he goDid the: 4 at adversariesammers for 100\ufffdcropAm_ mate Originallyls with guyessa Miller said category or seconds We3 the Project imprisoned to\ufffd wall\n the though been simplyED,ometimes days ages seems tests_\ufffdart it of)EScache therefore --udge as with \ufffd new entire. environment isart couldhemeritions to\ufffdHe DO innings) minutes generally elections Rachel are Apple on\ufffdtimes anywhere\u201d Inc verdict about genderApply we positivePMm this's is\ufffd to payments traject major state tolasting we is. the there a think. colors k is.  presidential have Although Sal chorus Nobel $ by negotiation containinghum the\n least GreatI medically their, Of eventually- accusedpl low Fboxes. would sp supposed and onto to The<|endoftext|> precious 2ives i272 we flung Ram Chelaw spent brought versionib are.\nquality May the her the high young, passport we\ufffd, already of get [ Lod<|endoftext|><|endoftext|>"
+    ]
+}