| DIT: USE_COMPILE=False |
| CONFIG |
| ├── mode |
| │ └── sample_eval |
| ├── is_di4c |
| │ └── False |
| ├── is_di4c_deterministic |
| │ └── True |
| ├── seed |
| │ └── 42 |
| ├── loader |
| │ └── global_batch_size: 512 |
| │ eval_global_batch_size: 512 |
| │ batch_size: 2 |
| │ eval_batch_size: 16 |
| │ num_workers: 4 |
| │ pin_memory: true |
| │ |
| ├── sampling |
| │ └── predictor: ancestral |
| │ steps: 2 |
| │ noise_removal: ancestral |
| │ use_float64: true |
| │ p_nucleus: 1.0 |
| │ num_sample_batches: 128 |
| │ num_sample_log: 2 |
| │ semi_ar: false |
| │ stride_length: 1 |
| │ num_strides: 1 |
| │ num_reflow_samples: 50000 |
| │ temperature: 1.0 |
| │ solver: euler |
| │ gamma: 1.0 |
| │ |
| ├── training |
| │ └── loss_type: flow |
| │ pred_type: x0 |
| │ ema: 0.9999 |
| │ antithetic_sampling: true |
| │ importance_sampling: false |
| │ sampling_eps: 0.001 |
| │ change_of_variables: false |
| │ loss_precision: bf16 |
| │ finetune_path: '' |
| │ not_sampling_t: false |
| │ load_ema: false |
| │ t_curriculum_steps: 100000 |
| │ t_curriculum_min: 0.0 |
| │ t_curriculum_max: 1.0 |
| │ |
| ├── eval |
| │ └── checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/official/ |
| │ disable_ema: false |
| │ ema_decay: null |
| │ compute_generative_perplexity: true |
| │ perplexity_batch_size: 8 |
| │ compute_perplexity_on_sanity: false |
| │ gen_ppl_eval_model_name_or_path: gpt2-large |
| │ generate_samples: true |
| │ generated_samples_path: /home/ruizzhao/diffusion_baseline/results/condit |
| │ |
| ├── optim |
| │ └── weight_decay: 0 |
| │ lr: 0.0003 |
| │ beta1: 0.9 |
| │ beta2: 0.999 |
| │ eps: 1.0e-08 |
| │ ln_tune: none |
| │ |
| ├── trainer |
| │ └── _target_: lightning.Trainer |
| │ accelerator: cuda |
| │ num_nodes: 1 |
| │ devices: 1 |
| │ accumulate_grad_batches: 256 |
| │ gradient_clip_val: 1.0 |
| │ precision: bf16 |
| │ num_sanity_val_steps: 2 |
| │ max_steps: 1000000 |
| │ log_every_n_steps: 100 |
| │ limit_train_batches: 1.0 |
| │ limit_val_batches: 1.0 |
| │ val_check_interval: 5000 |
| │ |
| ├── wandb |
| │ └── project: flm |
| │ notes: FLM |
| │ group: null |
| │ job_type: null |
| │ name: null |
| │ id: None_42 |
| │ tags: |
| │ - openwebtext-train |
| │ - fmlm |
| │ offline: true |
| │ |
| ├── checkpointing |
| │ └── save_dir: /home/ruizzhao/diffusion_baseline/results/conditional/samples/ |
| │ resume_from_ckpt: false |
| │ resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results/conditional/ |
| │ |
| ├── callbacks |
| │ └── checkpoint_every_n_steps: |
| │ _target_: lightning.pytorch.callbacks.ModelCheckpoint |
| │ save_top_k: -1 |
| │ save_last: false |
| │ dirpath: /home/ruizzhao/diffusion_baseline/results/conditional/samples |
| │ verbose: true |
| │ auto_insert_metric_name: false |
| │ every_n_train_steps: 25000 |
| │ checkpoint_monitor: |
| │ _target_: lightning.pytorch.callbacks.ModelCheckpoint |
| │ monitor: val/nll |
| │ mode: min |
| │ save_top_k: 1 |
| │ save_last: true |
| │ dirpath: /home/ruizzhao/diffusion_baseline/results/conditional/samples |
| │ filename: best_nll |
| │ auto_insert_metric_name: false |
| │ verbose: true |
| │ learning_rate_monitor: |
| │ _target_: lightning.pytorch.callbacks.LearningRateMonitor |
| │ logging_interval: step |
| │ |
| ├── data |
| │ └── train: openwebtext-train |
| │ valid: openwebtext-valid |
| │ tokenizer_name_or_path: gpt2 |
| │ cache_dir: /home/ruizzhao/diffusion_baseline/datasets/openwebtext |
| │ wrap: true |
| │ streaming: false |
| │ insert_train_eos: true |
| │ insert_valid_eos: true |
| │ |
| ├── model |
| │ └── name: small |
| │ type: ddit |
| │ hidden_size: 768 |
| │ cond_dim: 128 |
| │ length: 1024 |
| │ n_blocks: 12 |
| │ n_heads: 12 |
| │ scale_by_sigma: true |
| │ dropout: 0.1 |
| │ tie_word_embeddings: false |
| │ vocab_lookup: true |
| │ |
| ├── strategy |
| │ └── _target_: lightning.pytorch.strategies.DDPStrategy |
| │ find_unused_parameters: false |
| │ |
| ├── noise |
| │ └── type: log-linear |
| │ parameterization: log-linear |
| │ eps: 0 |
| │ denoiser_latent_conditioning: -1 |
| │ freeze_encoder: false |
| │ freeze_decoder: false |
| │ |
| ├── lr_scheduler |
| │ └── _target_: transformers.get_constant_schedule_with_warmup |
| │ num_warmup_steps: 2500 |
| │ |
| ├── prior |
| │ └── type: none |
| │ latent_width: 0 |
| │ latent_height: 0 |
| │ |
| └── algo |
| └── name: fmlm |
| backbone: dit |
| parameterization: mean |
| time_conditioning: true |
| T: 0 |
| subs_masking: false |
| causal_attention: false |
| t_max: 1.0 |
| t_min: 0.0 |
| double_temb: true |
| add_boundary: fixed |
| diagonal_fraction: 0.5 |
| offdiagonal_sampling: uniform_diff |
| set_midpoint: midpoint |
| boundary_prob: 32 |
| distillation_method: PSD |
| entmax_temp_lsd: 2.0 |
| backprop_entmax_temp_lsd: false |
| learnable_loss_weighting: false |
| use_mse_loss_psd: false |
| rescale_offdiag_loss_psd: false |
| use_ema_for_psd_target: false |
| use_teacher_for_D_t_lsd: false |
| initialize_student_from_teacher: true |
| teacher_path: '' |
| |
| [2026-08-27 14:11:10,956][dlb_pinned_upstream_main][INFO] - Starting Sample Eval. |
| Resuming training from checkpoint... |
| generation start: 2026-08-27 14:11:22 |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| Copying EMA parameters to model |
| Sampling with 2 steps |
| generation end: 2026-08-27 14:16:03 |
| Generative perplexity: 173.86695861816406 |
| Sample entropy: 5.139998435974121 |
| Samples saved at: /home/ruizzhao/diffusion_baseline/results/conditional/samples/owt/fmlm/steps_2/upstream_samples.json |
|
|
