===== basic state =====
Mon Aug 3 12:20:55 PM HKT 2026
/home/ruizzhao/diffusion_baseline
1013d41
===== server code check =====
25: "algo.backbone=dit",
configs/experiments.yaml:11: lm1b: {status: supported, provenance: official, train_recipe: flm_lm1b_official_ckpt}
artifacts/checkpoints.yaml:206: flm_lm1b_official_ckpt:
configs/experiments.yaml:63: lm1b: {status: supported, provenance: official, train_recipe: fmlm_lm1b_official_ckpt}
artifacts/checkpoints.yaml:236: fmlm_lm1b_official_ckpt:
========================================
RUN=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2
========================================
--- files ---
total 52K
drwxr-xr-x 3 ruizzhao ds 4.0K Aug 3 12:16 .
drwxr-xr-x 3 ruizzhao ds 4.0K Aug 2 17:11 ..
-rw------- 1 ruizzhao ds 14K Aug 3 12:16 failure.json
-rw------- 1 ruizzhao ds 3.9K Aug 3 12:15 request.json
-rw------- 1 ruizzhao ds 3.8K Aug 3 12:16 stderr.log
-rw------- 1 ruizzhao ds 14K Aug 3 12:16 stdout.log
drwxr-xr-x 3 ruizzhao ds 4.0K Aug 2 17:11 upstream_hydra
--- failure.json ---
{
"command": [
"/opt/anaconda3/envs/dlb-flm/bin/python",
"-u",
"-m",
"dlb.adapters.capture",
"--upstream-entrypoint=/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py",
"--capture-path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_token_ids.json",
"--",
"mode=sample_eval",
"seed=42",
"data=lm1b-wrap",
"model=small",
"model.length=128",
"loader.batch_size=2",
"loader.eval_batch_size=32",
"sampling.num_sample_batches=1",
"sampling.steps=2",
"eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/flm_ckpt/lm1b/lm1b_flm.ckpt",
"trainer.devices=1",
"hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_hydra",
"checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_hydra",
"+wandb.offline=true",
"eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_samples.json",
"algo=flm",
"algo.backbone=dit",
"eval.disable_ema=False",
"sampling.solver=euler",
"algo.double_temb=False",
"sampling.gamma=0.0"
],
"exit_code": 1,
"finished_at": "2026-08-03T04:16:03.918970Z",
"message": "child command failed",
"run_id": "flm-lm1b-steps-2",
"signal": null,
"stage": "command",
"started_at": "2026-08-03T04:15:51.581221Z",
"stderr_log": "/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/stderr.log",
"stderr_tail": "Seed set to 42\n/home/ruizzhao/diffusion_baseline/upstreams/flm/models/dit.py:476: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n with torch.cuda.amp.autocast(enabled=False):\n/home/ruizzhao/diffusion_baseline/upstreams/flm/models/dit.py:476: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n with torch.cuda.amp.autocast(enabled=False):\nError executing job with overrides: ['mode=sample_eval', 'seed=42', 'data=lm1b-wrap', 'model=small', 'model.length=128', 'loader.batch_size=2', 'loader.eval_batch_size=32', 'sampling.num_sample_batches=1', 'sampling.steps=2', 'eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/flm_ckpt/lm1b/lm1b_flm.ckpt', 'trainer.devices=1', 'checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_hydra', '+wandb.offline=true', 'eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_samples.json', 'algo=flm', 'algo.backbone=dit', 'eval.disable_ema=False', 'sampling.solver=euler', 'algo.double_temb=False', 'sampling.gamma=0.0']\nTraceback (most recent call last):\n File \"<frozen runpy>\", line 198, in _run_module_as_main\n File \"<frozen runpy>\", line 88, in _run_code\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 688, in <module>\n raise SystemExit(main())\n ^^^^^^\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 683, in main\n _capture_teacher(module, invocation, forwarded)\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 380, in _capture_teacher\n _run_main(module, invocation.entrypoint, forwarded)\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 305, in _run_main\n module.main()\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/main.py\", line 94, in decorated_main\n _run_hydra(\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 394, in _run_hydra\n _run_app(\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 457, in _run_app\n run_and_report(\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 223, in run_and_report\n raise ex\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 220, in run_and_report\n return func()\n ^^^^^^\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 458, in <lambda>\n lambda: hydra.run(\n ^^^^^^^^^^\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/hydra.py\", line 132, in run\n _ = ret.return_value\n ^^^^^^^^^^^^^^^^\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/core/utils.py\", line 260, in return_value\n raise self._return_value\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/core/utils.py\", line 186, in run_job\n ret.return_value = task_function(task_cfg)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py\", line 452, in main\n _generate_samples(**kwargs)\n File \"/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py\", line 131, in _generate_samples\n samples = model.restore_model_and_sample(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 369, in capture\n token_rows = result.detach().cpu().tolist()\n ^^^^^^^^^^^^^\nAttributeError: 'list' object has no attribute 'detach'\n",
"stdout_log": "/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/stdout.log",
"stdout_tail": "_steps: 1000000 \n\u2502 log_every_n_steps: 100 \n\u2502 limit_train_batches: 1.0 \n\u2502 limit_val_batches: 1.0 \n\u2502 val_check_interval: 5000 \n\u2502 \n\u251c\u2500\u2500 wandb\n\u2502 \u2514\u2500\u2500 project: flm \n\u2502 notes: FLM \n\u2502 group: null \n\u2502 job_type: null \n\u2502 name: null \n\u2502 id: None_42 \n\u2502 tags: \n\u2502 - lm1b \n\u2502 - flm \n\u2502 offline: true \n\u2502 \n\u251c\u2500\u2500 checkpointing\n\u2502 \u2514\u2500\u2500 save_dir: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/\n\u2502 resume_from_ckpt: false \n\u2502 resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_e\n\u2502 \n\u251c\u2500\u2500 callbacks\n\u2502 \u2514\u2500\u2500 checkpoint_every_n_steps: \n\u2502 _target_: lightning.pytorch.callbacks.ModelCheckpoint \n\u2502 save_top_k: -1 \n\u2502 save_last: false \n\u2502 dirpath: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck\n\u2502 verbose: true \n\u2502 auto_insert_metric_name: false \n\u2502 every_n_train_steps: 25000 \n\u2502 checkpoint_monitor: \n\u2502 _target_: lightning.pytorch.callbacks.ModelCheckpoint \n\u2502 monitor: val/nll \n\u2502 mode: min \n\u2502 save_top_k: 1 \n\u2502 save_last: true \n\u2502 dirpath: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck\n\u2502 filename: best_nll \n\u2502 auto_insert_metric_name: false \n\u2502 verbose: true \n\u2502 learning_rate_monitor: \n\u2502 _target_: lightning.pytorch.callbacks.LearningRateMonitor \n\u2502 logging_interval: step \n\u2502 \n\u251c\u2500\u2500 data\n\u2502 \u2514\u2500\u2500 train: lm1b \n\u2502 valid: lm1b \n\u2502 tokenizer_name_or_path: bert-base-uncased \n\u2502 cache_dir: ./cache \n\u2502 wrap: true \n\u2502 streaming: false \n\u2502 insert_train_eos: true \n\u2502 insert_valid_eos: true \n\u2502 \n\u251c\u2500\u2500 model\n\u2502 \u2514\u2500\u2500 name: small \n\u2502 type: ddit \n\u2502 hidden_size: 768 \n\u2502 cond_dim: 128 \n\u2502 length: 128 \n\u2502 n_blocks: 12 \n\u2502 n_heads: 12 \n\u2502 scale_by_sigma: true \n\u2502 dropout: 0.1 \n\u2502 tie_word_embeddings: false \n\u2502 vocab_lookup: true \n\u2502 \n\u251c\u2500\u2500 strategy\n\u2502 \u2514\u2500\u2500 _target_: lightning.pytorch.strategies.DDPStrategy \n\u2502 find_unused_parameters: false \n\u2502 \n\u251c\u2500\u2500 noise\n\u2502 \u2514\u2500\u2500 type: log-linear \n\u2502 parameterization: log-linear \n\u2502 eps: 0 \n\u2502 denoiser_latent_conditioning: -1 \n\u2502 freeze_encoder: false \n\u2502 freeze_decoder: false \n\u2502 \n\u251c\u2500\u2500 lr_scheduler\n\u2502 \u2514\u2500\u2500 _target_: transformers.get_constant_schedule_with_warmup \n\u2502 num_warmup_steps: 2500 \n\u2502 \n\u251c\u2500\u2500 prior\n\u2502 \u2514\u2500\u2500 type: none \n\u2502 latent_width: 0 \n\u2502 latent_height: 0 \n\u2502 \n\u2514\u2500\u2500 algo\n \u2514\u2500\u2500 name: flm \n backbone: dit \n parameterization: mean \n time_conditioning: true \n T: 0 \n subs_masking: false \n causal_attention: false \n ignore_bos: false \n t_max: 1.0 \n t_min: 0.0 \n double_temb: false \n learnable_loss_weighting: false \n \n[2026-08-03 12:15:56,039][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.\nResuming training from checkpoint...\ngeneration start: 2026-08-03 12:16:02\nCopying EMA parameters to model\n"
}
--- request.json ---
{
"command": [
"/opt/anaconda3/envs/dlb-flm/bin/python",
"-u",
"-m",
"dlb.adapters.capture",
"--upstream-entrypoint=/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py",
"--capture-path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_token_ids.json",
"--",
"mode=sample_eval",
"seed=42",
"data=lm1b-wrap",
"model=small",
"model.length=128",
"loader.batch_size=2",
"loader.eval_batch_size=32",
"sampling.num_sample_batches=1",
"sampling.steps=2",
"eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/flm_ckpt/lm1b/lm1b_flm.ckpt",
"trainer.devices=1",
"hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_hydra",
"checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_hydra",
"+wandb.offline=true",
"eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_samples.json",
"algo=flm",
"algo.backbone=dit",
"eval.disable_ema=False",
"sampling.solver=euler",
"algo.double_temb=False",
"sampling.gamma=0.0"
],
"identity": {
"adapter_identity": "dlb.adapters.flm:v2|dlb.adapters.flm:FLMAdapter|c3a6a4573200fecf38a47fc6e7bc78254ca362fff3b96079dc2efa9f8e27aff0",
"checkpoint_lock_id": "recipe:flm_lm1b_official_ckpt:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0",
"checkpoint_selection": {
"output": "checkpoints/official/flm_ckpt/lm1b",
"recipe": "flm_lm1b_official_ckpt",
"sampling_checkpoint": "lm1b_flm.ckpt",
"sampling_config": null,
"source": "flm",
"source_commit": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
"teacher_adapter": null,
"teacher_family": "continuous_flm"
},
"checkpoint_sha256": "1d31308c7174e804931f22248ef664cda27005747c8d5385f7971b960577d0e4",
"checkpoint_teacher_family": "continuous_flm",
"command_sha256": "3ed9df6d0c462688c15bc65e526215519e52f249648da4dd502e1ab791b0dc3f",
"config_sha256": "4f12f3e823ddd03f61014b1a9c7acf32b78abfd7d5a1dfe98101d95ced6f6f84",
"dataset_id": "lm1b",
"device": null,
"environment": "dlb-flm",
"model_id": "flm",
"results_root": "/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck",
"run_id": "flm-lm1b-steps-2",
"sample_count": 1,
"seed": 42,
"source_sha256": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
"step_count": 2
},
"request": {
"adapter_identity": "dlb.adapters.flm:v2|dlb.adapters.flm:FLMAdapter|c3a6a4573200fecf38a47fc6e7bc78254ca362fff3b96079dc2efa9f8e27aff0",
"checkpoint_lock_id": "recipe:flm_lm1b_official_ckpt:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0",
"checkpoint_selection": {
"output": "checkpoints/official/flm_ckpt/lm1b",
"recipe": "flm_lm1b_official_ckpt",
"sampling_checkpoint": "lm1b_flm.ckpt",
"sampling_config": null,
"source": "flm",
"source_commit": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
"teacher_adapter": null,
"teacher_family": "continuous_flm"
},
"checkpoint_sha256": "1d31308c7174e804931f22248ef664cda27005747c8d5385f7971b960577d0e4",
"checkpoint_teacher_family": "continuous_flm",
"command": null,
"config_sha256": "4f12f3e823ddd03f61014b1a9c7acf32b78abfd7d5a1dfe98101d95ced6f6f84",
"dataset_id": "lm1b",
"device": null,
"environment": "dlb-flm",
"model_id": "flm",
"results_root": "/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck",
"run_id": "flm-lm1b-steps-2",
"sample_count": 1,
"seed": 42,
"source_sha256": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
"step_count": 2
},
"started_at": "2026-08-03T04:15:51.581221Z"
}
--- stderr.log tail 240 ---
Seed set to 42
/home/ruizzhao/diffusion_baseline/upstreams/flm/models/dit.py:476: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
with torch.cuda.amp.autocast(enabled=False):
/home/ruizzhao/diffusion_baseline/upstreams/flm/models/dit.py:476: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
with torch.cuda.amp.autocast(enabled=False):
Error executing job with overrides: ['mode=sample_eval', 'seed=42', 'data=lm1b-wrap', 'model=small', 'model.length=128', 'loader.batch_size=2', 'loader.eval_batch_size=32', 'sampling.num_sample_batches=1', 'sampling.steps=2', 'eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/flm_ckpt/lm1b/lm1b_flm.ckpt', 'trainer.devices=1', 'checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_hydra', '+wandb.offline=true', 'eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_samples.json', 'algo=flm', 'algo.backbone=dit', 'eval.disable_ema=False', 'sampling.solver=euler', 'algo.double_temb=False', 'sampling.gamma=0.0']
Traceback (most recent call last):
File "<frozen runpy>", line 198, in _run_module_as_main
File "<frozen runpy>", line 88, in _run_code
File "/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py", line 688, in <module>
raise SystemExit(main())
^^^^^^
File "/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py", line 683, in main
_capture_teacher(module, invocation, forwarded)
File "/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py", line 380, in _capture_teacher
_run_main(module, invocation.entrypoint, forwarded)
File "/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py", line 305, in _run_main
module.main()
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/main.py", line 94, in decorated_main
_run_hydra(
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py", line 394, in _run_hydra
_run_app(
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py", line 457, in _run_app
run_and_report(
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py", line 223, in run_and_report
raise ex
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py", line 220, in run_and_report
return func()
^^^^^^
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py", line 458, in <lambda>
lambda: hydra.run(
^^^^^^^^^^
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/hydra.py", line 132, in run
_ = ret.return_value
^^^^^^^^^^^^^^^^
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/core/utils.py", line 260, in return_value
raise self._return_value
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/core/utils.py", line 186, in run_job
ret.return_value = task_function(task_cfg)
^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py", line 452, in main
_generate_samples(**kwargs)
File "/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py", line 131, in _generate_samples
samples = model.restore_model_and_sample(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py", line 369, in capture
token_rows = result.detach().cpu().tolist()
^^^^^^^^^^^^^
AttributeError: 'list' object has no attribute 'detach'
--- stdout.log tail 120 ---
│ eps: 1.0e-08
│ ln_tune: none
│
├── trainer
│ └── _target_: lightning.Trainer
│ accelerator: cuda
│ num_nodes: 1
│ devices: 1
│ accumulate_grad_batches: 256
│ gradient_clip_val: 1.0
│ precision: bf16
│ num_sanity_val_steps: 2
│ max_steps: 1000000
│ log_every_n_steps: 100
│ limit_train_batches: 1.0
│ limit_val_batches: 1.0
│ val_check_interval: 5000
│
├── wandb
│ └── project: flm
│ notes: FLM
│ group: null
│ job_type: null
│ name: null
│ id: None_42
│ tags:
│ - lm1b
│ - flm
│ offline: true
│
├── checkpointing
│ └── save_dir: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/
│ resume_from_ckpt: false
│ resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_e
│
├── callbacks
│ └── checkpoint_every_n_steps:
│ _target_: lightning.pytorch.callbacks.ModelCheckpoint
│ save_top_k: -1
│ save_last: false
│ dirpath: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck
│ verbose: true
│ auto_insert_metric_name: false
│ every_n_train_steps: 25000
│ checkpoint_monitor:
│ _target_: lightning.pytorch.callbacks.ModelCheckpoint
│ monitor: val/nll
│ mode: min
│ save_top_k: 1
│ save_last: true
│ dirpath: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck
│ filename: best_nll
│ auto_insert_metric_name: false
│ verbose: true
│ learning_rate_monitor:
│ _target_: lightning.pytorch.callbacks.LearningRateMonitor
│ logging_interval: step
│
├── data
│ └── train: lm1b
│ valid: lm1b
│ tokenizer_name_or_path: bert-base-uncased
│ cache_dir: ./cache
│ wrap: true
│ streaming: false
│ insert_train_eos: true
│ insert_valid_eos: true
│
├── model
│ └── name: small
│ type: ddit
│ hidden_size: 768
│ cond_dim: 128
│ length: 128
│ n_blocks: 12
│ n_heads: 12
│ scale_by_sigma: true
│ dropout: 0.1
│ tie_word_embeddings: false
│ vocab_lookup: true
│
├── strategy
│ └── _target_: lightning.pytorch.strategies.DDPStrategy
│ find_unused_parameters: false
│
├── noise
│ └── type: log-linear
│ parameterization: log-linear
│ eps: 0
│ denoiser_latent_conditioning: -1
│ freeze_encoder: false
│ freeze_decoder: false
│
├── lr_scheduler
│ └── _target_: transformers.get_constant_schedule_with_warmup
│ num_warmup_steps: 2500
│
├── prior
│ └── type: none
│ latent_width: 0
│ latent_height: 0
│
└── algo
└── name: flm
backbone: dit
parameterization: mean
time_conditioning: true
T: 0
subs_masking: false
causal_attention: false
ignore_bos: false
t_max: 1.0
t_min: 0.0
double_temb: false
learnable_loss_weighting: false
[2026-08-03 12:15:56,039][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
Resuming training from checkpoint...
generation start: 2026-08-03 12:16:02
Copying EMA parameters to model
--- hydra files ---
/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_hydra/config_tree.txt
/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_hydra/dlb_pinned_upstream_main.log
/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_hydra/.hydra/config.yaml
/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_hydra/.hydra/hydra.yaml
/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_hydra/.hydra/overrides.yaml
========================================
RUN=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/fmlm/steps_2
========================================
--- files ---
total 52K
drwxr-xr-x 3 ruizzhao ds 4.0K Aug 3 12:16 .
drwxr-xr-x 3 ruizzhao ds 4.0K Aug 3 12:01 ..
-rw------- 1 ruizzhao ds 14K Aug 3 12:16 failure.json
-rw------- 1 ruizzhao ds 3.9K Aug 3 12:16 request.json
-rw------- 1 ruizzhao ds 3.6K Aug 3 12:16 stderr.log
-rw------- 1 ruizzhao ds 15K Aug 3 12:16 stdout.log
drwxr-xr-x 3 ruizzhao ds 4.0K Aug 3 12:01 upstream_hydra
--- failure.json ---
{
"command": [
"/opt/anaconda3/envs/dlb-flm/bin/python",
"-u",
"-m",
"dlb.adapters.capture",
"--upstream-entrypoint=/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py",
"--capture-path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/fmlm/steps_2/upstream_token_ids.json",
"--",
"mode=sample_eval",
"seed=42",
"data=lm1b-wrap",
"model=small",
"model.length=128",
"loader.batch_size=2",
"loader.eval_batch_size=16",
"sampling.num_sample_batches=1",
"sampling.steps=2",
"eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/fmlm_ckpt/lm1b/lm1b_fmlm.ckpt",
"trainer.devices=1",
"hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/fmlm/steps_2/upstream_hydra",
"checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/fmlm/steps_2/upstream_hydra",
"+wandb.offline=true",
"eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/fmlm/steps_2/upstream_samples.json",
"algo=fmlm",
"algo.backbone=dit",
"eval.disable_ema=False",
"sampling.solver=euler",
"algo.double_temb=True",
"algo.learnable_loss_weighting=False",
"sampling.gamma=0.8"
],
"exit_code": 1,
"finished_at": "2026-08-03T04:16:20.022460Z",
"message": "child command failed",
"run_id": "fmlm-lm1b-steps-2",
"signal": null,
"stage": "command",
"started_at": "2026-08-03T04:16:06.859083Z",
"stderr_log": "/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/fmlm/steps_2/stderr.log",
"stderr_tail": "Seed set to 42\n/home/ruizzhao/diffusion_baseline/upstreams/flm/models/dit.py:476: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n with torch.cuda.amp.autocast(enabled=False):\nError executing job with overrides: ['mode=sample_eval', 'seed=42', 'data=lm1b-wrap', 'model=small', 'model.length=128', 'loader.batch_size=2', 'loader.eval_batch_size=16', 'sampling.num_sample_batches=1', 'sampling.steps=2', 'eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/fmlm_ckpt/lm1b/lm1b_fmlm.ckpt', 'trainer.devices=1', 'checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/fmlm/steps_2/upstream_hydra', '+wandb.offline=true', 'eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/fmlm/steps_2/upstream_samples.json', 'algo=fmlm', 'algo.backbone=dit', 'eval.disable_ema=False', 'sampling.solver=euler', 'algo.double_temb=True', 'algo.learnable_loss_weighting=False', 'sampling.gamma=0.8']\nTraceback (most recent call last):\n File \"<frozen runpy>\", line 198, in _run_module_as_main\n File \"<frozen runpy>\", line 88, in _run_code\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 688, in <module>\n raise SystemExit(main())\n ^^^^^^\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 683, in main\n _capture_teacher(module, invocation, forwarded)\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 380, in _capture_teacher\n _run_main(module, invocation.entrypoint, forwarded)\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 305, in _run_main\n module.main()\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/main.py\", line 94, in decorated_main\n _run_hydra(\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 394, in _run_hydra\n _run_app(\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 457, in _run_app\n run_and_report(\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 223, in run_and_report\n raise ex\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 220, in run_and_report\n return func()\n ^^^^^^\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 458, in <lambda>\n lambda: hydra.run(\n ^^^^^^^^^^\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/hydra.py\", line 132, in run\n _ = ret.return_value\n ^^^^^^^^^^^^^^^^\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/core/utils.py\", line 260, in return_value\n raise self._return_value\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/core/utils.py\", line 186, in run_job\n ret.return_value = task_function(task_cfg)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py\", line 452, in main\n _generate_samples(**kwargs)\n File \"/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py\", line 131, in _generate_samples\n samples = model.restore_model_and_sample(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 369, in capture\n token_rows = result.detach().cpu().tolist()\n ^^^^^^^^^^^^^\nAttributeError: 'list' object has no attribute 'detach'\n",
"stdout_log": "/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/fmlm/steps_2/stdout.log",
"stdout_tail": " \n\u2502 - lm1b \n\u2502 - fmlm \n\u2502 offline: true \n\u2502 \n\u251c\u2500\u2500 checkpointing\n\u2502 \u2514\u2500\u2500 save_dir: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/\n\u2502 resume_from_ckpt: false \n\u2502 resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_e\n\u2502 \n\u251c\u2500\u2500 callbacks\n\u2502 \u2514\u2500\u2500 checkpoint_every_n_steps: \n\u2502 _target_: lightning.pytorch.callbacks.ModelCheckpoint \n\u2502 save_top_k: -1 \n\u2502 save_last: false \n\u2502 dirpath: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck\n\u2502 verbose: true \n\u2502 auto_insert_metric_name: false \n\u2502 every_n_train_steps: 25000 \n\u2502 checkpoint_monitor: \n\u2502 _target_: lightning.pytorch.callbacks.ModelCheckpoint \n\u2502 monitor: val/nll \n\u2502 mode: min \n\u2502 save_top_k: 1 \n\u2502 save_last: true \n\u2502 dirpath: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck\n\u2502 filename: best_nll \n\u2502 auto_insert_metric_name: false \n\u2502 verbose: true \n\u2502 learning_rate_monitor: \n\u2502 _target_: lightning.pytorch.callbacks.LearningRateMonitor \n\u2502 logging_interval: step \n\u2502 \n\u251c\u2500\u2500 data\n\u2502 \u2514\u2500\u2500 train: lm1b \n\u2502 valid: lm1b \n\u2502 tokenizer_name_or_path: bert-base-uncased \n\u2502 cache_dir: ./cache \n\u2502 wrap: true \n\u2502 streaming: false \n\u2502 insert_train_eos: true \n\u2502 insert_valid_eos: true \n\u2502 \n\u251c\u2500\u2500 model\n\u2502 \u2514\u2500\u2500 name: small \n\u2502 type: ddit \n\u2502 hidden_size: 768 \n\u2502 cond_dim: 128 \n\u2502 length: 128 \n\u2502 n_blocks: 12 \n\u2502 n_heads: 12 \n\u2502 scale_by_sigma: true \n\u2502 dropout: 0.1 \n\u2502 tie_word_embeddings: false \n\u2502 vocab_lookup: true \n\u2502 \n\u251c\u2500\u2500 strategy\n\u2502 \u2514\u2500\u2500 _target_: lightning.pytorch.strategies.DDPStrategy \n\u2502 find_unused_parameters: false \n\u2502 \n\u251c\u2500\u2500 noise\n\u2502 \u2514\u2500\u2500 type: log-linear \n\u2502 parameterization: log-linear \n\u2502 eps: 0 \n\u2502 denoiser_latent_conditioning: -1 \n\u2502 freeze_encoder: false \n\u2502 freeze_decoder: false \n\u2502 \n\u251c\u2500\u2500 lr_scheduler\n\u2502 \u2514\u2500\u2500 _target_: transformers.get_constant_schedule_with_warmup \n\u2502 num_warmup_steps: 2500 \n\u2502 \n\u251c\u2500\u2500 prior\n\u2502 \u2514\u2500\u2500 type: none \n\u2502 latent_width: 0 \n\u2502 latent_height: 0 \n\u2502 \n\u2514\u2500\u2500 algo\n \u2514\u2500\u2500 name: fmlm \n backbone: dit \n parameterization: mean \n time_conditioning: true \n T: 0 \n subs_masking: false \n causal_attention: false \n t_max: 1.0 \n t_min: 0.0 \n double_temb: true \n add_boundary: fixed \n diagonal_fraction: 0.5 \n offdiagonal_sampling: uniform_diff \n set_midpoint: midpoint \n boundary_prob: 32 \n distillation_method: PSD \n entmax_temp_lsd: 2.0 \n backprop_entmax_temp_lsd: false \n learnable_loss_weighting: false \n use_mse_loss_psd: false \n rescale_offdiag_loss_psd: false \n use_ema_for_psd_target: false \n use_teacher_for_D_t_lsd: false \n initialize_student_from_teacher: true \n teacher_path: '' \n \n[2026-08-03 12:16:11,369][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.\nResuming training from checkpoint...\ngeneration start: 2026-08-03 12:16:18\nCopying EMA parameters to model\nSampling with 2 steps\n"
}
--- request.json ---
{
"command": [
"/opt/anaconda3/envs/dlb-flm/bin/python",
"-u",
"-m",
"dlb.adapters.capture",
"--upstream-entrypoint=/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py",
"--capture-path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/fmlm/steps_2/upstream_token_ids.json",
"--",
"mode=sample_eval",
"seed=42",
"data=lm1b-wrap",
"model=small",
"model.length=128",
"loader.batch_size=2",
"loader.eval_batch_size=16",
"sampling.num_sample_batches=1",
"sampling.steps=2",
"eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/fmlm_ckpt/lm1b/lm1b_fmlm.ckpt",
"trainer.devices=1",
"hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/fmlm/steps_2/upstream_hydra",
"checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/fmlm/steps_2/upstream_hydra",
"+wandb.offline=true",
"eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/fmlm/steps_2/upstream_samples.json",
"algo=fmlm",
"algo.backbone=dit",
"eval.disable_ema=False",
"sampling.solver=euler",
"algo.double_temb=True",
"algo.learnable_loss_weighting=False",
"sampling.gamma=0.8"
],
"identity": {
"adapter_identity": "dlb.adapters.flm:v2|dlb.adapters.flm:FLMAdapter|c3a6a4573200fecf38a47fc6e7bc78254ca362fff3b96079dc2efa9f8e27aff0",
"checkpoint_lock_id": "recipe:fmlm_lm1b_official_ckpt:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0",
"checkpoint_selection": {
"output": "checkpoints/official/fmlm_ckpt/lm1b",
"recipe": "fmlm_lm1b_official_ckpt",
"sampling_checkpoint": "lm1b_fmlm.ckpt",
"sampling_config": null,
"source": "flm",
"source_commit": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
"teacher_adapter": null,
"teacher_family": "continuous_flm"
},
"checkpoint_sha256": "b01b03c59150978cdd4865283b71028ec749936c21769eb32bfe2907f38d8918",
"checkpoint_teacher_family": "continuous_flm",
"command_sha256": "633e16611b3a9fc7acbf6586d17084db8d401ab84e8589b0371944cd8a34ee3a",
"config_sha256": "4f12f3e823ddd03f61014b1a9c7acf32b78abfd7d5a1dfe98101d95ced6f6f84",
"dataset_id": "lm1b",
"device": null,
"environment": "dlb-flm",
"model_id": "fmlm",
"results_root": "/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck",
"run_id": "fmlm-lm1b-steps-2",
"sample_count": 1,
"seed": 42,
"source_sha256": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
"step_count": 2
},
"request": {
"adapter_identity": "dlb.adapters.flm:v2|dlb.adapters.flm:FLMAdapter|c3a6a4573200fecf38a47fc6e7bc78254ca362fff3b96079dc2efa9f8e27aff0",
"checkpoint_lock_id": "recipe:fmlm_lm1b_official_ckpt:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0",
"checkpoint_selection": {
"output": "checkpoints/official/fmlm_ckpt/lm1b",
"recipe": "fmlm_lm1b_official_ckpt",
"sampling_checkpoint": "lm1b_fmlm.ckpt",
"sampling_config": null,
"source": "flm",
"source_commit": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
"teacher_adapter": null,
"teacher_family": "continuous_flm"
},
"checkpoint_sha256": "b01b03c59150978cdd4865283b71028ec749936c21769eb32bfe2907f38d8918",
"checkpoint_teacher_family": "continuous_flm",
"command": null,
"config_sha256": "4f12f3e823ddd03f61014b1a9c7acf32b78abfd7d5a1dfe98101d95ced6f6f84",
"dataset_id": "lm1b",
"device": null,
"environment": "dlb-flm",
"model_id": "fmlm",
"results_root": "/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck",
"run_id": "fmlm-lm1b-steps-2",
"sample_count": 1,
"seed": 42,
"source_sha256": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
"step_count": 2
},
"started_at": "2026-08-03T04:16:06.859083Z"
}
--- stderr.log tail 240 ---
Seed set to 42
/home/ruizzhao/diffusion_baseline/upstreams/flm/models/dit.py:476: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
with torch.cuda.amp.autocast(enabled=False):
Error executing job with overrides: ['mode=sample_eval', 'seed=42', 'data=lm1b-wrap', 'model=small', 'model.length=128', 'loader.batch_size=2', 'loader.eval_batch_size=16', 'sampling.num_sample_batches=1', 'sampling.steps=2', 'eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/fmlm_ckpt/lm1b/lm1b_fmlm.ckpt', 'trainer.devices=1', 'checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/fmlm/steps_2/upstream_hydra', '+wandb.offline=true', 'eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/fmlm/steps_2/upstream_samples.json', 'algo=fmlm', 'algo.backbone=dit', 'eval.disable_ema=False', 'sampling.solver=euler', 'algo.double_temb=True', 'algo.learnable_loss_weighting=False', 'sampling.gamma=0.8']
Traceback (most recent call last):
File "<frozen runpy>", line 198, in _run_module_as_main
File "<frozen runpy>", line 88, in _run_code
File "/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py", line 688, in <module>
raise SystemExit(main())
^^^^^^
File "/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py", line 683, in main
_capture_teacher(module, invocation, forwarded)
File "/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py", line 380, in _capture_teacher
_run_main(module, invocation.entrypoint, forwarded)
File "/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py", line 305, in _run_main
module.main()
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/main.py", line 94, in decorated_main
_run_hydra(
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py", line 394, in _run_hydra
_run_app(
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py", line 457, in _run_app
run_and_report(
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py", line 223, in run_and_report
raise ex
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py", line 220, in run_and_report
return func()
^^^^^^
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py", line 458, in <lambda>
lambda: hydra.run(
^^^^^^^^^^
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/hydra.py", line 132, in run
_ = ret.return_value
^^^^^^^^^^^^^^^^
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/core/utils.py", line 260, in return_value
raise self._return_value
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/core/utils.py", line 186, in run_job
ret.return_value = task_function(task_cfg)
^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py", line 452, in main
_generate_samples(**kwargs)
File "/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py", line 131, in _generate_samples
samples = model.restore_model_and_sample(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py", line 369, in capture
token_rows = result.detach().cpu().tolist()
^^^^^^^^^^^^^
AttributeError: 'list' object has no attribute 'detach'
--- stdout.log tail 120 ---
│ limit_train_batches: 1.0
│ limit_val_batches: 1.0
│ val_check_interval: 5000
│
├── wandb
│ └── project: flm
│ notes: FLM
│ group: null
│ job_type: null
│ name: null
│ id: None_42
│ tags:
│ - lm1b
│ - fmlm
│ offline: true
│
├── checkpointing
│ └── save_dir: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/
│ resume_from_ckpt: false
│ resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_e
│
├── callbacks
│ └── checkpoint_every_n_steps:
│ _target_: lightning.pytorch.callbacks.ModelCheckpoint
│ save_top_k: -1
│ save_last: false
│ dirpath: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck
│ verbose: true
│ auto_insert_metric_name: false
│ every_n_train_steps: 25000
│ checkpoint_monitor:
│ _target_: lightning.pytorch.callbacks.ModelCheckpoint
│ monitor: val/nll
│ mode: min
│ save_top_k: 1
│ save_last: true
│ dirpath: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck
│ filename: best_nll
│ auto_insert_metric_name: false
│ verbose: true
│ learning_rate_monitor:
│ _target_: lightning.pytorch.callbacks.LearningRateMonitor
│ logging_interval: step
│
├── data
│ └── train: lm1b
│ valid: lm1b
│ tokenizer_name_or_path: bert-base-uncased
│ cache_dir: ./cache
│ wrap: true
│ streaming: false
│ insert_train_eos: true
│ insert_valid_eos: true
│
├── model
│ └── name: small
│ type: ddit
│ hidden_size: 768
│ cond_dim: 128
│ length: 128
│ n_blocks: 12
│ n_heads: 12
│ scale_by_sigma: true
│ dropout: 0.1
│ tie_word_embeddings: false
│ vocab_lookup: true
│
├── strategy
│ └── _target_: lightning.pytorch.strategies.DDPStrategy
│ find_unused_parameters: false
│
├── noise
│ └── type: log-linear
│ parameterization: log-linear
│ eps: 0
│ denoiser_latent_conditioning: -1
│ freeze_encoder: false
│ freeze_decoder: false
│
├── lr_scheduler
│ └── _target_: transformers.get_constant_schedule_with_warmup
│ num_warmup_steps: 2500
│
├── prior
│ └── type: none
│ latent_width: 0
│ latent_height: 0
│
└── algo
└── name: fmlm
backbone: dit
parameterization: mean
time_conditioning: true
T: 0
subs_masking: false
causal_attention: false
t_max: 1.0
t_min: 0.0
double_temb: true
add_boundary: fixed
diagonal_fraction: 0.5
offdiagonal_sampling: uniform_diff
set_midpoint: midpoint
boundary_prob: 32
distillation_method: PSD
entmax_temp_lsd: 2.0
backprop_entmax_temp_lsd: false
learnable_loss_weighting: false
use_mse_loss_psd: false
rescale_offdiag_loss_psd: false
use_ema_for_psd_target: false
use_teacher_for_D_t_lsd: false
initialize_student_from_teacher: true
teacher_path: ''
[2026-08-03 12:16:11,369][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
Resuming training from checkpoint...
generation start: 2026-08-03 12:16:18
Copying EMA parameters to model
Sampling with 2 steps
--- hydra files ---
/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/fmlm/steps_2/upstream_hydra/config_tree.txt
/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/fmlm/steps_2/upstream_hydra/dlb_pinned_upstream_main.log
/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/fmlm/steps_2/upstream_hydra/.hydra/config.yaml
/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/fmlm/steps_2/upstream_hydra/.hydra/hydra.yaml
/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/fmlm/steps_2/upstream_hydra/.hydra/overrides.yaml
========================================
RUN=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/flm/steps_2
========================================
--- files ---
total 52K
drwxr-xr-x 3 ruizzhao ds 4.0K Aug 3 12:16 .
drwxr-xr-x 3 ruizzhao ds 4.0K Aug 3 12:01 ..
-rw------- 1 ruizzhao ds 14K Aug 3 12:16 failure.json
-rw------- 1 ruizzhao ds 3.9K Aug 3 12:16 request.json
-rw------- 1 ruizzhao ds 3.6K Aug 3 12:16 stderr.log
-rw------- 1 ruizzhao ds 14K Aug 3 12:16 stdout.log
drwxr-xr-x 3 ruizzhao ds 4.0K Aug 3 12:01 upstream_hydra
--- failure.json ---
{
"command": [
"/opt/anaconda3/envs/dlb-flm/bin/python",
"-u",
"-m",
"dlb.adapters.capture",
"--upstream-entrypoint=/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py",
"--capture-path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/flm/steps_2/upstream_token_ids.json",
"--",
"mode=sample_eval",
"seed=42",
"data=openwebtext-split",
"model=small",
"model.length=1024",
"loader.batch_size=2",
"loader.eval_batch_size=16",
"sampling.num_sample_batches=1",
"sampling.steps=2",
"eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/flm_ckpt/owt/owt_flm.ckpt",
"trainer.devices=1",
"hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/flm/steps_2/upstream_hydra",
"checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/flm/steps_2/upstream_hydra",
"+wandb.offline=true",
"eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/flm/steps_2/upstream_samples.json",
"algo=flm",
"algo.backbone=dit",
"eval.disable_ema=False",
"sampling.solver=euler",
"algo.double_temb=False",
"sampling.gamma=0.0"
],
"exit_code": 1,
"finished_at": "2026-08-03T04:16:36.652202Z",
"message": "child command failed",
"run_id": "flm-owt-steps-2",
"signal": null,
"stage": "command",
"started_at": "2026-08-03T04:16:23.419382Z",
"stderr_log": "/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/flm/steps_2/stderr.log",
"stderr_tail": "Seed set to 42\n/home/ruizzhao/diffusion_baseline/upstreams/flm/models/dit.py:476: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n with torch.cuda.amp.autocast(enabled=False):\nError executing job with overrides: ['mode=sample_eval', 'seed=42', 'data=openwebtext-split', 'model=small', 'model.length=1024', 'loader.batch_size=2', 'loader.eval_batch_size=16', 'sampling.num_sample_batches=1', 'sampling.steps=2', 'eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/flm_ckpt/owt/owt_flm.ckpt', 'trainer.devices=1', 'checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/flm/steps_2/upstream_hydra', '+wandb.offline=true', 'eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/flm/steps_2/upstream_samples.json', 'algo=flm', 'algo.backbone=dit', 'eval.disable_ema=False', 'sampling.solver=euler', 'algo.double_temb=False', 'sampling.gamma=0.0']\nTraceback (most recent call last):\n File \"<frozen runpy>\", line 198, in _run_module_as_main\n File \"<frozen runpy>\", line 88, in _run_code\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 688, in <module>\n raise SystemExit(main())\n ^^^^^^\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 683, in main\n _capture_teacher(module, invocation, forwarded)\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 380, in _capture_teacher\n _run_main(module, invocation.entrypoint, forwarded)\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 305, in _run_main\n module.main()\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/main.py\", line 94, in decorated_main\n _run_hydra(\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 394, in _run_hydra\n _run_app(\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 457, in _run_app\n run_and_report(\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 223, in run_and_report\n raise ex\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 220, in run_and_report\n return func()\n ^^^^^^\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 458, in <lambda>\n lambda: hydra.run(\n ^^^^^^^^^^\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/hydra.py\", line 132, in run\n _ = ret.return_value\n ^^^^^^^^^^^^^^^^\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/core/utils.py\", line 260, in return_value\n raise self._return_value\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/core/utils.py\", line 186, in run_job\n ret.return_value = task_function(task_cfg)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py\", line 452, in main\n _generate_samples(**kwargs)\n File \"/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py\", line 131, in _generate_samples\n samples = model.restore_model_and_sample(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 369, in capture\n token_rows = result.detach().cpu().tolist()\n ^^^^^^^^^^^^^\nAttributeError: 'list' object has no attribute 'detach'\n",
"stdout_log": "/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/flm/steps_2/stdout.log",
"stdout_tail": "_steps: 1000000 \n\u2502 log_every_n_steps: 100 \n\u2502 limit_train_batches: 1.0 \n\u2502 limit_val_batches: 1.0 \n\u2502 val_check_interval: 5000 \n\u2502 \n\u251c\u2500\u2500 wandb\n\u2502 \u2514\u2500\u2500 project: flm \n\u2502 notes: FLM \n\u2502 group: null \n\u2502 job_type: null \n\u2502 name: null \n\u2502 id: None_42 \n\u2502 tags: \n\u2502 - openwebtext-train \n\u2502 - flm \n\u2502 offline: true \n\u2502 \n\u251c\u2500\u2500 checkpointing\n\u2502 \u2514\u2500\u2500 save_dir: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/\n\u2502 resume_from_ckpt: false \n\u2502 resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_e\n\u2502 \n\u251c\u2500\u2500 callbacks\n\u2502 \u2514\u2500\u2500 checkpoint_every_n_steps: \n\u2502 _target_: lightning.pytorch.callbacks.ModelCheckpoint \n\u2502 save_top_k: -1 \n\u2502 save_last: false \n\u2502 dirpath: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck\n\u2502 verbose: true \n\u2502 auto_insert_metric_name: false \n\u2502 every_n_train_steps: 25000 \n\u2502 checkpoint_monitor: \n\u2502 _target_: lightning.pytorch.callbacks.ModelCheckpoint \n\u2502 monitor: val/nll \n\u2502 mode: min \n\u2502 save_top_k: 1 \n\u2502 save_last: true \n\u2502 dirpath: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck\n\u2502 filename: best_nll \n\u2502 auto_insert_metric_name: false \n\u2502 verbose: true \n\u2502 learning_rate_monitor: \n\u2502 _target_: lightning.pytorch.callbacks.LearningRateMonitor \n\u2502 logging_interval: step \n\u2502 \n\u251c\u2500\u2500 data\n\u2502 \u2514\u2500\u2500 train: openwebtext-train \n\u2502 valid: openwebtext-valid \n\u2502 tokenizer_name_or_path: gpt2 \n\u2502 cache_dir: /home/ruizzhao/diffusion_baseline/datasets/openwebtext \n\u2502 wrap: true \n\u2502 streaming: false \n\u2502 insert_train_eos: true \n\u2502 insert_valid_eos: true \n\u2502 \n\u251c\u2500\u2500 model\n\u2502 \u2514\u2500\u2500 name: small \n\u2502 type: ddit \n\u2502 hidden_size: 768 \n\u2502 cond_dim: 128 \n\u2502 length: 1024 \n\u2502 n_blocks: 12 \n\u2502 n_heads: 12 \n\u2502 scale_by_sigma: true \n\u2502 dropout: 0.1 \n\u2502 tie_word_embeddings: false \n\u2502 vocab_lookup: true \n\u2502 \n\u251c\u2500\u2500 strategy\n\u2502 \u2514\u2500\u2500 _target_: lightning.pytorch.strategies.DDPStrategy \n\u2502 find_unused_parameters: false \n\u2502 \n\u251c\u2500\u2500 noise\n\u2502 \u2514\u2500\u2500 type: log-linear \n\u2502 parameterization: log-linear \n\u2502 eps: 0 \n\u2502 denoiser_latent_conditioning: -1 \n\u2502 freeze_encoder: false \n\u2502 freeze_decoder: false \n\u2502 \n\u251c\u2500\u2500 lr_scheduler\n\u2502 \u2514\u2500\u2500 _target_: transformers.get_constant_schedule_with_warmup \n\u2502 num_warmup_steps: 2500 \n\u2502 \n\u251c\u2500\u2500 prior\n\u2502 \u2514\u2500\u2500 type: none \n\u2502 latent_width: 0 \n\u2502 latent_height: 0 \n\u2502 \n\u2514\u2500\u2500 algo\n \u2514\u2500\u2500 name: flm \n backbone: dit \n parameterization: mean \n time_conditioning: true \n T: 0 \n subs_masking: false \n causal_attention: false \n ignore_bos: false \n t_max: 1.0 \n t_min: 0.0 \n double_temb: false \n learnable_loss_weighting: false \n \n[2026-08-03 12:16:27,966][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.\nResuming training from checkpoint...\ngeneration start: 2026-08-03 12:16:34\nCopying EMA parameters to model\n"
}
--- request.json ---
{
"command": [
"/opt/anaconda3/envs/dlb-flm/bin/python",
"-u",
"-m",
"dlb.adapters.capture",
"--upstream-entrypoint=/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py",
"--capture-path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/flm/steps_2/upstream_token_ids.json",
"--",
"mode=sample_eval",
"seed=42",
"data=openwebtext-split",
"model=small",
"model.length=1024",
"loader.batch_size=2",
"loader.eval_batch_size=16",
"sampling.num_sample_batches=1",
"sampling.steps=2",
"eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/flm_ckpt/owt/owt_flm.ckpt",
"trainer.devices=1",
"hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/flm/steps_2/upstream_hydra",
"checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/flm/steps_2/upstream_hydra",
"+wandb.offline=true",
"eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/flm/steps_2/upstream_samples.json",
"algo=flm",
"algo.backbone=dit",
"eval.disable_ema=False",
"sampling.solver=euler",
"algo.double_temb=False",
"sampling.gamma=0.0"
],
"identity": {
"adapter_identity": "dlb.adapters.flm:v2|dlb.adapters.flm:FLMAdapter|c3a6a4573200fecf38a47fc6e7bc78254ca362fff3b96079dc2efa9f8e27aff0",
"checkpoint_lock_id": "recipe:flm_owt_official_ckpt:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0",
"checkpoint_selection": {
"output": "checkpoints/official/flm_ckpt/owt",
"recipe": "flm_owt_official_ckpt",
"sampling_checkpoint": "owt_flm.ckpt",
"sampling_config": null,
"source": "flm",
"source_commit": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
"teacher_adapter": null,
"teacher_family": "continuous_flm"
},
"checkpoint_sha256": "aac1a3b57497f69ee10aee2309431ce35e9e9b9d3ad4490f8315da1488a55023",
"checkpoint_teacher_family": "continuous_flm",
"command_sha256": "02a8cecee3003c5ea79140dd4bb47e4370a70d04e46f525abb4b50b8bfa6e472",
"config_sha256": "4f12f3e823ddd03f61014b1a9c7acf32b78abfd7d5a1dfe98101d95ced6f6f84",
"dataset_id": "owt",
"device": null,
"environment": "dlb-flm",
"model_id": "flm",
"results_root": "/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck",
"run_id": "flm-owt-steps-2",
"sample_count": 1,
"seed": 42,
"source_sha256": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
"step_count": 2
},
"request": {
"adapter_identity": "dlb.adapters.flm:v2|dlb.adapters.flm:FLMAdapter|c3a6a4573200fecf38a47fc6e7bc78254ca362fff3b96079dc2efa9f8e27aff0",
"checkpoint_lock_id": "recipe:flm_owt_official_ckpt:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0",
"checkpoint_selection": {
"output": "checkpoints/official/flm_ckpt/owt",
"recipe": "flm_owt_official_ckpt",
"sampling_checkpoint": "owt_flm.ckpt",
"sampling_config": null,
"source": "flm",
"source_commit": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
"teacher_adapter": null,
"teacher_family": "continuous_flm"
},
"checkpoint_sha256": "aac1a3b57497f69ee10aee2309431ce35e9e9b9d3ad4490f8315da1488a55023",
"checkpoint_teacher_family": "continuous_flm",
"command": null,
"config_sha256": "4f12f3e823ddd03f61014b1a9c7acf32b78abfd7d5a1dfe98101d95ced6f6f84",
"dataset_id": "owt",
"device": null,
"environment": "dlb-flm",
"model_id": "flm",
"results_root": "/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck",
"run_id": "flm-owt-steps-2",
"sample_count": 1,
"seed": 42,
"source_sha256": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
"step_count": 2
},
"started_at": "2026-08-03T04:16:23.419382Z"
}
--- stderr.log tail 240 ---
Seed set to 42
/home/ruizzhao/diffusion_baseline/upstreams/flm/models/dit.py:476: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
with torch.cuda.amp.autocast(enabled=False):
Error executing job with overrides: ['mode=sample_eval', 'seed=42', 'data=openwebtext-split', 'model=small', 'model.length=1024', 'loader.batch_size=2', 'loader.eval_batch_size=16', 'sampling.num_sample_batches=1', 'sampling.steps=2', 'eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/flm_ckpt/owt/owt_flm.ckpt', 'trainer.devices=1', 'checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/flm/steps_2/upstream_hydra', '+wandb.offline=true', 'eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/flm/steps_2/upstream_samples.json', 'algo=flm', 'algo.backbone=dit', 'eval.disable_ema=False', 'sampling.solver=euler', 'algo.double_temb=False', 'sampling.gamma=0.0']
Traceback (most recent call last):
File "<frozen runpy>", line 198, in _run_module_as_main
File "<frozen runpy>", line 88, in _run_code
File "/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py", line 688, in <module>
raise SystemExit(main())
^^^^^^
File "/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py", line 683, in main
_capture_teacher(module, invocation, forwarded)
File "/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py", line 380, in _capture_teacher
_run_main(module, invocation.entrypoint, forwarded)
File "/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py", line 305, in _run_main
module.main()
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/main.py", line 94, in decorated_main
_run_hydra(
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py", line 394, in _run_hydra
_run_app(
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py", line 457, in _run_app
run_and_report(
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py", line 223, in run_and_report
raise ex
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py", line 220, in run_and_report
return func()
^^^^^^
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py", line 458, in <lambda>
lambda: hydra.run(
^^^^^^^^^^
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/hydra.py", line 132, in run
_ = ret.return_value
^^^^^^^^^^^^^^^^
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/core/utils.py", line 260, in return_value
raise self._return_value
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/core/utils.py", line 186, in run_job
ret.return_value = task_function(task_cfg)
^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py", line 452, in main
_generate_samples(**kwargs)
File "/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py", line 131, in _generate_samples
samples = model.restore_model_and_sample(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py", line 369, in capture
token_rows = result.detach().cpu().tolist()
^^^^^^^^^^^^^
AttributeError: 'list' object has no attribute 'detach'
--- stdout.log tail 120 ---
│ eps: 1.0e-08
│ ln_tune: none
│
├── trainer
│ └── _target_: lightning.Trainer
│ accelerator: cuda
│ num_nodes: 1
│ devices: 1
│ accumulate_grad_batches: 256
│ gradient_clip_val: 1.0
│ precision: bf16
│ num_sanity_val_steps: 2
│ max_steps: 1000000
│ log_every_n_steps: 100
│ limit_train_batches: 1.0
│ limit_val_batches: 1.0
│ val_check_interval: 5000
│
├── wandb
│ └── project: flm
│ notes: FLM
│ group: null
│ job_type: null
│ name: null
│ id: None_42
│ tags:
│ - openwebtext-train
│ - flm
│ offline: true
│
├── checkpointing
│ └── save_dir: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/
│ resume_from_ckpt: false
│ resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_e
│
├── callbacks
│ └── checkpoint_every_n_steps:
│ _target_: lightning.pytorch.callbacks.ModelCheckpoint
│ save_top_k: -1
│ save_last: false
│ dirpath: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck
│ verbose: true
│ auto_insert_metric_name: false
│ every_n_train_steps: 25000
│ checkpoint_monitor:
│ _target_: lightning.pytorch.callbacks.ModelCheckpoint
│ monitor: val/nll
│ mode: min
│ save_top_k: 1
│ save_last: true
│ dirpath: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck
│ filename: best_nll
│ auto_insert_metric_name: false
│ verbose: true
│ learning_rate_monitor:
│ _target_: lightning.pytorch.callbacks.LearningRateMonitor
│ logging_interval: step
│
├── data
│ └── train: openwebtext-train
│ valid: openwebtext-valid
│ tokenizer_name_or_path: gpt2
│ cache_dir: /home/ruizzhao/diffusion_baseline/datasets/openwebtext
│ wrap: true
│ streaming: false
│ insert_train_eos: true
│ insert_valid_eos: true
│
├── model
│ └── name: small
│ type: ddit
│ hidden_size: 768
│ cond_dim: 128
│ length: 1024
│ n_blocks: 12
│ n_heads: 12
│ scale_by_sigma: true
│ dropout: 0.1
│ tie_word_embeddings: false
│ vocab_lookup: true
│
├── strategy
│ └── _target_: lightning.pytorch.strategies.DDPStrategy
│ find_unused_parameters: false
│
├── noise
│ └── type: log-linear
│ parameterization: log-linear
│ eps: 0
│ denoiser_latent_conditioning: -1
│ freeze_encoder: false
│ freeze_decoder: false
│
├── lr_scheduler
│ └── _target_: transformers.get_constant_schedule_with_warmup
│ num_warmup_steps: 2500
│
├── prior
│ └── type: none
│ latent_width: 0
│ latent_height: 0
│
└── algo
└── name: flm
backbone: dit
parameterization: mean
time_conditioning: true
T: 0
subs_masking: false
causal_attention: false
ignore_bos: false
t_max: 1.0
t_min: 0.0
double_temb: false
learnable_loss_weighting: false
[2026-08-03 12:16:27,966][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
Resuming training from checkpoint...
generation start: 2026-08-03 12:16:34
Copying EMA parameters to model
--- hydra files ---
/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/flm/steps_2/upstream_hydra/config_tree.txt
/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/flm/steps_2/upstream_hydra/dlb_pinned_upstream_main.log
/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/flm/steps_2/upstream_hydra/.hydra/config.yaml
/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/flm/steps_2/upstream_hydra/.hydra/hydra.yaml
/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/flm/steps_2/upstream_hydra/.hydra/overrides.yaml
========================================
RUN=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/fmlm/steps_2
========================================
--- files ---
total 52K
drwxr-xr-x 3 ruizzhao ds 4.0K Aug 3 12:16 .
drwxr-xr-x 3 ruizzhao ds 4.0K Aug 3 12:01 ..
-rw------- 1 ruizzhao ds 14K Aug 3 12:16 failure.json
-rw------- 1 ruizzhao ds 3.9K Aug 3 12:16 request.json
-rw------- 1 ruizzhao ds 3.8K Aug 3 12:16 stderr.log
-rw------- 1 ruizzhao ds 15K Aug 3 12:16 stdout.log
drwxr-xr-x 3 ruizzhao ds 4.0K Aug 3 12:01 upstream_hydra
--- failure.json ---
{
"command": [
"/opt/anaconda3/envs/dlb-flm/bin/python",
"-u",
"-m",
"dlb.adapters.capture",
"--upstream-entrypoint=/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py",
"--capture-path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/fmlm/steps_2/upstream_token_ids.json",
"--",
"mode=sample_eval",
"seed=42",
"data=openwebtext-split",
"model=small",
"model.length=1024",
"loader.batch_size=2",
"loader.eval_batch_size=16",
"sampling.num_sample_batches=1",
"sampling.steps=2",
"eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/fmlm_ckpt/owt/owt_fmlm.ckpt",
"trainer.devices=1",
"hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/fmlm/steps_2/upstream_hydra",
"checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/fmlm/steps_2/upstream_hydra",
"+wandb.offline=true",
"eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/fmlm/steps_2/upstream_samples.json",
"algo=fmlm",
"algo.backbone=dit",
"eval.disable_ema=False",
"sampling.solver=euler",
"algo.double_temb=True",
"algo.learnable_loss_weighting=False",
"sampling.gamma=1.0"
],
"exit_code": 1,
"finished_at": "2026-08-03T04:16:52.962157Z",
"message": "child command failed",
"run_id": "fmlm-owt-steps-2",
"signal": null,
"stage": "command",
"started_at": "2026-08-03T04:16:40.046665Z",
"stderr_log": "/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/fmlm/steps_2/stderr.log",
"stderr_tail": "Seed set to 42\n/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/lightning/pytorch/utilities/migration/utils.py:56: The loaded checkpoint was produced with Lightning v2.6.1, which is newer than your current Lightning version: v2.6.0\n/home/ruizzhao/diffusion_baseline/upstreams/flm/models/dit.py:476: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n with torch.cuda.amp.autocast(enabled=False):\nError executing job with overrides: ['mode=sample_eval', 'seed=42', 'data=openwebtext-split', 'model=small', 'model.length=1024', 'loader.batch_size=2', 'loader.eval_batch_size=16', 'sampling.num_sample_batches=1', 'sampling.steps=2', 'eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/fmlm_ckpt/owt/owt_fmlm.ckpt', 'trainer.devices=1', 'checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/fmlm/steps_2/upstream_hydra', '+wandb.offline=true', 'eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/fmlm/steps_2/upstream_samples.json', 'algo=fmlm', 'algo.backbone=dit', 'eval.disable_ema=False', 'sampling.solver=euler', 'algo.double_temb=True', 'algo.learnable_loss_weighting=False', 'sampling.gamma=1.0']\nTraceback (most recent call last):\n File \"<frozen runpy>\", line 198, in _run_module_as_main\n File \"<frozen runpy>\", line 88, in _run_code\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 688, in <module>\n raise SystemExit(main())\n ^^^^^^\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 683, in main\n _capture_teacher(module, invocation, forwarded)\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 380, in _capture_teacher\n _run_main(module, invocation.entrypoint, forwarded)\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 305, in _run_main\n module.main()\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/main.py\", line 94, in decorated_main\n _run_hydra(\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 394, in _run_hydra\n _run_app(\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 457, in _run_app\n run_and_report(\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 223, in run_and_report\n raise ex\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 220, in run_and_report\n return func()\n ^^^^^^\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 458, in <lambda>\n lambda: hydra.run(\n ^^^^^^^^^^\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/hydra.py\", line 132, in run\n _ = ret.return_value\n ^^^^^^^^^^^^^^^^\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/core/utils.py\", line 260, in return_value\n raise self._return_value\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/core/utils.py\", line 186, in run_job\n ret.return_value = task_function(task_cfg)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py\", line 452, in main\n _generate_samples(**kwargs)\n File \"/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py\", line 131, in _generate_samples\n samples = model.restore_model_and_sample(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 369, in capture\n token_rows = result.detach().cpu().tolist()\n ^^^^^^^^^^^^^\nAttributeError: 'list' object has no attribute 'detach'\n",
"stdout_log": "/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/fmlm/steps_2/stdout.log",
"stdout_tail": " \n\u2502 - openwebtext-train \n\u2502 - fmlm \n\u2502 offline: true \n\u2502 \n\u251c\u2500\u2500 checkpointing\n\u2502 \u2514\u2500\u2500 save_dir: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/\n\u2502 resume_from_ckpt: false \n\u2502 resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_e\n\u2502 \n\u251c\u2500\u2500 callbacks\n\u2502 \u2514\u2500\u2500 checkpoint_every_n_steps: \n\u2502 _target_: lightning.pytorch.callbacks.ModelCheckpoint \n\u2502 save_top_k: -1 \n\u2502 save_last: false \n\u2502 dirpath: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck\n\u2502 verbose: true \n\u2502 auto_insert_metric_name: false \n\u2502 every_n_train_steps: 25000 \n\u2502 checkpoint_monitor: \n\u2502 _target_: lightning.pytorch.callbacks.ModelCheckpoint \n\u2502 monitor: val/nll \n\u2502 mode: min \n\u2502 save_top_k: 1 \n\u2502 save_last: true \n\u2502 dirpath: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck\n\u2502 filename: best_nll \n\u2502 auto_insert_metric_name: false \n\u2502 verbose: true \n\u2502 learning_rate_monitor: \n\u2502 _target_: lightning.pytorch.callbacks.LearningRateMonitor \n\u2502 logging_interval: step \n\u2502 \n\u251c\u2500\u2500 data\n\u2502 \u2514\u2500\u2500 train: openwebtext-train \n\u2502 valid: openwebtext-valid \n\u2502 tokenizer_name_or_path: gpt2 \n\u2502 cache_dir: /home/ruizzhao/diffusion_baseline/datasets/openwebtext \n\u2502 wrap: true \n\u2502 streaming: false \n\u2502 insert_train_eos: true \n\u2502 insert_valid_eos: true \n\u2502 \n\u251c\u2500\u2500 model\n\u2502 \u2514\u2500\u2500 name: small \n\u2502 type: ddit \n\u2502 hidden_size: 768 \n\u2502 cond_dim: 128 \n\u2502 length: 1024 \n\u2502 n_blocks: 12 \n\u2502 n_heads: 12 \n\u2502 scale_by_sigma: true \n\u2502 dropout: 0.1 \n\u2502 tie_word_embeddings: false \n\u2502 vocab_lookup: true \n\u2502 \n\u251c\u2500\u2500 strategy\n\u2502 \u2514\u2500\u2500 _target_: lightning.pytorch.strategies.DDPStrategy \n\u2502 find_unused_parameters: false \n\u2502 \n\u251c\u2500\u2500 noise\n\u2502 \u2514\u2500\u2500 type: log-linear \n\u2502 parameterization: log-linear \n\u2502 eps: 0 \n\u2502 denoiser_latent_conditioning: -1 \n\u2502 freeze_encoder: false \n\u2502 freeze_decoder: false \n\u2502 \n\u251c\u2500\u2500 lr_scheduler\n\u2502 \u2514\u2500\u2500 _target_: transformers.get_constant_schedule_with_warmup \n\u2502 num_warmup_steps: 2500 \n\u2502 \n\u251c\u2500\u2500 prior\n\u2502 \u2514\u2500\u2500 type: none \n\u2502 latent_width: 0 \n\u2502 latent_height: 0 \n\u2502 \n\u2514\u2500\u2500 algo\n \u2514\u2500\u2500 name: fmlm \n backbone: dit \n parameterization: mean \n time_conditioning: true \n T: 0 \n subs_masking: false \n causal_attention: false \n t_max: 1.0 \n t_min: 0.0 \n double_temb: true \n add_boundary: fixed \n diagonal_fraction: 0.5 \n offdiagonal_sampling: uniform_diff \n set_midpoint: midpoint \n boundary_prob: 32 \n distillation_method: PSD \n entmax_temp_lsd: 2.0 \n backprop_entmax_temp_lsd: false \n learnable_loss_weighting: false \n use_mse_loss_psd: false \n rescale_offdiag_loss_psd: false \n use_ema_for_psd_target: false \n use_teacher_for_D_t_lsd: false \n initialize_student_from_teacher: true \n teacher_path: '' \n \n[2026-08-03 12:16:44,609][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.\nResuming training from checkpoint...\ngeneration start: 2026-08-03 12:16:51\nCopying EMA parameters to model\nSampling with 2 steps\n"
}
--- request.json ---
{
"command": [
"/opt/anaconda3/envs/dlb-flm/bin/python",
"-u",
"-m",
"dlb.adapters.capture",
"--upstream-entrypoint=/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py",
"--capture-path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/fmlm/steps_2/upstream_token_ids.json",
"--",
"mode=sample_eval",
"seed=42",
"data=openwebtext-split",
"model=small",
"model.length=1024",
"loader.batch_size=2",
"loader.eval_batch_size=16",
"sampling.num_sample_batches=1",
"sampling.steps=2",
"eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/fmlm_ckpt/owt/owt_fmlm.ckpt",
"trainer.devices=1",
"hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/fmlm/steps_2/upstream_hydra",
"checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/fmlm/steps_2/upstream_hydra",
"+wandb.offline=true",
"eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/fmlm/steps_2/upstream_samples.json",
"algo=fmlm",
"algo.backbone=dit",
"eval.disable_ema=False",
"sampling.solver=euler",
"algo.double_temb=True",
"algo.learnable_loss_weighting=False",
"sampling.gamma=1.0"
],
"identity": {
"adapter_identity": "dlb.adapters.flm:v2|dlb.adapters.flm:FLMAdapter|c3a6a4573200fecf38a47fc6e7bc78254ca362fff3b96079dc2efa9f8e27aff0",
"checkpoint_lock_id": "recipe:fmlm_owt_official_ckpt:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0",
"checkpoint_selection": {
"output": "checkpoints/official/fmlm_ckpt/owt",
"recipe": "fmlm_owt_official_ckpt",
"sampling_checkpoint": "owt_fmlm.ckpt",
"sampling_config": null,
"source": "flm",
"source_commit": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
"teacher_adapter": null,
"teacher_family": "continuous_flm"
},
"checkpoint_sha256": "a5ca77b77a26dec9746f627858d42c19803d11c1e932dcf14a3ce80f6bbbbd76",
"checkpoint_teacher_family": "continuous_flm",
"command_sha256": "c7010db873c1b94fd979778a0881b3186716d03d163eb6966f010b1557b540f7",
"config_sha256": "4f12f3e823ddd03f61014b1a9c7acf32b78abfd7d5a1dfe98101d95ced6f6f84",
"dataset_id": "owt",
"device": null,
"environment": "dlb-flm",
"model_id": "fmlm",
"results_root": "/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck",
"run_id": "fmlm-owt-steps-2",
"sample_count": 1,
"seed": 42,
"source_sha256": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
"step_count": 2
},
"request": {
"adapter_identity": "dlb.adapters.flm:v2|dlb.adapters.flm:FLMAdapter|c3a6a4573200fecf38a47fc6e7bc78254ca362fff3b96079dc2efa9f8e27aff0",
"checkpoint_lock_id": "recipe:fmlm_owt_official_ckpt:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0",
"checkpoint_selection": {
"output": "checkpoints/official/fmlm_ckpt/owt",
"recipe": "fmlm_owt_official_ckpt",
"sampling_checkpoint": "owt_fmlm.ckpt",
"sampling_config": null,
"source": "flm",
"source_commit": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
"teacher_adapter": null,
"teacher_family": "continuous_flm"
},
"checkpoint_sha256": "a5ca77b77a26dec9746f627858d42c19803d11c1e932dcf14a3ce80f6bbbbd76",
"checkpoint_teacher_family": "continuous_flm",
"command": null,
"config_sha256": "4f12f3e823ddd03f61014b1a9c7acf32b78abfd7d5a1dfe98101d95ced6f6f84",
"dataset_id": "owt",
"device": null,
"environment": "dlb-flm",
"model_id": "fmlm",
"results_root": "/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck",
"run_id": "fmlm-owt-steps-2",
"sample_count": 1,
"seed": 42,
"source_sha256": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3",
"step_count": 2
},
"started_at": "2026-08-03T04:16:40.046665Z"
}
--- stderr.log tail 240 ---
Seed set to 42
/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/lightning/pytorch/utilities/migration/utils.py:56: The loaded checkpoint was produced with Lightning v2.6.1, which is newer than your current Lightning version: v2.6.0
/home/ruizzhao/diffusion_baseline/upstreams/flm/models/dit.py:476: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
with torch.cuda.amp.autocast(enabled=False):
Error executing job with overrides: ['mode=sample_eval', 'seed=42', 'data=openwebtext-split', 'model=small', 'model.length=1024', 'loader.batch_size=2', 'loader.eval_batch_size=16', 'sampling.num_sample_batches=1', 'sampling.steps=2', 'eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/fmlm_ckpt/owt/owt_fmlm.ckpt', 'trainer.devices=1', 'checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/fmlm/steps_2/upstream_hydra', '+wandb.offline=true', 'eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/fmlm/steps_2/upstream_samples.json', 'algo=fmlm', 'algo.backbone=dit', 'eval.disable_ema=False', 'sampling.solver=euler', 'algo.double_temb=True', 'algo.learnable_loss_weighting=False', 'sampling.gamma=1.0']
Traceback (most recent call last):
File "<frozen runpy>", line 198, in _run_module_as_main
File "<frozen runpy>", line 88, in _run_code
File "/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py", line 688, in <module>
raise SystemExit(main())
^^^^^^
File "/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py", line 683, in main
_capture_teacher(module, invocation, forwarded)
File "/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py", line 380, in _capture_teacher
_run_main(module, invocation.entrypoint, forwarded)
File "/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py", line 305, in _run_main
module.main()
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/main.py", line 94, in decorated_main
_run_hydra(
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py", line 394, in _run_hydra
_run_app(
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py", line 457, in _run_app
run_and_report(
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py", line 223, in run_and_report
raise ex
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py", line 220, in run_and_report
return func()
^^^^^^
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py", line 458, in <lambda>
lambda: hydra.run(
^^^^^^^^^^
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/hydra.py", line 132, in run
_ = ret.return_value
^^^^^^^^^^^^^^^^
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/core/utils.py", line 260, in return_value
raise self._return_value
File "/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/core/utils.py", line 186, in run_job
ret.return_value = task_function(task_cfg)
^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py", line 452, in main
_generate_samples(**kwargs)
File "/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py", line 131, in _generate_samples
samples = model.restore_model_and_sample(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py", line 369, in capture
token_rows = result.detach().cpu().tolist()
^^^^^^^^^^^^^
AttributeError: 'list' object has no attribute 'detach'
--- stdout.log tail 120 ---
│ limit_train_batches: 1.0
│ limit_val_batches: 1.0
│ val_check_interval: 5000
│
├── wandb
│ └── project: flm
│ notes: FLM
│ group: null
│ job_type: null
│ name: null
│ id: None_42
│ tags:
│ - openwebtext-train
│ - fmlm
│ offline: true
│
├── checkpointing
│ └── save_dir: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/
│ resume_from_ckpt: false
│ resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_e
│
├── callbacks
│ └── checkpoint_every_n_steps:
│ _target_: lightning.pytorch.callbacks.ModelCheckpoint
│ save_top_k: -1
│ save_last: false
│ dirpath: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck
│ verbose: true
│ auto_insert_metric_name: false
│ every_n_train_steps: 25000
│ checkpoint_monitor:
│ _target_: lightning.pytorch.callbacks.ModelCheckpoint
│ monitor: val/nll
│ mode: min
│ save_top_k: 1
│ save_last: true
│ dirpath: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck
│ filename: best_nll
│ auto_insert_metric_name: false
│ verbose: true
│ learning_rate_monitor:
│ _target_: lightning.pytorch.callbacks.LearningRateMonitor
│ logging_interval: step
│
├── data
│ └── train: openwebtext-train
│ valid: openwebtext-valid
│ tokenizer_name_or_path: gpt2
│ cache_dir: /home/ruizzhao/diffusion_baseline/datasets/openwebtext
│ wrap: true
│ streaming: false
│ insert_train_eos: true
│ insert_valid_eos: true
│
├── model
│ └── name: small
│ type: ddit
│ hidden_size: 768
│ cond_dim: 128
│ length: 1024
│ n_blocks: 12
│ n_heads: 12
│ scale_by_sigma: true
│ dropout: 0.1
│ tie_word_embeddings: false
│ vocab_lookup: true
│
├── strategy
│ └── _target_: lightning.pytorch.strategies.DDPStrategy
│ find_unused_parameters: false
│
├── noise
│ └── type: log-linear
│ parameterization: log-linear
│ eps: 0
│ denoiser_latent_conditioning: -1
│ freeze_encoder: false
│ freeze_decoder: false
│
├── lr_scheduler
│ └── _target_: transformers.get_constant_schedule_with_warmup
│ num_warmup_steps: 2500
│
├── prior
│ └── type: none
│ latent_width: 0
│ latent_height: 0
│
└── algo
└── name: fmlm
backbone: dit
parameterization: mean
time_conditioning: true
T: 0
subs_masking: false
causal_attention: false
t_max: 1.0
t_min: 0.0
double_temb: true
add_boundary: fixed
diagonal_fraction: 0.5
offdiagonal_sampling: uniform_diff
set_midpoint: midpoint
boundary_prob: 32
distillation_method: PSD
entmax_temp_lsd: 2.0
backprop_entmax_temp_lsd: false
learnable_loss_weighting: false
use_mse_loss_psd: false
rescale_offdiag_loss_psd: false
use_ema_for_psd_target: false
use_teacher_for_D_t_lsd: false
initialize_student_from_teacher: true
teacher_path: ''
[2026-08-03 12:16:44,609][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.
Resuming training from checkpoint...
generation start: 2026-08-03 12:16:51
Copying EMA parameters to model
Sampling with 2 steps
--- hydra files ---
/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/fmlm/steps_2/upstream_hydra/config_tree.txt
/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/fmlm/steps_2/upstream_hydra/dlb_pinned_upstream_main.log
/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/fmlm/steps_2/upstream_hydra/.hydra/config.yaml
/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/fmlm/steps_2/upstream_hydra/.hydra/hydra.yaml
/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/owt/fmlm/steps_2/upstream_hydra/.hydra/overrides.yaml
===== basic state =====
Mon Aug 3 12:20:55 PM HKT 2026
/home/ruizzhao/diffusion_baseline
1013d41
===== server code check =====
25: "algo.backbone=dit",
configs/experiments.yaml:11: lm1b: {status: supported, provenance: official, train_recipe: flm_lm1b_official_ckpt}
artifacts/checkpoints.yaml:206: flm_lm1b_official_ckpt:
configs/experiments.yaml:63: lm1b: {status: supported, provenance: official, train_recipe: fmlm_lm1b_official_ckpt}
artifacts/checkpoints.yaml:236: fmlm_lm1b_official_ckpt:
========================================
RUN=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2
========================================
--- files ---
total 52K
drwxr-xr-x 3 ruizzhao ds 4.0K Aug 3 12:16 .
drwxr-xr-x 3 ruizzhao ds 4.0K Aug 2 17:11 ..
-rw------- 1 ruizzhao ds 14K Aug 3 12:16 failure.json
-rw------- 1 ruizzhao ds 3.9K Aug 3 12:15 request.json
-rw------- 1 ruizzhao ds 3.8K Aug 3 12:16 stderr.log
-rw------- 1 ruizzhao ds 14K Aug 3 12:16 stdout.log
drwxr-xr-x 3 ruizzhao ds 4.0K Aug 2 17:11 upstream_hydra
--- failure.json ---
{
"command": [
"/opt/anaconda3/envs/dlb-flm/bin/python",
"-u",
"-m",
"dlb.adapters.capture",
"--upstream-entrypoint=/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py",
"--capture-path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_token_ids.json",
"--",
"mode=sample_eval",
"seed=42",
"data=lm1b-wrap",
"model=small",
"model.length=128",
"loader.batch_size=2",
"loader.eval_batch_size=32",
"sampling.num_sample_batches=1",
"sampling.steps=2",
"eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/flm_ckpt/lm1b/lm1b_flm.ckpt",
"trainer.devices=1",
"hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_hydra",
"checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_hydra",
"+wandb.offline=true",
"eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_samples.json",
"algo=flm",
"algo.backbone=dit",
"eval.disable_ema=False",
"sampling.solver=euler",
"algo.double_temb=False",
"sampling.gamma=0.0"
],
"exit_code": 1,
"finished_at": "2026-08-03T04:16:03.918970Z",
"message": "child command failed",
"run_id": "flm-lm1b-steps-2",
"signal": null,
"stage": "command",
"started_at": "2026-08-03T04:15:51.581221Z",
"stderr_log": "/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/stderr.log",
"stderr_tail": "Seed set to 42\n/home/ruizzhao/diffusion_baseline/upstreams/flm/models/dit.py:476: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n with torch.cuda.amp.autocast(enabled=False):\n/home/ruizzhao/diffusion_baseline/upstreams/flm/models/dit.py:476: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n with torch.cuda.amp.autocast(enabled=False):\nError executing job with overrides: ['mode=sample_eval', 'seed=42', 'data=lm1b-wrap', 'model=small', 'model.length=128', 'loader.batch_size=2', 'loader.eval_batch_size=32', 'sampling.num_sample_batches=1', 'sampling.steps=2', 'eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/flm_ckpt/lm1b/lm1b_flm.ckpt', 'trainer.devices=1', 'checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_hydra', '+wandb.offline=true', 'eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_samples.json', 'algo=flm', 'algo.backbone=dit', 'eval.disable_ema=False', 'sampling.solver=euler', 'algo.double_temb=False', 'sampling.gamma=0.0']\nTraceback (most recent call last):\n File \"<frozen runpy>\", line 198, in _run_module_as_main\n File \"<frozen runpy>\", line 88, in _run_code\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 688, in <module>\n raise SystemExit(main())\n ^^^^^^\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 683, in main\n _capture_teacher(module, invocation, forwarded)\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 380, in _capture_teacher\n _run_main(module, invocation.entrypoint, forwarded)\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 305, in _run_main\n module.main()\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/main.py\", line 94, in decorated_main\n _run_hydra(\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 394, in _run_hydra\n _run_app(\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 457, in _run_app\n run_and_report(\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 223, in run_and_report\n raise ex\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 220, in run_and_report\n return func()\n ^^^^^^\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/utils.py\", line 458, in <lambda>\n lambda: hydra.run(\n ^^^^^^^^^^\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/_internal/hydra.py\", line 132, in run\n _ = ret.return_value\n ^^^^^^^^^^^^^^^^\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/core/utils.py\", line 260, in return_value\n raise self._return_value\n File \"/opt/anaconda3/envs/dlb-flm/lib/python3.11/site-packages/hydra/core/utils.py\", line 186, in run_job\n ret.return_value = task_function(task_cfg)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py\", line 452, in main\n _generate_samples(**kwargs)\n File \"/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py\", line 131, in _generate_samples\n samples = model.restore_model_and_sample(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/ruizzhao/diffusion_baseline/src/dlb/adapters/capture.py\", line 369, in capture\n token_rows = result.detach().cpu().tolist()\n ^^^^^^^^^^^^^\nAttributeError: 'list' object has no attribute 'detach'\n",
"stdout_log": "/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/stdout.log",
"stdout_tail": "_steps: 1000000 \n\u2502 log_every_n_steps: 100 \n\u2502 limit_train_batches: 1.0 \n\u2502 limit_val_batches: 1.0 \n\u2502 val_check_interval: 5000 \n\u2502 \n\u251c\u2500\u2500 wandb\n\u2502 \u2514\u2500\u2500 project: flm \n\u2502 notes: FLM \n\u2502 group: null \n\u2502 job_type: null \n\u2502 name: null \n\u2502 id: None_42 \n\u2502 tags: \n\u2502 - lm1b \n\u2502 - flm \n\u2502 offline: true \n\u2502 \n\u251c\u2500\u2500 checkpointing\n\u2502 \u2514\u2500\u2500 save_dir: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/\n\u2502 resume_from_ckpt: false \n\u2502 resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_e\n\u2502 \n\u251c\u2500\u2500 callbacks\n\u2502 \u2514\u2500\u2500 checkpoint_every_n_steps: \n\u2502 _target_: lightning.pytorch.callbacks.ModelCheckpoint \n\u2502 save_top_k: -1 \n\u2502 save_last: false \n\u2502 dirpath: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck\n\u2502 verbose: true \n\u2502 auto_insert_metric_name: false \n\u2502 every_n_train_steps: 25000 \n\u2502 checkpoint_monitor: \n\u2502 _target_: lightning.pytorch.callbacks.ModelCheckpoint \n\u2502 monitor: val/nll \n\u2502 mode: min \n\u2502 save_top_k: 1 \n\u2502 save_last: true \n\u2502 dirpath: /home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck\n\u2502 filename: best_nll \n\u2502 auto_insert_metric_name: false \n\u2502 verbose: true \n\u2502 learning_rate_monitor: \n\u2502 _target_: lightning.pytorch.callbacks.LearningRateMonitor \n\u2502 logging_interval: step \n\u2502 \n\u251c\u2500\u2500 data\n\u2502 \u2514\u2500\u2500 train: lm1b \n\u2502 valid: lm1b \n\u2502 tokenizer_name_or_path: bert-base-uncased \n\u2502 cache_dir: ./cache \n\u2502 wrap: true \n\u2502 streaming: false \n\u2502 insert_train_eos: true \n\u2502 insert_valid_eos: true \n\u2502 \n\u251c\u2500\u2500 model\n\u2502 \u2514\u2500\u2500 name: small \n\u2502 type: ddit \n\u2502 hidden_size: 768 \n\u2502 cond_dim: 128 \n\u2502 length: 128 \n\u2502 n_blocks: 12 \n\u2502 n_heads: 12 \n\u2502 scale_by_sigma: true \n\u2502 dropout: 0.1 \n\u2502 tie_word_embeddings: false \n\u2502 vocab_lookup: true \n\u2502 \n\u251c\u2500\u2500 strategy\n\u2502 \u2514\u2500\u2500 _target_: lightning.pytorch.strategies.DDPStrategy \n\u2502 find_unused_parameters: false \n\u2502 \n\u251c\u2500\u2500 noise\n\u2502 \u2514\u2500\u2500 type: log-linear \n\u2502 parameterization: log-linear \n\u2502 eps: 0 \n\u2502 denoiser_latent_conditioning: -1 \n\u2502 freeze_encoder: false \n\u2502 freeze_decoder: false \n\u2502 \n\u251c\u2500\u2500 lr_scheduler\n\u2502 \u2514\u2500\u2500 _target_: transformers.get_constant_schedule_with_warmup \n\u2502 num_warmup_steps: 2500 \n\u2502 \n\u251c\u2500\u2500 prior\n\u2502 \u2514\u2500\u2500 type: none \n\u2502 latent_width: 0 \n\u2502 latent_height: 0 \n\u2502 \n\u2514\u2500\u2500 algo\n \u2514\u2500\u2500 name: flm \n backbone: dit \n parameterization: mean \n time_conditioning: true \n T: 0 \n subs_masking: false \n causal_attention: false \n ignore_bos: false \n t_max: 1.0 \n t_min: 0.0 \n double_temb: false \n learnable_loss_weighting: false \n \n[2026-08-03 12:15:56,039][dlb_pinned_upstream_main][INFO] - Starting Sample Eval.\nResuming training from checkpoint...\ngeneration start: 2026-08-03 12:16:02\nCopying EMA parameters to model\n"
}
--- request.json ---
{
"command": [
"/opt/anaconda3/envs/dlb-flm/bin/python",
"-u",
"-m",
"dlb.adapters.capture",
"--upstream-entrypoint=/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py",
"--capture-path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_token_ids.json",
"--",
"mode=sample_eval",
"seed=42",
"data=lm1b-wrap",
"model=small",
"model.length=128",
"loader.batch_size=2",
"loader.eval_batch_size=32",
"sampling.num_sample_batches=1",
"sampling.steps=2",
"eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/flm_ckpt/lm1b/lm1b_flm.ckpt",
"trainer.devices=1",
"hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_hydra",
"checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_hydra",
"+wandb.offline=true",
"eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke_gpu6_envcheck/samples/lm1b/flm/steps_2/upstream_samples.json",
"algo=flm",
"algo.backbone=dit",
"eval.disable_ema=False",
"sampling.solver=euler",
"algo.double_temb=False",
"sampling.gamma=0.0"