{ "config": { "algo": { "T": 0, "add_boundary": "fixed", "backbone": "dit", "backprop_entmax_temp_lsd": false, "boundary_prob": 32, "causal_attention": false, "diagonal_fraction": 0.5, "distillation_method": "PSD", "double_temb": true, "entmax_temp_lsd": 2.0, "initialize_student_from_teacher": true, "learnable_loss_weighting": false, "loss_scale": "none", "loss_type": null, "name": "fmlm", "offdiagonal_sampling": "uniform_diff", "parameterization": "mean", "pred_log_interval": 2000, "rescale_offdiag_loss_psd": false, "subs_masking": false, "t_max": 1.0, "t_min": 0.0, "teacher_path": "/home/irteam/ours_owt_flow_ce_1m.ckpt", "time_condition": "alpha_t", "time_conditioning": true, "use_discrete_schedule": true, "use_ema_for_psd_target": false, "use_mse_loss_psd": false, "use_simulated_entropy": false, "use_teacher_for_D_t_lsd": false }, "callbacks": { "checkpoint_every_n_steps": { "_target_": "lightning.pytorch.callbacks.ModelCheckpoint", "auto_insert_metric_name": false, "dirpath": "/home/david3684/projects/flm/hf_upload/checkpoints", "every_n_train_steps": 5000, "save_last": false, "save_top_k": -1, "verbose": true }, "checkpoint_monitor": { "_target_": "lightning.pytorch.callbacks.ModelCheckpoint", "auto_insert_metric_name": false, "dirpath": "/home/david3684/projects/flm/hf_upload/checkpoints", "filename": "best_nll", "mode": "min", "monitor": "val/nll", "save_last": true, "save_top_k": 1, "verbose": true }, "learning_rate_monitor": { "_target_": "lightning.pytorch.callbacks.LearningRateMonitor", "logging_interval": "step" } }, "checkpointing": { "resume_ckpt_path": "/home/david3684/projects/flm/hf_upload/checkpoints/last.ckpt", "resume_from_ckpt": false, "save_dir": "/home/david3684/projects/flm/hf_upload" }, "data": { "cache_dir": "/home/irteam/workspace/datasets/openwebtext", "insert_train_eos": true, "insert_valid_eos": true, "streaming": false, "tokenizer_name_or_path": "gpt2", "train": "openwebtext-train", "valid": "openwebtext-valid", "wrap": true }, "eval": { "checkpoint_path": "", "compute_generative_perplexity": true, "compute_perplexity_on_sanity": false, "disable_ema": false, "ema_decay": null, "gen_ppl_eval_model_name_or_path": "gpt2-large", "generate_samples": true, "generated_samples_path": "/home/david3684/projects/flm/hf_upload/samples.json", "perplexity_batch_size": 8 }, "is_di4c": false, "is_di4c_deterministic": true, "loader": { "batch_size": 16, "eval_batch_size": 16, "eval_global_batch_size": 128, "global_batch_size": 128, "num_workers": 4, "pin_memory": true }, "lr_scheduler": { "_target_": "transformers.get_constant_schedule_with_warmup", "num_warmup_steps": 2500 }, "mode": "train", "model": { "cond_dim": 128, "dropout": 0.1, "hidden_size": 768, "l2_normalize_embeddings": false, "length": 1024, "n_blocks": 12, "n_heads": 12, "name": "small", "scale_by_sigma": true, "tie_word_embeddings": false, "type": "ddit", "vocab_lookup": true }, "noise": { "denoiser_latent_conditioning": -1, "eps": 0, "freeze_decoder": false, "freeze_encoder": false, "parameterization": "log-linear", "type": "log-linear" }, "optim": { "beta1": 0.9, "beta2": 0.95, "eps": 1e-08, "ln_tune": "none", "lr": 0.0003, "muon_lr": 0.03, "muon_momentum": 0.95, "muon_weight_decay": 0.01, "use_muon": false, "weight_decay": 0 }, "prior": { "latent_height": 0, "latent_width": 0, "type": "none" }, "sampling": { "argmax_correction": false, "autoguidance_weight": 0.0, "hard_start": false, "noise_removal": "uniform_alpha", "num_reflow_samples": 50000, "num_sample_batches": 1, "num_sample_log": 2, "num_strides": 1, "p_nucleus": 1.0, "predictor": "ancestral", "semi_ar": false, "solver": "euler", "steps": [ 1, 2, 4, 8, 16, 32, 64, 128 ], "stride_length": 1, "tau_log10": -1.0, "temperature": 1.0, "use_float64": true }, "seed": 1, "strategy": { "_target_": "lightning.pytorch.strategies.DDPStrategy", "find_unused_parameters": false }, "trainer": { "_target_": "lightning.Trainer", "accelerator": "cuda", "accumulate_grad_batches": 1, "devices": 8, "gradient_clip_val": 1.0, "limit_train_batches": 1.0, "limit_val_batches": 10, "log_every_n_steps": 100, "max_steps": 1000000, "num_nodes": 1, "num_sanity_val_steps": 2, "precision": "bf16", "val_check_interval": 10000 }, "training": { "antithetic_sampling": true, "change_of_variables": false, "double_ema": 0.0, "ema": "0.9999, 0.999, 0.99, 0.95", "finetune_path": "", "importance_sampling": false, "load_ema": false, "loss_precision": "bf16", "loss_type": "sd", "loss_weighting": false, "not_sampling_t": false, "pred_type": "x0", "sampling_eps": 0.001, "t_curriculum_max": 1.0, "t_curriculum_min": 0.0, "t_curriculum_steps": 100000, "torch_compile_mode": "default", "use_torch_compile": true }, "wandb": { "group": null, "id": "9efb5cfb", "job_type": null, "name": "owt_PSD_teacher_distill_LR_3e-4_DF_0.5_BP_32_SE_False_IT_True_EDMW_False_EMA_False_beta2_0.95_9efb5cfb", "notes": "DOS", "project": "owt_full", "tags": [ "openwebtext-train", "dos_denoiser" ] } }, "vocab_size": 50258 }