jyliuAI's picture
Add files using upload-large-folder tool
784f572 verified
Raw History Blame Contribute Delete
6.28 kB
{
"config": {
"algo": {
"T": 0,
"add_boundary": "fixed",
"backbone": "dit",
"backprop_entmax_temp_lsd": false,
"boundary_prob": 32,
"causal_attention": false,
"diagonal_fraction": 0.5,
"distillation_method": "PSD",
"double_temb": true,
"entmax_temp_lsd": 2.0,
"initialize_student_from_teacher": true,
"learnable_loss_weighting": false,
"loss_scale": "none",
"loss_type": null,
"name": "fmlm",
"offdiagonal_sampling": "uniform_diff",
"parameterization": "mean",
"pred_log_interval": 2000,
"rescale_offdiag_loss_psd": false,
"subs_masking": false,
"t_max": 1.0,
"t_min": 0.0,
"teacher_path": "/home/irteam/ours_owt_flow_ce_1m.ckpt",
"time_condition": "alpha_t",
"time_conditioning": true,
"use_discrete_schedule": true,
"use_ema_for_psd_target": false,
"use_mse_loss_psd": false,
"use_simulated_entropy": false,
"use_teacher_for_D_t_lsd": false
},
"callbacks": {
"checkpoint_every_n_steps": {
"_target_": "lightning.pytorch.callbacks.ModelCheckpoint",
"auto_insert_metric_name": false,
"dirpath": "/home/david3684/projects/flm/hf_upload/checkpoints",
"every_n_train_steps": 5000,
"save_last": false,
"save_top_k": -1,
"verbose": true
},
"checkpoint_monitor": {
"_target_": "lightning.pytorch.callbacks.ModelCheckpoint",
"auto_insert_metric_name": false,
"dirpath": "/home/david3684/projects/flm/hf_upload/checkpoints",
"filename": "best_nll",
"mode": "min",
"monitor": "val/nll",
"save_last": true,
"save_top_k": 1,
"verbose": true
},
"learning_rate_monitor": {
"_target_": "lightning.pytorch.callbacks.LearningRateMonitor",
"logging_interval": "step"
}
},
"checkpointing": {
"resume_ckpt_path": "/home/david3684/projects/flm/hf_upload/checkpoints/last.ckpt",
"resume_from_ckpt": false,
"save_dir": "/home/david3684/projects/flm/hf_upload"
},
"data": {
"cache_dir": "/home/irteam/workspace/datasets/openwebtext",
"insert_train_eos": true,
"insert_valid_eos": true,
"streaming": false,
"tokenizer_name_or_path": "gpt2",
"train": "openwebtext-train",
"valid": "openwebtext-valid",
"wrap": true
},
"eval": {
"checkpoint_path": "",
"compute_generative_perplexity": true,
"compute_perplexity_on_sanity": false,
"disable_ema": false,
"ema_decay": null,
"gen_ppl_eval_model_name_or_path": "gpt2-large",
"generate_samples": true,
"generated_samples_path": "/home/david3684/projects/flm/hf_upload/samples.json",
"perplexity_batch_size": 8
},
"is_di4c": false,
"is_di4c_deterministic": true,
"loader": {
"batch_size": 16,
"eval_batch_size": 16,
"eval_global_batch_size": 128,
"global_batch_size": 128,
"num_workers": 4,
"pin_memory": true
},
"lr_scheduler": {
"_target_": "transformers.get_constant_schedule_with_warmup",
"num_warmup_steps": 2500
},
"mode": "train",
"model": {
"cond_dim": 128,
"dropout": 0.1,
"hidden_size": 768,
"l2_normalize_embeddings": false,
"length": 1024,
"n_blocks": 12,
"n_heads": 12,
"name": "small",
"scale_by_sigma": true,
"tie_word_embeddings": false,
"type": "ddit",
"vocab_lookup": true
},
"noise": {
"denoiser_latent_conditioning": -1,
"eps": 0,
"freeze_decoder": false,
"freeze_encoder": false,
"parameterization": "log-linear",
"type": "log-linear"
},
"optim": {
"beta1": 0.9,
"beta2": 0.95,
"eps": 1e-08,
"ln_tune": "none",
"lr": 0.0003,
"muon_lr": 0.03,
"muon_momentum": 0.95,
"muon_weight_decay": 0.01,
"use_muon": false,
"weight_decay": 0
},
"prior": {
"latent_height": 0,
"latent_width": 0,
"type": "none"
},
"sampling": {
"argmax_correction": false,
"autoguidance_weight": 0.0,
"hard_start": false,
"noise_removal": "uniform_alpha",
"num_reflow_samples": 50000,
"num_sample_batches": 1,
"num_sample_log": 2,
"num_strides": 1,
"p_nucleus": 1.0,
"predictor": "ancestral",
"semi_ar": false,
"solver": "euler",
"steps": [
1,
2,
4,
8,
16,
32,
64,
128
],
"stride_length": 1,
"tau_log10": -1.0,
"temperature": 1.0,
"use_float64": true
},
"seed": 1,
"strategy": {
"_target_": "lightning.pytorch.strategies.DDPStrategy",
"find_unused_parameters": false
},
"trainer": {
"_target_": "lightning.Trainer",
"accelerator": "cuda",
"accumulate_grad_batches": 1,
"devices": 8,
"gradient_clip_val": 1.0,
"limit_train_batches": 1.0,
"limit_val_batches": 10,
"log_every_n_steps": 100,
"max_steps": 1000000,
"num_nodes": 1,
"num_sanity_val_steps": 2,
"precision": "bf16",
"val_check_interval": 10000
},
"training": {
"antithetic_sampling": true,
"change_of_variables": false,
"double_ema": 0.0,
"ema": "0.9999, 0.999, 0.99, 0.95",
"finetune_path": "",
"importance_sampling": false,
"load_ema": false,
"loss_precision": "bf16",
"loss_type": "sd",
"loss_weighting": false,
"not_sampling_t": false,
"pred_type": "x0",
"sampling_eps": 0.001,
"t_curriculum_max": 1.0,
"t_curriculum_min": 0.0,
"t_curriculum_steps": 100000,
"torch_compile_mode": "default",
"use_torch_compile": true
},
"wandb": {
"group": null,
"id": "9efb5cfb",
"job_type": null,
"name": "owt_PSD_teacher_distill_LR_3e-4_DF_0.5_BP_32_SE_False_IT_True_EDMW_False_EMA_False_beta2_0.95_9efb5cfb",
"notes": "DOS",
"project": "owt_full",
"tags": [
"openwebtext-train",
"dos_denoiser"
]
}
},
"vocab_size": 50258
}