Download checkpoints/official/fmlm/owt/config.json from jyliuAI/diffusion_baseline: direct link, hf CLI and curl.
- Browser
- Download file 6.28 kB
-
https://huggingface.co/jyliuAI/diffusion_baseline/resolve/main/checkpoints/official/fmlm/owt/config.json
- Command line
-
hf download hf://jyliuAI/diffusion_baseline/checkpoints/official/fmlm/owt/config.json
-
curl -L -o config.json https://huggingface.co/jyliuAI/diffusion_baseline/resolve/main/checkpoints/official/fmlm/owt/config.json
6.28 kB
| { | |
| "config": { | |
| "algo": { | |
| "T": 0, | |
| "add_boundary": "fixed", | |
| "backbone": "dit", | |
| "backprop_entmax_temp_lsd": false, | |
| "boundary_prob": 32, | |
| "causal_attention": false, | |
| "diagonal_fraction": 0.5, | |
| "distillation_method": "PSD", | |
| "double_temb": true, | |
| "entmax_temp_lsd": 2.0, | |
| "initialize_student_from_teacher": true, | |
| "learnable_loss_weighting": false, | |
| "loss_scale": "none", | |
| "loss_type": null, | |
| "name": "fmlm", | |
| "offdiagonal_sampling": "uniform_diff", | |
| "parameterization": "mean", | |
| "pred_log_interval": 2000, | |
| "rescale_offdiag_loss_psd": false, | |
| "subs_masking": false, | |
| "t_max": 1.0, | |
| "t_min": 0.0, | |
| "teacher_path": "/home/irteam/ours_owt_flow_ce_1m.ckpt", | |
| "time_condition": "alpha_t", | |
| "time_conditioning": true, | |
| "use_discrete_schedule": true, | |
| "use_ema_for_psd_target": false, | |
| "use_mse_loss_psd": false, | |
| "use_simulated_entropy": false, | |
| "use_teacher_for_D_t_lsd": false | |
| }, | |
| "callbacks": { | |
| "checkpoint_every_n_steps": { | |
| "_target_": "lightning.pytorch.callbacks.ModelCheckpoint", | |
| "auto_insert_metric_name": false, | |
| "dirpath": "/home/david3684/projects/flm/hf_upload/checkpoints", | |
| "every_n_train_steps": 5000, | |
| "save_last": false, | |
| "save_top_k": -1, | |
| "verbose": true | |
| }, | |
| "checkpoint_monitor": { | |
| "_target_": "lightning.pytorch.callbacks.ModelCheckpoint", | |
| "auto_insert_metric_name": false, | |
| "dirpath": "/home/david3684/projects/flm/hf_upload/checkpoints", | |
| "filename": "best_nll", | |
| "mode": "min", | |
| "monitor": "val/nll", | |
| "save_last": true, | |
| "save_top_k": 1, | |
| "verbose": true | |
| }, | |
| "learning_rate_monitor": { | |
| "_target_": "lightning.pytorch.callbacks.LearningRateMonitor", | |
| "logging_interval": "step" | |
| } | |
| }, | |
| "checkpointing": { | |
| "resume_ckpt_path": "/home/david3684/projects/flm/hf_upload/checkpoints/last.ckpt", | |
| "resume_from_ckpt": false, | |
| "save_dir": "/home/david3684/projects/flm/hf_upload" | |
| }, | |
| "data": { | |
| "cache_dir": "/home/irteam/workspace/datasets/openwebtext", | |
| "insert_train_eos": true, | |
| "insert_valid_eos": true, | |
| "streaming": false, | |
| "tokenizer_name_or_path": "gpt2", | |
| "train": "openwebtext-train", | |
| "valid": "openwebtext-valid", | |
| "wrap": true | |
| }, | |
| "eval": { | |
| "checkpoint_path": "", | |
| "compute_generative_perplexity": true, | |
| "compute_perplexity_on_sanity": false, | |
| "disable_ema": false, | |
| "ema_decay": null, | |
| "gen_ppl_eval_model_name_or_path": "gpt2-large", | |
| "generate_samples": true, | |
| "generated_samples_path": "/home/david3684/projects/flm/hf_upload/samples.json", | |
| "perplexity_batch_size": 8 | |
| }, | |
| "is_di4c": false, | |
| "is_di4c_deterministic": true, | |
| "loader": { | |
| "batch_size": 16, | |
| "eval_batch_size": 16, | |
| "eval_global_batch_size": 128, | |
| "global_batch_size": 128, | |
| "num_workers": 4, | |
| "pin_memory": true | |
| }, | |
| "lr_scheduler": { | |
| "_target_": "transformers.get_constant_schedule_with_warmup", | |
| "num_warmup_steps": 2500 | |
| }, | |
| "mode": "train", | |
| "model": { | |
| "cond_dim": 128, | |
| "dropout": 0.1, | |
| "hidden_size": 768, | |
| "l2_normalize_embeddings": false, | |
| "length": 1024, | |
| "n_blocks": 12, | |
| "n_heads": 12, | |
| "name": "small", | |
| "scale_by_sigma": true, | |
| "tie_word_embeddings": false, | |
| "type": "ddit", | |
| "vocab_lookup": true | |
| }, | |
| "noise": { | |
| "denoiser_latent_conditioning": -1, | |
| "eps": 0, | |
| "freeze_decoder": false, | |
| "freeze_encoder": false, | |
| "parameterization": "log-linear", | |
| "type": "log-linear" | |
| }, | |
| "optim": { | |
| "beta1": 0.9, | |
| "beta2": 0.95, | |
| "eps": 1e-08, | |
| "ln_tune": "none", | |
| "lr": 0.0003, | |
| "muon_lr": 0.03, | |
| "muon_momentum": 0.95, | |
| "muon_weight_decay": 0.01, | |
| "use_muon": false, | |
| "weight_decay": 0 | |
| }, | |
| "prior": { | |
| "latent_height": 0, | |
| "latent_width": 0, | |
| "type": "none" | |
| }, | |
| "sampling": { | |
| "argmax_correction": false, | |
| "autoguidance_weight": 0.0, | |
| "hard_start": false, | |
| "noise_removal": "uniform_alpha", | |
| "num_reflow_samples": 50000, | |
| "num_sample_batches": 1, | |
| "num_sample_log": 2, | |
| "num_strides": 1, | |
| "p_nucleus": 1.0, | |
| "predictor": "ancestral", | |
| "semi_ar": false, | |
| "solver": "euler", | |
| "steps": [ | |
| 1, | |
| 2, | |
| 4, | |
| 8, | |
| 16, | |
| 32, | |
| 64, | |
| 128 | |
| ], | |
| "stride_length": 1, | |
| "tau_log10": -1.0, | |
| "temperature": 1.0, | |
| "use_float64": true | |
| }, | |
| "seed": 1, | |
| "strategy": { | |
| "_target_": "lightning.pytorch.strategies.DDPStrategy", | |
| "find_unused_parameters": false | |
| }, | |
| "trainer": { | |
| "_target_": "lightning.Trainer", | |
| "accelerator": "cuda", | |
| "accumulate_grad_batches": 1, | |
| "devices": 8, | |
| "gradient_clip_val": 1.0, | |
| "limit_train_batches": 1.0, | |
| "limit_val_batches": 10, | |
| "log_every_n_steps": 100, | |
| "max_steps": 1000000, | |
| "num_nodes": 1, | |
| "num_sanity_val_steps": 2, | |
| "precision": "bf16", | |
| "val_check_interval": 10000 | |
| }, | |
| "training": { | |
| "antithetic_sampling": true, | |
| "change_of_variables": false, | |
| "double_ema": 0.0, | |
| "ema": "0.9999, 0.999, 0.99, 0.95", | |
| "finetune_path": "", | |
| "importance_sampling": false, | |
| "load_ema": false, | |
| "loss_precision": "bf16", | |
| "loss_type": "sd", | |
| "loss_weighting": false, | |
| "not_sampling_t": false, | |
| "pred_type": "x0", | |
| "sampling_eps": 0.001, | |
| "t_curriculum_max": 1.0, | |
| "t_curriculum_min": 0.0, | |
| "t_curriculum_steps": 100000, | |
| "torch_compile_mode": "default", | |
| "use_torch_compile": true | |
| }, | |
| "wandb": { | |
| "group": null, | |
| "id": "9efb5cfb", | |
| "job_type": null, | |
| "name": "owt_PSD_teacher_distill_LR_3e-4_DF_0.5_BP_32_SE_False_IT_True_EDMW_False_EMA_False_beta2_0.95_9efb5cfb", | |
| "notes": "DOS", | |
| "project": "owt_full", | |
| "tags": [ | |
| "openwebtext-train", | |
| "dos_denoiser" | |
| ] | |
| } | |
| }, | |
| "vocab_size": 50258 | |
| } |