| { | |
| "step": 38000, | |
| "config": { | |
| "run_name": "e05", | |
| "data": "data/v05", | |
| "seed": 0, | |
| "model": { | |
| "hidden": 512, | |
| "layers": 10, | |
| "heads": 8, | |
| "intermediate": 2176, | |
| "max_pos": 130, | |
| "dropout": 0.0, | |
| "latent_dim": 256, | |
| "pooling": "cls", | |
| "A_init": "identity", | |
| "A_inv": false, | |
| "deriv": "factored", | |
| "deriv_hidden": 1024, | |
| "gain_hidden": 256, | |
| "proj_init": 1.0, | |
| "init": "scaled_normal", | |
| "init_std": 0.02, | |
| "vocab_size": null, | |
| "magnitude": "computed" | |
| }, | |
| "train": { | |
| "batch_size": 384, | |
| "steps": 40000, | |
| "lr": 0.0003, | |
| "A_lr": 0.001, | |
| "weight_decay": 0.01, | |
| "warmup": 700, | |
| "min_lr_frac": 0.1, | |
| "grad_clip": 1.0, | |
| "bf16": true, | |
| "mask_p": 0.15, | |
| "log_every": 50, | |
| "eval_every": 1000, | |
| "ckpt_every": 1000, | |
| "batch_size_after_mlm": 384, | |
| "grad_ckpt": true, | |
| "len_buckets": 4, | |
| "p_group": 0.15, | |
| "p_sweep": 0.25, | |
| "p_combo": 0.15, | |
| "p_single": 0.5, | |
| "combo_coefs": [ | |
| 1, | |
| -1, | |
| 2, | |
| -2, | |
| 3, | |
| -3, | |
| 5, | |
| -5, | |
| 10, | |
| -10, | |
| "1/2", | |
| "-1/2" | |
| ], | |
| "combo_min_rel": 0.1 | |
| }, | |
| "loss": { | |
| "w_mlm": 0.0, | |
| "w_jepa": 1.0, | |
| "w_gram": 1.0, | |
| "gram_mode": "cosine", | |
| "jepa_start": 1000, | |
| "jepa_ramp": 1000, | |
| "jepa_enc_grad": 1.0, | |
| "norm_eps": 0.05, | |
| "jepa_stopgrad": true, | |
| "combine": "sum", | |
| "jepa_max_ratio": 0.5, | |
| "lam_K": 0.1, | |
| "guard": false, | |
| "w_ijepa": 0.0, | |
| "w_inv_reg": 0.0, | |
| "mlm_hold": 0, | |
| "mlm_anneal": 0, | |
| "w_lin": 1.0, | |
| "lin_ramp": 1000, | |
| "lin_max_ratio": 1.0, | |
| "w_jepa_d2": 0.0, | |
| "w_jepa_pow": 0.0, | |
| "gram_inner": "H1", | |
| "sobolev_lambda": 0.5, | |
| "lambda_ramp": 0, | |
| "magnitude_ramp": 0, | |
| "w_vnorm": 0.05, | |
| "vnorm_ramp": 0, | |
| "fprime_nograd": false, | |
| "w_cent": 1.5, | |
| "w_var": 1.0, | |
| "var_gamma": 0.5, | |
| "w_lrvar": 0.0, | |
| "lin_start": 1000, | |
| "jepa_norm": "split", | |
| "jepa_gain_w": 1.0 | |
| } | |
| }, | |
| "K": null, | |
| "r": 0.5, | |
| "sobolev_lambda": 0.5, | |
| "mag_t": 1.0, | |
| "component": "encoder_05", | |
| "description": "encoder + derivative operator D on [0, 5] (paper: E_[0,5])", | |
| "source_checkpoint": "runs/e05/best.pt", | |
| "domain": [ | |
| 0.0, | |
| 5.0 | |
| ], | |
| "format": "func-jepa-1" | |
| } |