| { | |
| "config": { | |
| "run_name": "dec_v3", | |
| "data": "data/v4", | |
| "cache": "data/v4/dec_v14", | |
| "seed": 0, | |
| "model": { | |
| "d": 384, | |
| "layers": 6, | |
| "heads": 6, | |
| "ff": 1536, | |
| "K": 8, | |
| "dropout": 0.05 | |
| }, | |
| "train": { | |
| "steps": 105000, | |
| "batch": 512, | |
| "lr": 1.5e-05, | |
| "wd": 0.01, | |
| "warmup": 2000, | |
| "min_lr_frac": 0.05, | |
| "schedule": "wsd", | |
| "decay_frac": 0.0, | |
| "label_smoothing": 0.02, | |
| "grad_clip": 1.0, | |
| "p_noise": 0.4, | |
| "max_angle_cos": 0.99, | |
| "logn_jitter": 0.02, | |
| "p_dz": 0.3, | |
| "rewarm": 500, | |
| "log_every": 100, | |
| "eval_every": 5000, | |
| "eval_n": 1024, | |
| "eval_ce_n": 4096, | |
| "len_buckets": 16, | |
| "max_tokens": 32768 | |
| }, | |
| "encoder": "runs/v14/best.pt" | |
| }, | |
| "step": 105000, | |
| "d_lat": 256, | |
| "best": 0.8779296875, | |
| "component": "decoder", | |
| "description": "latent -> prefix tokens, on the frozen encoder_pi", | |
| "source_checkpoint": "runs/dec_v3/best.pt", | |
| "domain": [ | |
| -3.141592653589793, | |
| 3.141592653589793 | |
| ], | |
| "format": "func-jepa-1" | |
| } |