| { |
| "arch": "cascade-model", |
| "config": { |
| "d_model": 512, |
| "num_layers": 6, |
| "num_heads": 8, |
| "head_dim": 64, |
| "patch_size": 32, |
| "mlp_expansion": 2, |
| "d_ff": 1368, |
| "num_quantiles": 9, |
| "head_mlp_hidden": 0, |
| "ffn_swiglu": false, |
| "embed_skip_mlp": 0, |
| "embed_mask_token": false, |
| "no_out_mlp": false, |
| "context_length": 4096, |
| "horizon": 64, |
| "max_patches": 134, |
| "layer_group_size": 6, |
| "cpm_c_max": 16, |
| "cpm_p_max": 0.4, |
| "residual_mult": 0.75, |
| "use_variate_roles": true, |
| "binary_passthrough": true, |
| "time_window": 64, |
| "attn_sinks": 0, |
| "attn_impl": "sdpa", |
| "use_rope": true, |
| "rope_base": 10000.0, |
| "rope_scale": 1.0, |
| "xpos_scale_base": 512.0, |
| "yarn": false, |
| "yarn_alpha": 0.5, |
| "yarn_beta": 8.0, |
| "attn_temp": 1.0, |
| "rope_partial_k": 0, |
| "rope_learnable": false, |
| "rope_scale_jitter": 0.0, |
| "time_mixer": "attention" |
| }, |
| "quantile_levels": [ |
| 0.1, |
| 0.2, |
| 0.3, |
| 0.4, |
| 0.5, |
| 0.6, |
| 0.7, |
| 0.8, |
| 0.9 |
| ], |
| "arm": "fgl0-s0", |
| "env": { |
| "torch": "2.11.0+cu128", |
| "cuda": "12.8", |
| "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition", |
| "arch": [ |
| "sm_75", |
| "sm_80", |
| "sm_86", |
| "sm_90", |
| "sm_100", |
| "sm_120" |
| ], |
| "batch_size": 64, |
| "context_length": 4096, |
| "token_budget": 1429855920, |
| "channels": 5, |
| "channel_mode": "mixed", |
| "obs_loss_weight": 9.0, |
| "n_targets": 4, |
| "n_future_cov": 0, |
| "mix_p": 0.5, |
| "mixup_p": 0.5, |
| "mixup_alpha": 1.5, |
| "mixup_k_max": 3, |
| "tirex_aug": true, |
| "aug_freq": false, |
| "aug_warp": false, |
| "aug_jitter": false, |
| "cpm_p_max": 0.4, |
| "cpm_c_max": 16, |
| "time_window": 64, |
| "optimizer": "normuon", |
| "base_lr": 0.0002, |
| "normuon_lr": 0.001, |
| "normuon_momentum": 0.96, |
| "normuon_beta2": 0.999, |
| "adam_betas": [ |
| 0.91, |
| 0.972 |
| ], |
| "lr_schedule": "toto2", |
| "warmup_fraction": 0.01, |
| "decay_tail_fraction": 0.0175, |
| "grad_clip": 7.0, |
| "mup_base_width": 512, |
| "toto2_recipe": true, |
| "gen_children": [ |
| "/home/ubuntu/work/cascade-model/generators/fitproc2m" |
| ], |
| "gen_weights": null, |
| "real_root": "data/subnet_ft2", |
| "real_frac": 1.0, |
| "tempopfn_repo": null, |
| "tempopfn_length": 4096, |
| "tempopfn_split_long": false, |
| "tempopfn_shards": null, |
| "shard_unique_points": null, |
| "init_weights": "results/e19-22m400k/t22m400k-s0/weights.safetensors", |
| "shards_real_root": null, |
| "dynsys": null, |
| "dyn_loss": null, |
| "tempopfn_frac": 0.0, |
| "tempopfn_augment": null, |
| "tempopfn_priors": null, |
| "tempopfn_drop_priors": null, |
| "corpus_stats_scope": "synthetic-only" |
| } |
| } |
