| { |
| "arch": "ali", |
| "step": 6000, |
| "model_config": { |
| "vocab_size": 256, |
| "d_enc": 256, |
| "d_main": 512, |
| "n_enc": 2, |
| "n_dec": 2, |
| "n_main": 6, |
| "n_heads_enc": 4, |
| "n_heads_main": 8, |
| "seq_len": 1024, |
| "dropout": 0.0, |
| "target_ratio": 6.0, |
| "max_chunk_mult": 3.0, |
| "dechunk": "ema", |
| "router_noise": 0.0, |
| "rope_base": 10000.0 |
| }, |
| "ali_config": { |
| "enabled": true, |
| "budget_B": 65536, |
| "staging_capacity": 262144, |
| "policy": "utility", |
| "candidate_types": [ |
| "surface", |
| "split", |
| "merge", |
| "skel" |
| ], |
| "gate": "vector", |
| "retrieval": true, |
| "memory_scope": "persistent", |
| "read_dropout": 0.1, |
| "write_rule": "mean", |
| "eta_min": 0.0, |
| "consolidate_every": 50, |
| "probe_seqs": 2, |
| "probe_reps": 8, |
| "horizon": 32, |
| "predictor": "mlp", |
| "lambda_util": 0.1, |
| "max_span": 64, |
| "byte_classes_path": "/local/scratch/sas245/beetle-hnet/scratch/data/byte_classes.json", |
| "gate_hidden": 0, |
| "type_dim": 16, |
| "gate_bias_init": -2.0 |
| }, |
| "tied_weights": { |
| "head.weight": "emb.weight" |
| }, |
| "config": { |
| "model": { |
| "architecture": "ali", |
| "d_enc": 256, |
| "d_main": 512, |
| "n_enc": 2, |
| "n_dec": 2, |
| "n_main": 6, |
| "n_heads_enc": 4, |
| "n_heads_main": 8, |
| "seq_len": 1024, |
| "dropout": 0.0, |
| "target_ratio": 6.0, |
| "max_chunk_mult": 3.0, |
| "dechunk": "ema", |
| "router_noise": 0.0, |
| "rope_base": 10000.0 |
| }, |
| "memory": { |
| "ali": { |
| "budget_B": 65536, |
| "staging_capacity": 262144, |
| "policy": "utility", |
| "candidate_types": [ |
| "surface", |
| "split", |
| "merge", |
| "skel" |
| ], |
| "write_rule": "mean", |
| "eta_min": 0.0, |
| "memory_scope": "persistent", |
| "consolidate_every": 50, |
| "max_span": 64, |
| "gate": "vector", |
| "retrieval": true, |
| "read_dropout": 0.1, |
| "gate_hidden": 0, |
| "type_dim": 16, |
| "gate_bias_init": -2.0, |
| "probe_seqs": 2, |
| "probe_reps": 8, |
| "horizon": 32, |
| "predictor": "mlp", |
| "lambda_util": 0.1, |
| "enabled": true, |
| "inventory_kwargs": {}, |
| "probe_mix": null, |
| "util_batch_slots": 1024 |
| }, |
| "pkm": { |
| "n_keys": 256, |
| "topk": 8, |
| "heads": 1, |
| "d_key": null, |
| "value_lr_mult": 1.0 |
| } |
| }, |
| "objective": { |
| "name": "ratio", |
| "schedule": null, |
| "alpha": 0.03, |
| "alpha_parity_cap": 0.06, |
| "global_anchor": 0.0, |
| "premium_clip": 4.0, |
| "premium_kind": "flores_per_sent", |
| "tau1": 0.4, |
| "tau2": 0.7, |
| "freeze_scope": "router", |
| "cpc_decay": 0.9, |
| "genreg_target_entropy": null, |
| "genreg_eta": 0.05 |
| }, |
| "data": { |
| "corpus": "corpus", |
| "langs": [ |
| "en", |
| "tr", |
| "fi" |
| ], |
| "mixture": null, |
| "doc_filters": null, |
| "byte_classes": "scratch/data/byte_classes.json" |
| }, |
| "training": { |
| "seed": 2, |
| "max_steps": 6000, |
| "batch_size": 32, |
| "grad_accum": 1, |
| "lr": 0.0003, |
| "min_lr_frac": 0.1, |
| "warmup": 300, |
| "weight_decay": 0.1, |
| "grad_clip": 1.0, |
| "betas": [ |
| 0.9, |
| 0.95 |
| ], |
| "router_lr_mult": 1.0, |
| "log_every": 50, |
| "amp": "bf16", |
| "compile": false, |
| "device": "cuda" |
| }, |
| "checkpointing": { |
| "run_name": "ali_ali_s2", |
| "runs_dir": "runs", |
| "ckpt_every": 200, |
| "keep": false, |
| "milestones": [ |
| 1000, |
| 3000 |
| ], |
| "resume": "auto" |
| }, |
| "evaluation": { |
| "eval_every": 200, |
| "eval_batches": 8, |
| "probe_sents": 128, |
| "probe_bpb_batches": 4, |
| "log_spaced_probe": true, |
| "dense_early_every": 0, |
| "dense_early_until": 0, |
| "dump_reps": true, |
| "dump_reps_thresholds": [ |
| 0.3, |
| 0.4, |
| 0.5, |
| 0.6, |
| 0.7 |
| ] |
| } |
| }, |
| "format": "beetle-hnet step branch v1 (src/checkpointing/hub.py)" |
| } |
