| model: {architecture: ali, d_enc: 256, d_main: 512, n_enc: 2, n_dec: 2, n_main: 6, |
| n_heads_enc: 4, n_heads_main: 8, seq_len: 1024, dropout: 0.0, target_ratio: 6.0, |
| max_chunk_mult: 3.0, dechunk: ema, router_noise: 0.0, rope_base: 10000.0} |
| memory: |
| ali: |
| budget_B: 65536 |
| staging_capacity: 262144 |
| policy: frequency |
| value_mode: learned |
| candidate_types: [split] |
| write_rule: mean |
| eta_min: 0.0 |
| memory_scope: persistent |
| consolidate_every: 50 |
| max_span: 64 |
| gate: vector |
| retrieval: true |
| read_dropout: 0.1 |
| gate_hidden: 0 |
| type_dim: 16 |
| gate_bias_init: -2.0 |
| probe_seqs: 2 |
| probe_reps: 8 |
| horizon: 32 |
| predictor: mlp |
| lambda_util: 0.1 |
| enabled: true |
| inventory_kwargs: {} |
| probe_mix: null |
| util_batch_slots: 1024 |
| pkm: {n_keys: 256, topk: 8, heads: 1, d_key: null, value_lr_mult: 1.0} |
| objective: {name: ratio, schedule: null, alpha: 0.03, alpha_parity_cap: 0.06, global_anchor: 0.0, |
| premium_clip: 4.0, premium_kind: flores_per_sent, tau1: 0.4, tau2: 0.7, freeze_scope: router, |
| cpc_decay: 0.9, genreg_target_entropy: null, genreg_eta: 0.05} |
| data: |
| corpus: corpus |
| langs: [en, tr, fi] |
| mixture: null |
| doc_filters: null |
| byte_classes: scratch/data/byte_classes.json |
| training: |
| seed: 1 |
| max_steps: 6000 |
| batch_size: 32 |
| grad_accum: 1 |
| lr: 0.0003 |
| min_lr_frac: 0.1 |
| warmup: 300 |
| weight_decay: 0.1 |
| grad_clip: 1.0 |
| betas: [0.9, 0.95] |
| router_lr_mult: 1.0 |
| log_every: 50 |
| amp: bf16 |
| compile: false |
| device: cuda |
| checkpointing: |
| run_name: ali_mem-split_s1 |
| runs_dir: runs |
| ckpt_every: 200 |
| keep: false |
| milestones: [1000, 3000] |
| resume: auto |
| keep_schedule: log2 |
| keep_extra_offsets: [2, 4, 6, 8] |
| hub_repo: AdaptiveChunking/{run_name} |
| hub_private: false |
| keep_local: false |
| evaluation: |
| eval_every: 200 |
| eval_batches: 8 |
| probe_sents: 128 |
| probe_bpb_batches: 4 |
| log_spaced_probe: true |
| dense_early_every: 0 |
| dense_early_until: 0 |
| dump_reps: true |
| dump_reps_thresholds: [0.3, 0.4, 0.5, 0.6, 0.7] |
|
|
