| model: |
| architecture: ali |
| d_enc: 256 |
| d_main: 512 |
| n_enc: 2 |
| n_dec: 2 |
| n_main: 6 |
| n_heads_enc: 4 |
| n_heads_main: 8 |
| seq_len: 1024 |
| dropout: 0.0 |
| target_ratio: 6.0 |
| max_chunk_mult: 3.0 |
| dechunk: ema |
| router_noise: 0.0 |
| rope_base: 10000.0 |
| memory: |
| ali: |
| budget_B: 65536 |
| staging_capacity: 262144 |
| policy: frequency |
| value_mode: static |
| candidate_types: |
| - surface |
| write_rule: mean |
| eta_min: 0.0 |
| memory_scope: persistent |
| consolidate_every: 50 |
| max_span: 64 |
| gate: vector |
| retrieval: true |
| read_dropout: 0.1 |
| gate_hidden: 0 |
| type_dim: 16 |
| gate_bias_init: -2.0 |
| probe_seqs: 2 |
| probe_reps: 8 |
| horizon: 32 |
| predictor: mlp |
| lambda_util: 0.1 |
| enabled: true |
| inventory_kwargs: {} |
| probe_mix: null |
| util_batch_slots: 1024 |
| pkm: |
| n_keys: 256 |
| topk: 8 |
| heads: 1 |
| d_key: null |
| value_lr_mult: 1.0 |
| objective: |
| name: ratio |
| schedule: null |
| alpha: 0.03 |
| alpha_parity_cap: 0.06 |
| global_anchor: 0.0 |
| premium_clip: 4.0 |
| premium_kind: flores_per_sent |
| tau1: 0.4 |
| tau2: 0.7 |
| freeze_scope: router |
| cpc_decay: 0.9 |
| genreg_target_entropy: null |
| genreg_eta: 0.05 |
| data: |
| corpus: corpus |
| langs: |
| - en |
| - tr |
| - fi |
| mixture: null |
| doc_filters: null |
| byte_classes: scratch/data/byte_classes.json |
| training: |
| seed: 2 |
| max_steps: 6000 |
| batch_size: 32 |
| grad_accum: 1 |
| lr: 0.0003 |
| min_lr_frac: 0.1 |
| warmup: 300 |
| weight_decay: 0.1 |
| grad_clip: 1.0 |
| betas: |
| - 0.9 |
| - 0.95 |
| router_lr_mult: 1.0 |
| log_every: 50 |
| amp: bf16 |
| compile: false |
| device: cuda |
| checkpointing: |
| run_name: ali_mem-static_s2 |
| runs_dir: runs |
| ckpt_every: 200 |
| keep: false |
| milestones: |
| - 1000 |
| - 3000 |
| resume: auto |
| keep_schedule: log2 |
| keep_extra_offsets: |
| - 2 |
| - 4 |
| - 6 |
| - 8 |
| hub_repo: AdaptiveChunking/{run_name} |
| hub_private: false |
| keep_local: false |
| evaluation: |
| eval_every: 200 |
| eval_batches: 8 |
| probe_sents: 128 |
| probe_bpb_batches: 4 |
| log_spaced_probe: true |
| dense_early_every: 0 |
| dense_early_until: 0 |
| dump_reps: true |
| dump_reps_thresholds: |
| - 0.3 |
| - 0.4 |
| - 0.5 |
| - 0.6 |
| - 0.7 |
|
|
