run_name: t0cap_matched seed: 42 data: dataset: xsum root: data model: arch: ncm d: 384 in_vocab_size: 50304 text_tokenizer: gpt2 context_length: 48 pad_id: 50257 eos_id: 50256 encoder_blocks: 5 decoder_blocks: 3 n_heads: 6 dropout: 0.1 levels: - 4 - 6 - 10 - 14 - 20 - 26 - 32 - 40 - 48 codebook_size: 4096 pool_self_attn: false attn_prior: true prior_sharpness: 4.0 qk_norm: true query_init_std: 0.02 final_level_identity: true ste: per_level m: 3 vq: decay: 0.99 epsilon: 1.0e-05 commitment: 0.25 restart_threshold: 0.0 nested_dropout: prob: 0.5 min_keep: 1 pre_quant_dropout: 0.25 cascade_norm: true train: steps: 30000 micro_batch: 256 grad_accum: 1 lr: 0.0005 lr_floor: 5.0e-05 warmup: 1000 betas: - 0.9 - 0.95 weight_decay: 0.01 grad_clip: 1.0 max_nonfinite_steps: 200 runtime: precision: bf16 compile: false ckpt_every: 1000 keep_ckpts: 2 log_every: 50 eval_every: 5000 eval_examples: 2000 eval_batch_size: 64 wandb: false wandb_project: cruxtok output_root: /workspace/runs/t0match