cruxtok / t0cap_matched /config.yaml
TimothDevA's picture
CruxTok (T1) and NCM baseline tokenizers trained on XSum
16772ec verified
Raw History Blame Contribute Delete
1.2 kB
run_name: t0cap_matched
seed: 42
data:
dataset: xsum
root: data
model:
arch: ncm
d: 384
in_vocab_size: 50304
text_tokenizer: gpt2
context_length: 48
pad_id: 50257
eos_id: 50256
encoder_blocks: 5
decoder_blocks: 3
n_heads: 6
dropout: 0.1
levels:
- 4
- 6
- 10
- 14
- 20
- 26
- 32
- 40
- 48
codebook_size: 4096
pool_self_attn: false
attn_prior: true
prior_sharpness: 4.0
qk_norm: true
query_init_std: 0.02
final_level_identity: true
ste: per_level
m: 3
vq:
decay: 0.99
epsilon: 1.0e-05
commitment: 0.25
restart_threshold: 0.0
nested_dropout:
prob: 0.5
min_keep: 1
pre_quant_dropout: 0.25
cascade_norm: true
train:
steps: 30000
micro_batch: 256
grad_accum: 1
lr: 0.0005
lr_floor: 5.0e-05
warmup: 1000
betas:
- 0.9
- 0.95
weight_decay: 0.01
grad_clip: 1.0
max_nonfinite_steps: 200
runtime:
precision: bf16
compile: false
ckpt_every: 1000
keep_ckpts: 2
log_every: 50
eval_every: 5000
eval_examples: 2000
eval_batch_size: 64
wandb: false
wandb_project: cruxtok
output_root: /workspace/runs/t0match