Download t0cap_matched/config.yaml from TimothDevA/cruxtok: direct link, hf CLI and curl.
- Browser
- Download file 1.2 kB
-
https://huggingface.co/TimothDevA/cruxtok/resolve/main/t0cap_matched/config.yaml
- Command line
-
hf download hf://TimothDevA/cruxtok/t0cap_matched/config.yaml
-
curl -L -o config.yaml https://huggingface.co/TimothDevA/cruxtok/resolve/main/t0cap_matched/config.yaml
1.2 kB
| run_name: t0cap_matched | |
| seed: 42 | |
| data: | |
| dataset: xsum | |
| root: data | |
| model: | |
| arch: ncm | |
| d: 384 | |
| in_vocab_size: 50304 | |
| text_tokenizer: gpt2 | |
| context_length: 48 | |
| pad_id: 50257 | |
| eos_id: 50256 | |
| encoder_blocks: 5 | |
| decoder_blocks: 3 | |
| n_heads: 6 | |
| dropout: 0.1 | |
| levels: | |
| - 4 | |
| - 6 | |
| - 10 | |
| - 14 | |
| - 20 | |
| - 26 | |
| - 32 | |
| - 40 | |
| - 48 | |
| codebook_size: 4096 | |
| pool_self_attn: false | |
| attn_prior: true | |
| prior_sharpness: 4.0 | |
| qk_norm: true | |
| query_init_std: 0.02 | |
| final_level_identity: true | |
| ste: per_level | |
| m: 3 | |
| vq: | |
| decay: 0.99 | |
| epsilon: 1.0e-05 | |
| commitment: 0.25 | |
| restart_threshold: 0.0 | |
| nested_dropout: | |
| prob: 0.5 | |
| min_keep: 1 | |
| pre_quant_dropout: 0.25 | |
| cascade_norm: true | |
| train: | |
| steps: 30000 | |
| micro_batch: 256 | |
| grad_accum: 1 | |
| lr: 0.0005 | |
| lr_floor: 5.0e-05 | |
| warmup: 1000 | |
| betas: | |
| - 0.9 | |
| - 0.95 | |
| weight_decay: 0.01 | |
| grad_clip: 1.0 | |
| max_nonfinite_steps: 200 | |
| runtime: | |
| precision: bf16 | |
| compile: false | |
| ckpt_every: 1000 | |
| keep_ckpts: 2 | |
| log_every: 50 | |
| eval_every: 5000 | |
| eval_examples: 2000 | |
| eval_batch_size: 64 | |
| wandb: false | |
| wandb_project: cruxtok | |
| output_root: /workspace/runs/t0match | |