| { |
| "accum_steps": 8, |
| "arch_version": 2, |
| "auto_map": { |
| "AutoConfig": "configuration_fwkv.FWKVConfig", |
| "AutoModel": "modeling_fwkv.FWKVLanguageModel", |
| "AutoModelForCausalLM": "modeling_fwkv.FWKVLanguageModel" |
| }, |
| "architectures": [ |
| "FWKVLanguageModel" |
| ], |
| "batch_size": 4, |
| "ce_chunk": 512, |
| "chinchilla_scale": true, |
| "d_emb": 384, |
| "d_model": 768, |
| "dtype": "float32", |
| "early_stop_patience": 2, |
| "epochs": 1, |
| "eval_every_opt_steps": 100, |
| "ffn_mult": 4, |
| "grad_clip": 1.0, |
| "head_size": 64, |
| "keep_last_ckpts": 2, |
| "log_every": 50, |
| "lr": 0.0004, |
| "max_val_examples": 128, |
| "min_lr_ratio": 0.1, |
| "model_type": "fwkv", |
| "n_layers": 13, |
| "prefetch_chunks": 512, |
| "save_every_steps": 500, |
| "seq_len": 1024, |
| "sft_epochs": 3, |
| "sft_lr": 5e-05, |
| "sft_max_train_examples": 40000, |
| "sft_max_val_examples": 1000, |
| "sft_val_skip_rows": 2000, |
| "shuffle_buffer_size": 10000, |
| "tau_max": 1000.0, |
| "tau_min": 2.0, |
| "tie_word_embeddings": true, |
| "train_tokens": 20000000000, |
| "transformers_version": "5.16.1", |
| "use_amp": true, |
| "vocab_size": 50280, |
| "warmup_steps": 1000, |
| "weight_decay": 0.1, |
| "wikitext_max_chunks": 128, |
| "wikitext_val": true, |
| "wkv_ceil": 0.999, |
| "wkv_floor": 0.1 |
| } |
|
|
