Axiom-1-Chat / config.json
Atlas Ai
Axiom-1-Chat (65M): weights, tokenizer, config and model card
d09bd31 verified
Raw History Blame Contribute Delete
1.4 kB
{
"architecture": "axiom-1",
"source_checkpoint": "axiom1_chat",
"step": 2000,
"parameters": 64722776,
"axiom_config": {
"seed": 0,
"data": {
"tokenizer": "data/tokens/tokenizer.json",
"train": "data/tokens/chat_train_identity.bin",
"val": "data/tokens/chat_val.bin",
"mix": "data/tokens/fineweb_train.bin"
},
"model": {
"type": "axiom",
"memory": "recurrence",
"vocab": 16384,
"d_model": 640,
"layers": 12,
"d_cell": 1280,
"k_max": 2,
"top_k": 2,
"n_init": 2,
"heads": 10,
"d_ff": 2560
},
"train": {
"phase": "chat",
"seq_len": 1024,
"micro_batch": 8,
"tokens_per_step": 131072,
"steps": 2000,
"lr": 0.0003,
"min_lr_frac": 0.1,
"warmup": 50,
"weight_decay": 0.1,
"clip": 1.0,
"balance": 0.01,
"activation_checkpointing": false,
"eval_windows": 64,
"eval_every": 100,
"log_every": 5,
"save_every": 200,
"aligned": true,
"mix_micro": 2,
"keep_every": 500
},
"structure": {
"enabled": true,
"start": 150,
"every": 200,
"stop_frac": 0.6,
"trial_steps": 20,
"min_tokens": 256,
"max_probe_tokens": 4096,
"max_cells": 8,
"lambda_cell": 0.0002,
"z": 2.0,
"split_scale": 1.0
}
},
"special_tokens": [
"<|endoftext|>",
"<|system|>",
"<|user|>",
"<|assistant|>",
"<|end|>"
]
}