{ "config": { "architecture": "mamba", "dim": 256, "layers": 6, "heads": 4, "state_size": 16, "expand": 2, "conv_kernel": 4, "chunk": 64, "max_positions": 16384, "train_window": 1024, "ffn_hidden": 512, "dropout": 0.0, "use_triton": true, "gradient_checkpointing": true, "data_root": "data/data_new", "train_split": "train", "val_split": "validation", "max_tokens": 24576, "eval_max_tokens": 8192, "max_batch": 64, "decode_batch": 32, "decode_tokens": 65536, "eval_decode_examples": 64, "lr": 0.001, "min_lr": 0.0001, "warmup_steps": 500, "weight_decay": 0.1, "betas": [ 0.9, 0.95 ], "grad_clip": 1.0, "max_steps": 25000, "eval_every": 2500, "log_every": 50, "seed": 42, "amp": "bf16", "wandb_project": "anlp-a3-part1", "wandb_mode": "online", "output_dir": "models/part1/mamba" }, "model_config": { "architecture": "mamba", "dim": 256, "layers": 6, "heads": 4, "state_size": 16, "expand": 2, "conv_kernel": 4, "chunk": 64, "ffn_hidden": 512, "dropout": 0.0, "max_positions": 16384, "input_size": 83, "feedback_size": 30, "output_size": 28, "use_triton": true }, "parameters": 6858752, "train_examples": 569670, "skipped_train": 0, "val_examples": 1153, "skipped_val": 0, "codec": { "cipher_chars": [ " ", "!", "'", "+", ",", "-", ".", "/", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", ":", ";", "<", "=", ">", "?", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "[", "]", "_", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "{", "}" ], "plain_chars": [ " ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z" ], "noise_chars": [ "!", "'", "+", ",", "-", ".", "/", ":", ";", "<", "=", ">", "?", "[", "]", "_", "{", "}" ] }, "device": "cuda", "gpu": "NVIDIA RTX 6000 Ada Generation" }