anlp-assignment3 / part1 /mamba /protocol.json
sahoosk's picture
Upload Assignment 3 models, results, logs and report
73c0af4 verified
Raw History Blame Contribute Delete
2.96 kB
{
"config": {
"architecture": "mamba",
"dim": 256,
"layers": 6,
"heads": 4,
"state_size": 16,
"expand": 2,
"conv_kernel": 4,
"chunk": 64,
"max_positions": 16384,
"train_window": 1024,
"ffn_hidden": 512,
"dropout": 0.0,
"use_triton": true,
"gradient_checkpointing": true,
"data_root": "data/data_new",
"train_split": "train",
"val_split": "validation",
"max_tokens": 24576,
"eval_max_tokens": 8192,
"max_batch": 64,
"decode_batch": 32,
"decode_tokens": 65536,
"eval_decode_examples": 64,
"lr": 0.001,
"min_lr": 0.0001,
"warmup_steps": 500,
"weight_decay": 0.1,
"betas": [
0.9,
0.95
],
"grad_clip": 1.0,
"max_steps": 25000,
"eval_every": 2500,
"log_every": 50,
"seed": 42,
"amp": "bf16",
"wandb_project": "anlp-a3-part1",
"wandb_mode": "online",
"output_dir": "models/part1/mamba"
},
"model_config": {
"architecture": "mamba",
"dim": 256,
"layers": 6,
"heads": 4,
"state_size": 16,
"expand": 2,
"conv_kernel": 4,
"chunk": 64,
"ffn_hidden": 512,
"dropout": 0.0,
"max_positions": 16384,
"input_size": 83,
"feedback_size": 30,
"output_size": 28,
"use_triton": true
},
"parameters": 6858752,
"train_examples": 569670,
"skipped_train": 0,
"val_examples": 1153,
"skipped_val": 0,
"codec": {
"cipher_chars": [
" ",
"!",
"'",
"+",
",",
"-",
".",
"/",
"0",
"1",
"2",
"3",
"4",
"5",
"6",
"7",
"8",
"9",
":",
";",
"<",
"=",
">",
"?",
"A",
"B",
"C",
"D",
"E",
"F",
"G",
"H",
"I",
"J",
"K",
"L",
"M",
"N",
"O",
"P",
"Q",
"R",
"S",
"T",
"U",
"V",
"W",
"X",
"Y",
"Z",
"[",
"]",
"_",
"a",
"b",
"c",
"d",
"e",
"f",
"g",
"h",
"i",
"j",
"k",
"l",
"m",
"n",
"o",
"p",
"q",
"r",
"s",
"t",
"u",
"v",
"w",
"x",
"y",
"z",
"{",
"}"
],
"plain_chars": [
" ",
"a",
"b",
"c",
"d",
"e",
"f",
"g",
"h",
"i",
"j",
"k",
"l",
"m",
"n",
"o",
"p",
"q",
"r",
"s",
"t",
"u",
"v",
"w",
"x",
"y",
"z"
],
"noise_chars": [
"!",
"'",
"+",
",",
"-",
".",
"/",
":",
";",
"<",
"=",
">",
"?",
"[",
"]",
"_",
"{",
"}"
]
},
"device": "cuda",
"gpu": "NVIDIA RTX 6000 Ada Generation"
}