ChatGBA / model.json
gszauer's picture
ChatGBA: chat model, Game Boy Advance cartridge and source, Transformers port, reference.js
546acd4 verified
Raw History Blame Contribute Delete
2.11 kB
{
"name": "ChatGBA",
"architecture": "chatgba",
"vocabSize": 256,
"featureDim": 192,
"numHeads": 6,
"numKVHeads": 2,
"headDim": 32,
"ropeBase": 10000,
"numBlocks": 10,
"mlpHiddenDim": 512,
"convKernel": 4,
"contextLength": 512,
"attentionWindows": [
64,
64,
64,
64,
0,
64,
64,
64,
64,
0
],
"derived": {
"parameterCount": 4003264,
"queryHeadsPerKVHead": 3,
"kvCacheBytesInt8": "2 global blocks x 512 positions x 128 + 8 windowed blocks x 64 positions x 128 = 196,608"
},
"chat": {
"user": "<|u|>",
"assistant": "<|a|>",
"end": "<|e|>",
"endOfText": "<|endoftext|>",
"encoding": "markers are plain UTF-8 text spelled out byte by byte; no reserved ids",
"template": "<|u|>{user}<|e|><|a|>{assistant}<|e|>, repeated for each turn pair",
"structure": "one or more user/assistant turn pairs, starting with the user; no system prompt",
"generationPrompt": "the conversation so far followed by <|a|>; the reply ends at <|e|>"
},
"weights": {
"format": "self-describing GAB_ file: 12-byte preamble (magic GAB_, version, header length), JSON header, then little-endian float32 weights in this order",
"matrixLayout": "row-major [in, out]; activations are multiplied as x @ W",
"order": [
"tokenEmbeddings [vocabSize, featureDim] (tied: also the output projection, transposed)",
"for each block: attentionNorm.gamma [1, featureDim], canonA.weight [convKernel, featureDim] (row j weighs the input j positions back)",
" for each key/value head g: learnedK [featureDim, headDim], learnedV [featureDim, headDim], keyNorm.gamma [1, headDim]",
" then for each of its numHeads / numKVHeads query heads: learnedQ [featureDim, headDim], learnedO [headDim, featureDim], queryNorm.gamma [1, headDim]",
" mlpNorm.gamma [1, featureDim], canonC.weight [convKernel, featureDim], mlp.learnedGate [featureDim, mlpHiddenDim], mlp.learnedUp [featureDim, mlpHiddenDim], mlp.learnedDown [mlpHiddenDim, featureDim]",
"finalNorm.gamma [1, featureDim]"
]
}
}