File size: 2,105 Bytes
546acd4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
{
  "name": "ChatGBA",
  "architecture": "chatgba",
  "vocabSize": 256,
  "featureDim": 192,
  "numHeads": 6,
  "numKVHeads": 2,
  "headDim": 32,
  "ropeBase": 10000,
  "numBlocks": 10,
  "mlpHiddenDim": 512,
  "convKernel": 4,
  "contextLength": 512,
  "attentionWindows": [
    64,
    64,
    64,
    64,
    0,
    64,
    64,
    64,
    64,
    0
  ],
  "derived": {
    "parameterCount": 4003264,
    "queryHeadsPerKVHead": 3,
    "kvCacheBytesInt8": "2 global blocks x 512 positions x 128 + 8 windowed blocks x 64 positions x 128 = 196,608"
  },
  "chat": {
    "user": "<|u|>",
    "assistant": "<|a|>",
    "end": "<|e|>",
    "endOfText": "<|endoftext|>",
    "encoding": "markers are plain UTF-8 text spelled out byte by byte; no reserved ids",
    "template": "<|u|>{user}<|e|><|a|>{assistant}<|e|>, repeated for each turn pair",
    "structure": "one or more user/assistant turn pairs, starting with the user; no system prompt",
    "generationPrompt": "the conversation so far followed by <|a|>; the reply ends at <|e|>"
  },
  "weights": {
    "format": "self-describing GAB_ file: 12-byte preamble (magic GAB_, version, header length), JSON header, then little-endian float32 weights in this order",
    "matrixLayout": "row-major [in, out]; activations are multiplied as x @ W",
    "order": [
      "tokenEmbeddings [vocabSize, featureDim] (tied: also the output projection, transposed)",
      "for each block: attentionNorm.gamma [1, featureDim], canonA.weight [convKernel, featureDim] (row j weighs the input j positions back)",
      "  for each key/value head g: learnedK [featureDim, headDim], learnedV [featureDim, headDim], keyNorm.gamma [1, headDim]",
      "    then for each of its numHeads / numKVHeads query heads: learnedQ [featureDim, headDim], learnedO [headDim, featureDim], queryNorm.gamma [1, headDim]",
      "  mlpNorm.gamma [1, featureDim], canonC.weight [convKernel, featureDim], mlp.learnedGate [featureDim, mlpHiddenDim], mlp.learnedUp [featureDim, mlpHiddenDim], mlp.learnedDown [mlpHiddenDim, featureDim]",
      "finalNorm.gamma [1, featureDim]"
    ]
  }
}