File size: 2,302 Bytes
04036d0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
{
  "name": "Gab10M",
  "architecture": "gab10m",
  "vocabSize": 256,
  "featureDim": 256,
  "numHeads": 4,
  "ropeBase": 10000,
  "numPreludeBlocks": 2,
  "numCoreBlocks": 9,
  "numCodaBlocks": 2,
  "loopCount": 2,
  "mlpHiddenDim": 680,
  "convKernel": 4,
  "contextLength": 2048,
  "derived": {
    "headDim": 64,
    "parameterCount": 10302720,
    "uniqueBlocks": 13,
    "effectiveDepth": 22
  },
  "looping": {
    "schedule": "prelude once, core loopCount times with shared weights, coda once",
    "injection": "additive: the prelude output is added to the residual stream at the start of every core pass after the first",
    "kvCache": "one slot per layer pass (effectiveDepth slots), so each core block has loopCount slots; each slot also holds that pass's two canon histories"
  },
  "chat": {
    "user": "<|u|>",
    "assistant": "<|a|>",
    "end": "<|e|>",
    "endOfText": "<|endoftext|>",
    "encoding": "markers are plain UTF-8 text spelled out byte by byte; no reserved ids",
    "template": "<|u|>{user}<|e|><|a|>{assistant}<|e|>, repeated for each turn pair",
    "structure": "one or more user/assistant turn pairs, starting with the user; no system prompt",
    "generationPrompt": "the conversation so far followed by <|a|>; the reply ends at <|e|>"
  },
  "weights": {
    "format": "self-describing GAB_ file: 12-byte preamble (magic GAB_, version, header length), JSON header, then little-endian float32 weights in this order",
    "matrixLayout": "row-major [in, out]; activations are multiplied as x @ W",
    "order": [
      "tokenEmbeddings [vocabSize, featureDim] (tied: also the output projection, transposed)",
      "for each unique block, prelude then core then coda: attentionNorm.gamma [1, featureDim], canonA.weight [convKernel, featureDim] (row j weighs the input j positions back)",
      "  for each head: learnedQ [featureDim, headDim], learnedK [featureDim, headDim], learnedV [featureDim, headDim], learnedO [headDim, featureDim], queryNorm.gamma [1, headDim], keyNorm.gamma [1, headDim]",
      "  mlpNorm.gamma [1, featureDim], canonC.weight [convKernel, featureDim], mlp.learnedGate [featureDim, mlpHiddenDim], mlp.learnedUp [featureDim, mlpHiddenDim], mlp.learnedDown [mlpHiddenDim, featureDim]",
      "finalNorm.gamma [1, featureDim]"
    ]
  }
}