| { | |
| "architecture": "TritEmbed", | |
| "name": "trit-embed-3l", | |
| "format": "trit-packed-v1", | |
| "dims": { | |
| "vocab": 30522, | |
| "emb": 384, | |
| "d": 384, | |
| "heads": 12, | |
| "ffn": 1536, | |
| "layers": 3, | |
| "max_pos": 128, | |
| "eps": 1e-12, | |
| "out_dim": 384, | |
| "loops": 1, | |
| "pool": "mean", | |
| "gelu": "tanh" | |
| }, | |
| "weights": "encoder linears: 2-bit ternary codes {-1,0,1} (code+1, four per byte) and one bf16 scale per output row; word/position embeddings: int8 with one fp32 scale per row; LayerNorms, biases, token-type embedding and output projection: fp32", | |
| "activations": "int8, power-of-two scale per token per 256 channels (per token for 384)", | |
| "max_tokens": 128, | |
| "pooling": "mean", | |
| "normalized": true, | |
| "source_checkpoint_sha256": "ad94bc05dbb4bc233cf2768491183a7e0847ca6cf42c0de34cf7e736aa3440b0", | |
| "training_step": 512 | |
| } | |