Metis-1.6-Logos / NGRAM_CANONICAL_IDS.json
GiuliannoV's picture
Metis-1.6 Logos
ef4a643
Raw History Blame Contribute Delete
1.1 kB
{
"algorithm": "deepseek-engram-normalization/v1",
"algorithm_reference": "https://github.com/deepseek-ai/Engram/blob/main/engram_demo_v1.py",
"binary": "NGRAM_CANONICAL_IDS.uint16",
"binary_sha256": "6359312eb0b5e892828430d2c9d049021f452580f724265f2f21003565a732a3",
"binary_size_bytes": 131072,
"canonical_ids_contiguous": true,
"canonical_vocabulary_size": 39789,
"created_at": "2026-08-15T05:26:32.143022+00:00",
"dtype": "uint16",
"empty_normalization_fallback": "decoded_token",
"endianness": "little",
"entry_count": 65536,
"manifest_sha256": "0d502c64584cf435bfff6f244ff1dc7d879ca120cd8b060c7616d6dc453c4101",
"maximum_canonical_id": 39788,
"minimum_canonical_id": 0,
"normalization_steps": [
"NFKC",
"NFD",
"StripAccents",
"Lowercase",
"CollapseAsciiWhitespace",
"PreserveSingleSpace",
"Strip"
],
"replacement_character_fallback": "raw_tokenizer_vocab_token",
"schema": "metis.ngram-canonical-ids/v1",
"tokenizer_sha256": "2304b0e1c7b1ff18a3797c40d035bc781652485db4f1fc89381bfde44bbfc76c",
"vocabulary_size": 65536
}