Download tokenizer_meta.json from eshanized/M31Genesis: direct link, hf CLI and curl.
- Browser
- Download file 655 Bytes
-
https://huggingface.co/eshanized/M31Genesis/resolve/main/tokenizer_meta.json
- Command line
-
hf download hf://eshanized/M31Genesis/tokenizer_meta.json
-
curl -L -o tokenizer_meta.json https://huggingface.co/eshanized/M31Genesis/resolve/main/tokenizer_meta.json
655 Bytes
| { | |
| "vocab_size": 65536, | |
| "special_tokens": [ | |
| "<pad>", | |
| "<bos>", | |
| "<eos>", | |
| "<unk>", | |
| "<system>", | |
| "<user>", | |
| "<assistant>", | |
| "<tool>", | |
| "<tool_call>", | |
| "<tool_result>", | |
| "<repo>", | |
| "<file>", | |
| "<patch>", | |
| "<test>", | |
| "<done>" | |
| ], | |
| "special_ids": { | |
| "<pad>": 0, | |
| "<bos>": 1, | |
| "<eos>": 2, | |
| "<unk>": 3, | |
| "<system>": 4, | |
| "<user>": 5, | |
| "<assistant>": 6, | |
| "<tool>": 7, | |
| "<tool_call>": 8, | |
| "<tool_result>": 9, | |
| "<repo>": 10, | |
| "<file>": 11, | |
| "<patch>": 12, | |
| "<test>": 13, | |
| "<done>": 14 | |
| }, | |
| "source": "code-first: Stack v3 + OpenCodeInstruct + SWE-rebench OpenHands" | |
| } |