Translation
Transformers
Safetensors
PyTorch
Khmer
blenderbot
text2text-generation
transliteration
romanization
romanized-khmer
khmer
character-level
pretrained
Instructions to use samdei/samdei-type-mini-pretrained with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use samdei/samdei-type-mini-pretrained with Transformers:
# Use a pipeline as a high-level helper # Warning: Pipeline type "translation" is no longer supported in transformers v5. # You must load the model directly (see below) or downgrade to v4.x with: # pip install "transformers<5.0.0" from transformers import pipeline pipe = pipeline("translation", model="samdei/samdei-type-mini-pretrained")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForSeq2SeqLM tokenizer = AutoTokenizer.from_pretrained("samdei/samdei-type-mini-pretrained") model = AutoModelForSeq2SeqLM.from_pretrained("samdei/samdei-type-mini-pretrained", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download tokenizer.json from samdei/samdei-type-mini-pretrained: direct link, hf CLI and curl.
- Browser
- Download file 9.24 kB
-
https://huggingface.co/samdei/samdei-type-mini-pretrained/resolve/main/tokenizer.json
- Command line
-
hf download hf://samdei/samdei-type-mini-pretrained/tokenizer.json
-
curl -L -o tokenizer.json https://huggingface.co/samdei/samdei-type-mini-pretrained/resolve/main/tokenizer.json
9.24 kB
| { | |
| "version": "1.0", | |
| "truncation": null, | |
| "padding": null, | |
| "added_tokens": [ | |
| { | |
| "id": 0, | |
| "content": "<pad>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 1, | |
| "content": "<s>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 2, | |
| "content": "</s>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 3, | |
| "content": "<unk>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 228, | |
| "content": "<src_pad>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 229, | |
| "content": "<src_s>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 230, | |
| "content": "</src>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 231, | |
| "content": "<src_unk>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| } | |
| ], | |
| "normalizer": null, | |
| "pre_tokenizer": { | |
| "type": "Split", | |
| "pattern": { | |
| "Regex": "[\\s\\S]" | |
| }, | |
| "behavior": "Isolated", | |
| "invert": false | |
| }, | |
| "post_processor": { | |
| "type": "TemplateProcessing", | |
| "single": [ | |
| { | |
| "Sequence": { | |
| "id": "A", | |
| "type_id": 0 | |
| } | |
| }, | |
| { | |
| "SpecialToken": { | |
| "id": "</src>", | |
| "type_id": 0 | |
| } | |
| } | |
| ], | |
| "pair": [ | |
| { | |
| "Sequence": { | |
| "id": "A", | |
| "type_id": 0 | |
| } | |
| }, | |
| { | |
| "SpecialToken": { | |
| "id": "</src>", | |
| "type_id": 0 | |
| } | |
| }, | |
| { | |
| "Sequence": { | |
| "id": "B", | |
| "type_id": 1 | |
| } | |
| }, | |
| { | |
| "SpecialToken": { | |
| "id": "</src>", | |
| "type_id": 1 | |
| } | |
| } | |
| ], | |
| "special_tokens": { | |
| "</src>": { | |
| "id": "</src>", | |
| "ids": [ | |
| 230 | |
| ], | |
| "tokens": [ | |
| "</src>" | |
| ] | |
| } | |
| } | |
| }, | |
| "decoder": { | |
| "type": "Sequence", | |
| "decoders": [ | |
| { | |
| "type": "Replace", | |
| "pattern": { | |
| "String": "๎" | |
| }, | |
| "content": "" | |
| }, | |
| { | |
| "type": "Fuse" | |
| } | |
| ] | |
| }, | |
| "model": { | |
| "type": "WordLevel", | |
| "vocab": { | |
| "<pad>": 0, | |
| "<s>": 1, | |
| "</s>": 2, | |
| "<unk>": 3, | |
| "๎ ": 4, | |
| "๎!": 5, | |
| "๎\"": 6, | |
| "๎#": 7, | |
| "๎$": 8, | |
| "๎%": 9, | |
| "๎&": 10, | |
| "๎'": 11, | |
| "๎(": 12, | |
| "๎)": 13, | |
| "๎*": 14, | |
| "๎+": 15, | |
| "๎,": 16, | |
| "๎-": 17, | |
| "๎.": 18, | |
| "๎/": 19, | |
| "๎0": 20, | |
| "๎1": 21, | |
| "๎2": 22, | |
| "๎3": 23, | |
| "๎4": 24, | |
| "๎5": 25, | |
| "๎6": 26, | |
| "๎7": 27, | |
| "๎8": 28, | |
| "๎9": 29, | |
| "๎:": 30, | |
| "๎;": 31, | |
| "๎<": 32, | |
| "๎=": 33, | |
| "๎>": 34, | |
| "๎?": 35, | |
| "๎@": 36, | |
| "๎A": 37, | |
| "๎B": 38, | |
| "๎C": 39, | |
| "๎D": 40, | |
| "๎E": 41, | |
| "๎F": 42, | |
| "๎G": 43, | |
| "๎H": 44, | |
| "๎I": 45, | |
| "๎J": 46, | |
| "๎K": 47, | |
| "๎L": 48, | |
| "๎M": 49, | |
| "๎N": 50, | |
| "๎O": 51, | |
| "๎P": 52, | |
| "๎Q": 53, | |
| "๎R": 54, | |
| "๎S": 55, | |
| "๎T": 56, | |
| "๎U": 57, | |
| "๎V": 58, | |
| "๎W": 59, | |
| "๎X": 60, | |
| "๎Y": 61, | |
| "๎Z": 62, | |
| "๎[": 63, | |
| "๎]": 64, | |
| "๎^": 65, | |
| "๎_": 66, | |
| "๎a": 67, | |
| "๎b": 68, | |
| "๎c": 69, | |
| "๎d": 70, | |
| "๎e": 71, | |
| "๎f": 72, | |
| "๎g": 73, | |
| "๎h": 74, | |
| "๎i": 75, | |
| "๎j": 76, | |
| "๎k": 77, | |
| "๎l": 78, | |
| "๎m": 79, | |
| "๎n": 80, | |
| "๎o": 81, | |
| "๎p": 82, | |
| "๎q": 83, | |
| "๎r": 84, | |
| "๎s": 85, | |
| "๎t": 86, | |
| "๎u": 87, | |
| "๎v": 88, | |
| "๎w": 89, | |
| "๎x": 90, | |
| "๎y": 91, | |
| "๎z": 92, | |
| "๎|": 93, | |
| "๎~": 94, | |
| "๎ยก": 95, | |
| "๎ยซ": 96, | |
| "๎ยฎ": 97, | |
| "๎ยฐ": 98, | |
| "๎ยท": 99, | |
| "๎ยป": 100, | |
| "๎ร": 101, | |
| "๎ร ": 102, | |
| "๎รก": 103, | |
| "๎รง": 104, | |
| "๎รจ": 105, | |
| "๎รฉ": 106, | |
| "๎รญ": 107, | |
| "๎ว": 108, | |
| "๎เธ": 109, | |
| "๎เธ": 110, | |
| "๎เธ": 111, | |
| "๎เธ ": 112, | |
| "๎เธข": 113, | |
| "๎เธฃ": 114, | |
| "๎เธฉ": 115, | |
| "๎เธฒ": 116, | |
| "๎เธด": 117, | |
| "๎เน": 118, | |
| "๎เน": 119, | |
| "๎เน": 120, | |
| "๎เน": 121, | |
| "๎เบฅ": 122, | |
| "๎เบง": 123, | |
| "๎เบช": 124, | |
| "๎เบฒ": 125, | |
| "๎แ": 126, | |
| "๎แ": 127, | |
| "๎แ": 128, | |
| "๎แ": 129, | |
| "๎แ": 130, | |
| "๎แ ": 131, | |
| "๎แ": 132, | |
| "๎แ": 133, | |
| "๎แ": 134, | |
| "๎แ": 135, | |
| "๎แ": 136, | |
| "๎แ": 137, | |
| "๎แ": 138, | |
| "๎แ": 139, | |
| "๎แ": 140, | |
| "๎แ": 141, | |
| "๎แ": 142, | |
| "๎แ": 143, | |
| "๎แ": 144, | |
| "๎แ": 145, | |
| "๎แ": 146, | |
| "๎แ": 147, | |
| "๎แ": 148, | |
| "๎แ": 149, | |
| "๎แ": 150, | |
| "๎แ": 151, | |
| "๎แ": 152, | |
| "๎แ": 153, | |
| "๎แ": 154, | |
| "๎แ": 155, | |
| "๎แ": 156, | |
| "๎แ": 157, | |
| "๎แ ": 158, | |
| "๎แก": 159, | |
| "๎แข": 160, | |
| "๎แฅ": 161, | |
| "๎แฆ": 162, | |
| "๎แง": 163, | |
| "๎แฉ": 164, | |
| "๎แช": 165, | |
| "๎แซ": 166, | |
| "๎แฌ": 167, | |
| "๎แญ": 168, | |
| "๎แฎ": 169, | |
| "๎แฏ": 170, | |
| "๎แฐ": 171, | |
| "๎แฑ": 172, | |
| "๎แฒ": 173, | |
| "๎แณ": 174, | |
| "๎แถ": 175, | |
| "๎แท": 176, | |
| "๎แธ": 177, | |
| "๎แน": 178, | |
| "๎แบ": 179, | |
| "๎แป": 180, | |
| "๎แผ": 181, | |
| "๎แฝ": 182, | |
| "๎แพ": 183, | |
| "๎แฟ": 184, | |
| "๎แ": 185, | |
| "๎แ": 186, | |
| "๎แ": 187, | |
| "๎แ": 188, | |
| "๎แ": 189, | |
| "๎แ ": 190, | |
| "๎แ": 191, | |
| "๎แ": 192, | |
| "๎แ": 193, | |
| "๎แ": 194, | |
| "๎แ": 195, | |
| "๎แ": 196, | |
| "๎แ": 197, | |
| "๎แ": 198, | |
| "๎แ": 199, | |
| "๎แ": 200, | |
| "๎แ": 201, | |
| "๎แ": 202, | |
| "๎แ": 203, | |
| "๎แ": 204, | |
| "๎แ": 205, | |
| "๎แ": 206, | |
| "๎แ": 207, | |
| "๎แ": 208, | |
| "๎แ": 209, | |
| "๎โ": 210, | |
| "๎โ": 211, | |
| "๎โ": 212, | |
| "๎โ": 213, | |
| "๎โ": 214, | |
| "๎โ": 215, | |
| "๎โ": 216, | |
| "๎โข": 217, | |
| "๎โฆ": 218, | |
| "๎โน": 219, | |
| "๎โบ": 220, | |
| "๎โ": 221, | |
| "๎โ": 222, | |
| "๎โ": 223, | |
| "๎๏ฟฝ": 224, | |
| "๎๐": 225, | |
| "๎๐": 226, | |
| "๎๐": 227, | |
| "<src_pad>": 228, | |
| "<src_s>": 229, | |
| "</src>": 230, | |
| "<src_unk>": 231, | |
| " ": 232, | |
| "!": 233, | |
| "\"": 234, | |
| "#": 235, | |
| "$": 236, | |
| "%": 237, | |
| "&": 238, | |
| "'": 239, | |
| "(": 240, | |
| ")": 241, | |
| "*": 242, | |
| "+": 243, | |
| ",": 244, | |
| "-": 245, | |
| ".": 246, | |
| "/": 247, | |
| "0": 248, | |
| "1": 249, | |
| "2": 250, | |
| "3": 251, | |
| "4": 252, | |
| "5": 253, | |
| "6": 254, | |
| "7": 255, | |
| "8": 256, | |
| "9": 257, | |
| ":": 258, | |
| ";": 259, | |
| "<": 260, | |
| "=": 261, | |
| ">": 262, | |
| "?": 263, | |
| "@": 264, | |
| "A": 265, | |
| "B": 266, | |
| "C": 267, | |
| "D": 268, | |
| "E": 269, | |
| "F": 270, | |
| "G": 271, | |
| "H": 272, | |
| "I": 273, | |
| "J": 274, | |
| "K": 275, | |
| "L": 276, | |
| "M": 277, | |
| "N": 278, | |
| "O": 279, | |
| "P": 280, | |
| "Q": 281, | |
| "R": 282, | |
| "S": 283, | |
| "T": 284, | |
| "U": 285, | |
| "V": 286, | |
| "W": 287, | |
| "X": 288, | |
| "Y": 289, | |
| "Z": 290, | |
| "[": 291, | |
| "]": 292, | |
| "^": 293, | |
| "_": 294, | |
| "a": 295, | |
| "b": 296, | |
| "c": 297, | |
| "d": 298, | |
| "e": 299, | |
| "f": 300, | |
| "g": 301, | |
| "h": 302, | |
| "i": 303, | |
| "j": 304, | |
| "k": 305, | |
| "l": 306, | |
| "m": 307, | |
| "n": 308, | |
| "o": 309, | |
| "p": 310, | |
| "q": 311, | |
| "r": 312, | |
| "s": 313, | |
| "t": 314, | |
| "u": 315, | |
| "v": 316, | |
| "w": 317, | |
| "x": 318, | |
| "y": 319, | |
| "z": 320, | |
| "|": 321, | |
| "~": 322, | |
| "ยก": 323, | |
| "ยซ": 324, | |
| "ยฎ": 325, | |
| "ยฐ": 326, | |
| "ยท": 327, | |
| "ยป": 328, | |
| "ร": 329, | |
| "รก": 330, | |
| "รง": 331, | |
| "รจ": 332, | |
| "รฉ": 333, | |
| "รญ": 334, | |
| "ว": 335, | |
| "ษ": 336, | |
| "ษ": 337, | |
| "เธ": 338, | |
| "เธ": 339, | |
| "เธ": 340, | |
| "เธ ": 341, | |
| "เธข": 342, | |
| "เธฃ": 343, | |
| "เธฉ": 344, | |
| "เธฒ": 345, | |
| "เธด": 346, | |
| "เน": 347, | |
| "เน": 348, | |
| "เน": 349, | |
| "เน": 350, | |
| "เบฅ": 351, | |
| "เบง": 352, | |
| "เบช": 353, | |
| "เบฒ": 354, | |
| "แ": 355, | |
| "แ": 356, | |
| "แ": 357, | |
| "แ": 358, | |
| "แ": 359, | |
| "แ ": 360, | |
| "แ": 361, | |
| "แ": 362, | |
| "แ": 363, | |
| "แ": 364, | |
| "แ": 365, | |
| "แ": 366, | |
| "แ": 367, | |
| "แ": 368, | |
| "แ": 369, | |
| "แ": 370, | |
| "แ": 371, | |
| "แ": 372, | |
| "แ": 373, | |
| "แ": 374, | |
| "แ": 375, | |
| "แ": 376, | |
| "แ": 377, | |
| "แ": 378, | |
| "แ": 379, | |
| "แ": 380, | |
| "แ": 381, | |
| "แ": 382, | |
| "แ": 383, | |
| "แ ": 384, | |
| "แก": 385, | |
| "แข": 386, | |
| "แง": 387, | |
| "แซ": 388, | |
| "แฌ": 389, | |
| "แฏ": 390, | |
| "แฒ": 391, | |
| "แถ": 392, | |
| "แท": 393, | |
| "แธ": 394, | |
| "แน": 395, | |
| "แบ": 396, | |
| "แป": 397, | |
| "แผ": 398, | |
| "แฝ": 399, | |
| "แพ": 400, | |
| "แฟ": 401, | |
| "แ": 402, | |
| "แ": 403, | |
| "แ": 404, | |
| "แ": 405, | |
| "แ": 406, | |
| "แ ": 407, | |
| "แ": 408, | |
| "แ": 409, | |
| "แ": 410, | |
| "แ": 411, | |
| "แ": 412, | |
| "แ": 413, | |
| "แ": 414, | |
| "แ": 415, | |
| "แ": 416, | |
| "แ": 417, | |
| "แ": 418, | |
| "แ": 419, | |
| "แ": 420, | |
| "แ": 421, | |
| "แ": 422, | |
| "แ": 423, | |
| "แ": 424, | |
| "โ": 425, | |
| "โ": 426, | |
| "โ": 427, | |
| "โ": 428, | |
| "โ": 429, | |
| "โ": 430, | |
| "โ": 431, | |
| "โข": 432, | |
| "โฆ": 433, | |
| "โน": 434, | |
| "โบ": 435, | |
| "โ": 436, | |
| "โ": 437, | |
| "โ": 438, | |
| "๏ฟฝ": 439, | |
| "๐": 440, | |
| "๐": 441, | |
| "๐": 442 | |
| }, | |
| "unk_token": "<src_unk>" | |
| } | |
| } | |