Spaces:

zeguru
/

tiny-transformers

Running on Zero

tiny-transformers / checkpoints /shakespear_stories_tokenizer.json

Upload folder using huggingface_hub

1717c0c verified 19 days ago

1.05 kB

{
  "version": 1,
  "stoi": {
    "<PAD>": 0,
    "<UNK>": 1,
    "\n": 2,
    " ": 3,
    "!": 4,
    "\"": 5,
    "$": 6,
    "&": 7,
    "'": 8,
    ",": 9,
    "-": 10,
    ".": 11,
    "0": 12,
    "1": 13,
    "3": 14,
    "8": 15,
    ":": 16,
    ";": 17,
    "?": 18,
    "A": 19,
    "B": 20,
    "C": 21,
    "D": 22,
    "E": 23,
    "F": 24,
    "G": 25,
    "H": 26,
    "I": 27,
    "J": 28,
    "K": 29,
    "L": 30,
    "M": 31,
    "N": 32,
    "O": 33,
    "P": 34,
    "Q": 35,
    "R": 36,
    "S": 37,
    "T": 38,
    "U": 39,
    "V": 40,
    "W": 41,
    "X": 42,
    "Y": 43,
    "Z": 44,
    "a": 45,
    "b": 46,
    "c": 47,
    "d": 48,
    "e": 49,
    "f": 50,
    "g": 51,
    "h": 52,
    "i": 53,
    "j": 54,
    "k": 55,
    "l": 56,
    "m": 57,
    "n": 58,
    "o": 59,
    "p": 60,
    "q": 61,
    "r": 62,
    "s": 63,
    "t": 64,
    "u": 65,
    "v": 66,
    "w": 67,
    "x": 68,
    "y": 69,
    "z": 70,
    "â": 71,
    "œ": 72,
    "“": 73,
    "”": 74,
    "€": 75,
    "™": 76
  }
}

	{
	"version": 1,
	"stoi": {
	"<PAD>": 0,
	"<UNK>": 1,
	"\n": 2,
	" ": 3,
	"!": 4,
	"\"": 5,
	"$": 6,
	"&": 7,
	"'": 8,
	",": 9,
	"-": 10,
	".": 11,
	"0": 12,
	"1": 13,
	"3": 14,
	"8": 15,
	":": 16,
	";": 17,
	"?": 18,
	"A": 19,
	"B": 20,
	"C": 21,
	"D": 22,
	"E": 23,
	"F": 24,
	"G": 25,
	"H": 26,
	"I": 27,
	"J": 28,
	"K": 29,
	"L": 30,
	"M": 31,
	"N": 32,
	"O": 33,
	"P": 34,
	"Q": 35,
	"R": 36,
	"S": 37,
	"T": 38,
	"U": 39,
	"V": 40,
	"W": 41,
	"X": 42,
	"Y": 43,
	"Z": 44,
	"a": 45,
	"b": 46,
	"c": 47,
	"d": 48,
	"e": 49,
	"f": 50,
	"g": 51,
	"h": 52,
	"i": 53,
	"j": 54,
	"k": 55,
	"l": 56,
	"m": 57,
	"n": 58,
	"o": 59,
	"p": 60,
	"q": 61,
	"r": 62,
	"s": 63,
	"t": 64,
	"u": 65,
	"v": 66,
	"w": 67,
	"x": 68,
	"y": 69,
	"z": 70,
	"â": 71,
	"œ": 72,
	"“": 73,
	"”": 74,
	"€": 75,
	"™": 76
	}
	}