DMTLLM-Translation-Research / tokenizer_meta.json

Upload folder using huggingface_hub

2e7d3d7 verified 3 months ago

214 Bytes

{
  "algorithm": "ByteLevel BPE",
  "requested_vocab_size": 48000,
  "actual_vocab_size": 48000,
  "special_tokens": {
    "<pad>": 0,
    "<s>": 1,
    "</s>": 2,
    "<unk>": 3,
    "<ko>": 4,
    "<en>": 5
  }
}

	{
	"algorithm": "ByteLevel BPE",
	"requested_vocab_size": 48000,
	"actual_vocab_size": 48000,
	"special_tokens": {
	"<pad>": 0,
	"<s>": 1,
	"</s>": 2,
	"<unk>": 3,
	"<ko>": 4,
	"<en>": 5
	}
	}