sirk390org
/

test1

Model card Files Files and versions

xet

Community

test1 / main.py

sirk390

Create main.py

e0a5d4f almost 4 years ago

Raw History Blame Contribute Delete

224 Bytes

from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
example = "My name is Sylvain and I work at Hugging Face in Brooklyn."
encoding = tokenizer(example)
print(type(encoding))

	from transformers import AutoTokenizer

	tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
	example = "My name is Sylvain and I work at Hugging Face in Brooklyn."
	encoding = tokenizer(example)
	print(type(encoding))