Spaces:

kachaf
/

qutuf

Running

qutuf / SourceCode /Views /Tokenizer /test_Tokenizer.py

Upload 210 files

21baa2f verified 7 months ago

232 Bytes

from Tokenizer import *

t = Tokenizer("بسم الله الرحمن الرحيم. الحمد لله رب العالمين.");
#t = Tokenizer("the avg: 8.3 time 2:23:21 am... This is   nice...");
print(t.FinalCharsType);
print(t);


	from Tokenizer import *

	t = Tokenizer("بسم الله الرحمن الرحيم. الحمد لله رب العالمين.");
	#t = Tokenizer("the avg: 8.3 time 2:23:21 am... This is nice...");
	print(t.FinalCharsType);
	print(t);