results/tokenizer.csv

id,lang,name,stripped,tokens_marked,tokens_stripped,ratio
yo01,yoruba,Olúwadámilárè,Oluwadamilare,7,5,1.4
yo02,yoruba,Adéṣọlá,Adesola,7,3,2.333
yo03,yoruba,Ọláyínká,Olayinka,6,3,2.0
yo04,yoruba,Fọláṣadé,Folasade,8,3,2.667
yo05,yoruba,Olúwaṣeun,Oluwaseun,8,5,1.6
yo06,yoruba,Ayọ̀délé,Ayodele,5,3,1.667
yo07,yoruba,Ìyábọ̀dé,Iyabode,7,4,1.75
yo08,yoruba,Bọ́lájí,Bolaji,6,2,3.0
yo09,yoruba,Fúnmiláyọ̀,Funmilayo,7,3,2.333
yo10,yoruba,Adébáyọ̀,Adebayo,5,3,1.667
yo11,yoruba,Ọlámidé,Olamide,5,2,2.5
yo12,yoruba,Ìfẹ́olúwa,Ifeoluwa,8,4,2.0
yo13,yoruba,Táíwò,Taiwo,4,2,2.0
yo14,yoruba,Kẹ́hìndé,Kehinde,7,2,3.5
yo15,yoruba,Ìdòwú,Idowu,6,3,2.0
yo16,yoruba,Babátúndé,Babatunde,5,3,1.667
yo17,yoruba,Olúṣẹ́gun,Olusegun,8,3,2.667
yo18,yoruba,Àbẹ̀ní,Abeni,5,2,2.5
yo19,yoruba,Yéwándé,Yewande,5,3,1.667
yo20,yoruba,Adéwálé,Adewale,5,3,1.667
ig01,igbo,Chiọma,Chioma,3,2,1.5
ig02,igbo,Ụchenna,Uchenna,4,3,1.333
ig03,igbo,Obiọma,Obioma,4,3,1.333
ig04,igbo,Ifeọma,Ifeoma,4,3,1.333
ig05,igbo,Chinọnso,Chinonso,3,2,1.5
ig06,igbo,Ụzọamaka,Uzoamaka,6,4,1.5
ig07,igbo,Ifeanyịchukwu,Ifeanyichukwu,6,6,1.0
ig08,igbo,Ngọzi,Ngozi,2,2,1.0
ha01,hausa,Ɗanjuma,Danjuma,5,3,1.667
ha02,hausa,Ɗanladi,Danladi,6,3,2.0
ha03,hausa,Ɗanazumi,Danazumi,6,3,2.0
ct01,control,José,Jose,1,1,1.0
ct02,control,François,Francois,1,1,1.0
ct03,control,Søren,Soren,2,2,1.0
ct04,control,Björn,Bjorn,2,2,1.0
ct05,control,Łukasz,Lukasz,3,2,1.5
ct06,control,Jürgen,Jurgen,1,2,0.5
ct07,control,Dvořák,Dvorak,4,3,1.333
ct08,control,Nguyễn Thị Hương,Nguyen Thi Huong,3,4,0.75
ct09,control,Zhāng Wěi,Zhang Wei,6,2,3.0