stage2-LLM / tokenizer_config.json
jainil-ifcp's picture
Upload 5 files
a72002f verified
Raw History Blame Contribute Delete
26 kB
{ "backend": "tokenizers", "bos_token": "<|begin_of_text|>", "clean_up_tokenization_spaces": true, "eos_token": "<|end_of_text|>", "extra_special_tokens": [ "<boLatent>", "<eoLatent>", "<Latent-code-0>", "<Latent-code-1>", "<Latent-code-2>", "<Latent-code-3>", "<Latent-code-4>", "<Latent-code-5>", "<Latent-code-6>", "<Latent-code-7>", "<Latent-code-8>", "<Latent-code-9>", "<Latent-code-10>", "<Latent-code-11>", "<Latent-code-12>", "<Latent-code-13>", "<Latent-code-14>", "<Latent-code-15>", "<Latent-code-16>", "<Latent-code-17>", "<Latent-code-18>", "<Latent-code-19>", "<Latent-code-20>", "<Latent-code-21>", "<Latent-code-22>", "<Latent-code-23>", "<Latent-code-24>", "<Latent-code-25>", "<Latent-code-26>", "<Latent-code-27>", "<Latent-code-28>", "<Latent-code-29>", "<Latent-code-30>", "<Latent-code-31>", "<Latent-code-32>", "<Latent-code-33>", "<Latent-code-34>", "<Latent-code-35>", "<Latent-code-36>", "<Latent-code-37>", "<Latent-code-38>", "<Latent-code-39>", "<Latent-code-40>", "<Latent-code-41>", "<Latent-code-42>", "<Latent-code-43>", "<Latent-code-44>", "<Latent-code-45>", "<Latent-code-46>", "<Latent-code-47>", "<Latent-code-48>", "<Latent-code-49>", "<Latent-code-50>", "<Latent-code-51>", "<Latent-code-52>", "<Latent-code-53>", "<Latent-code-54>", "<Latent-code-55>", "<Latent-code-56>", "<Latent-code-57>", "<Latent-code-58>", "<Latent-code-59>", "<Latent-code-60>", "<Latent-code-61>", "<Latent-code-62>", "<Latent-code-63>", "<Latent-code-64>", "<Latent-code-65>", "<Latent-code-66>", "<Latent-code-67>", "<Latent-code-68>", "<Latent-code-69>", "<Latent-code-70>", "<Latent-code-71>", "<Latent-code-72>", "<Latent-code-73>", "<Latent-code-74>", "<Latent-code-75>", "<Latent-code-76>", "<Latent-code-77>", "<Latent-code-78>", "<Latent-code-79>", "<Latent-code-80>", "<Latent-code-81>", "<Latent-code-82>", "<Latent-code-83>", "<Latent-code-84>", "<Latent-code-85>", "<Latent-code-86>", "<Latent-code-87>", "<Latent-code-88>", "<Latent-code-89>", "<Latent-code-90>", "<Latent-code-91>", "<Latent-code-92>", "<Latent-code-93>", "<Latent-code-94>", "<Latent-code-95>", "<Latent-code-96>", "<Latent-code-97>", "<Latent-code-98>", "<Latent-code-99>", "<Latent-code-100>", "<Latent-code-101>", "<Latent-code-102>", "<Latent-code-103>", "<Latent-code-104>", "<Latent-code-105>", "<Latent-code-106>", "<Latent-code-107>", "<Latent-code-108>", "<Latent-code-109>", "<Latent-code-110>", "<Latent-code-111>", "<Latent-code-112>", "<Latent-code-113>", "<Latent-code-114>", "<Latent-code-115>", "<Latent-code-116>", "<Latent-code-117>", "<Latent-code-118>", "<Latent-code-119>", "<Latent-code-120>", "<Latent-code-121>", "<Latent-code-122>", "<Latent-code-123>", "<Latent-code-124>", "<Latent-code-125>", "<Latent-code-126>", "<Latent-code-127>", "<Latent-code-128>", "<Latent-code-129>", "<Latent-code-130>", "<Latent-code-131>", "<Latent-code-132>", "<Latent-code-133>", "<Latent-code-134>", "<Latent-code-135>", "<Latent-code-136>", "<Latent-code-137>", "<Latent-code-138>", "<Latent-code-139>", "<Latent-code-140>", "<Latent-code-141>", "<Latent-code-142>", "<Latent-code-143>", "<Latent-code-144>", "<Latent-code-145>", "<Latent-code-146>", "<Latent-code-147>", "<Latent-code-148>", "<Latent-code-149>", "<Latent-code-150>", "<Latent-code-151>", "<Latent-code-152>", "<Latent-code-153>", "<Latent-code-154>", "<Latent-code-155>", "<Latent-code-156>", "<Latent-code-157>", "<Latent-code-158>", "<Latent-code-159>", "<Latent-code-160>", "<Latent-code-161>", "<Latent-code-162>", "<Latent-code-163>", "<Latent-code-164>", "<Latent-code-165>", "<Latent-code-166>", "<Latent-code-167>", "<Latent-code-168>", "<Latent-code-169>", "<Latent-code-170>", "<Latent-code-171>", "<Latent-code-172>", "<Latent-code-173>", "<Latent-code-174>", "<Latent-code-175>", "<Latent-code-176>", "<Latent-code-177>", "<Latent-code-178>", "<Latent-code-179>", "<Latent-code-180>", "<Latent-code-181>", "<Latent-code-182>", "<Latent-code-183>", "<Latent-code-184>", "<Latent-code-185>", "<Latent-code-186>", "<Latent-code-187>", "<Latent-code-188>", "<Latent-code-189>", "<Latent-code-190>", "<Latent-code-191>", "<Latent-code-192>", "<Latent-code-193>", "<Latent-code-194>", "<Latent-code-195>", "<Latent-code-196>", "<Latent-code-197>", "<Latent-code-198>", "<Latent-code-199>", "<Latent-code-200>", "<Latent-code-201>", "<Latent-code-202>", "<Latent-code-203>", "<Latent-code-204>", "<Latent-code-205>", "<Latent-code-206>", "<Latent-code-207>", "<Latent-code-208>", "<Latent-code-209>", "<Latent-code-210>", "<Latent-code-211>", "<Latent-code-212>", "<Latent-code-213>", "<Latent-code-214>", "<Latent-code-215>", "<Latent-code-216>", "<Latent-code-217>", "<Latent-code-218>", "<Latent-code-219>", "<Latent-code-220>", "<Latent-code-221>", "<Latent-code-222>", "<Latent-code-223>", "<Latent-code-224>", "<Latent-code-225>", "<Latent-code-226>", "<Latent-code-227>", "<Latent-code-228>", "<Latent-code-229>", "<Latent-code-230>", "<Latent-code-231>", "<Latent-code-232>", "<Latent-code-233>", "<Latent-code-234>", "<Latent-code-235>", "<Latent-code-236>", "<Latent-code-237>", "<Latent-code-238>", "<Latent-code-239>", "<Latent-code-240>", "<Latent-code-241>", "<Latent-code-242>", "<Latent-code-243>", "<Latent-code-244>", "<Latent-code-245>", "<Latent-code-246>", "<Latent-code-247>", "<Latent-code-248>", "<Latent-code-249>", "<Latent-code-250>", "<Latent-code-251>", "<Latent-code-252>", "<Latent-code-253>", "<Latent-code-254>", "<Latent-code-255>", "<Latent-code-256>", "<Latent-code-257>", "<Latent-code-258>", "<Latent-code-259>", "<Latent-code-260>", "<Latent-code-261>", "<Latent-code-262>", "<Latent-code-263>", "<Latent-code-264>", "<Latent-code-265>", "<Latent-code-266>", "<Latent-code-267>", "<Latent-code-268>", "<Latent-code-269>", "<Latent-code-270>", "<Latent-code-271>", "<Latent-code-272>", "<Latent-code-273>", "<Latent-code-274>", "<Latent-code-275>", "<Latent-code-276>", "<Latent-code-277>", "<Latent-code-278>", "<Latent-code-279>", "<Latent-code-280>", "<Latent-code-281>", "<Latent-code-282>", "<Latent-code-283>", "<Latent-code-284>", "<Latent-code-285>", "<Latent-code-286>", "<Latent-code-287>", "<Latent-code-288>", "<Latent-code-289>", "<Latent-code-290>", "<Latent-code-291>", "<Latent-code-292>", "<Latent-code-293>", "<Latent-code-294>", "<Latent-code-295>", "<Latent-code-296>", "<Latent-code-297>", "<Latent-code-298>", "<Latent-code-299>", "<Latent-code-300>", "<Latent-code-301>", "<Latent-code-302>", "<Latent-code-303>", "<Latent-code-304>", "<Latent-code-305>", "<Latent-code-306>", "<Latent-code-307>", "<Latent-code-308>", "<Latent-code-309>", "<Latent-code-310>", "<Latent-code-311>", "<Latent-code-312>", "<Latent-code-313>", "<Latent-code-314>", "<Latent-code-315>", "<Latent-code-316>", "<Latent-code-317>", "<Latent-code-318>", "<Latent-code-319>", "<Latent-code-320>", "<Latent-code-321>", "<Latent-code-322>", "<Latent-code-323>", "<Latent-code-324>", "<Latent-code-325>", "<Latent-code-326>", "<Latent-code-327>", "<Latent-code-328>", "<Latent-code-329>", "<Latent-code-330>", "<Latent-code-331>", "<Latent-code-332>", "<Latent-code-333>", "<Latent-code-334>", "<Latent-code-335>", "<Latent-code-336>", "<Latent-code-337>", "<Latent-code-338>", "<Latent-code-339>", "<Latent-code-340>", "<Latent-code-341>", "<Latent-code-342>", "<Latent-code-343>", "<Latent-code-344>", "<Latent-code-345>", "<Latent-code-346>", "<Latent-code-347>", "<Latent-code-348>", "<Latent-code-349>", "<Latent-code-350>", "<Latent-code-351>", "<Latent-code-352>", "<Latent-code-353>", "<Latent-code-354>", "<Latent-code-355>", "<Latent-code-356>", "<Latent-code-357>", "<Latent-code-358>", "<Latent-code-359>", "<Latent-code-360>", "<Latent-code-361>", "<Latent-code-362>", "<Latent-code-363>", "<Latent-code-364>", "<Latent-code-365>", "<Latent-code-366>", "<Latent-code-367>", "<Latent-code-368>", "<Latent-code-369>", "<Latent-code-370>", "<Latent-code-371>", "<Latent-code-372>", "<Latent-code-373>", "<Latent-code-374>", "<Latent-code-375>", "<Latent-code-376>", "<Latent-code-377>", "<Latent-code-378>", "<Latent-code-379>", "<Latent-code-380>", "<Latent-code-381>", "<Latent-code-382>", "<Latent-code-383>", "<Latent-code-384>", "<Latent-code-385>", "<Latent-code-386>", "<Latent-code-387>", "<Latent-code-388>", "<Latent-code-389>", "<Latent-code-390>", "<Latent-code-391>", "<Latent-code-392>", "<Latent-code-393>", "<Latent-code-394>", "<Latent-code-395>", "<Latent-code-396>", "<Latent-code-397>", "<Latent-code-398>", "<Latent-code-399>", "<Latent-code-400>", "<Latent-code-401>", "<Latent-code-402>", "<Latent-code-403>", "<Latent-code-404>", "<Latent-code-405>", "<Latent-code-406>", "<Latent-code-407>", "<Latent-code-408>", "<Latent-code-409>", "<Latent-code-410>", "<Latent-code-411>", "<Latent-code-412>", "<Latent-code-413>", "<Latent-code-414>", "<Latent-code-415>", "<Latent-code-416>", "<Latent-code-417>", "<Latent-code-418>", "<Latent-code-419>", "<Latent-code-420>", "<Latent-code-421>", "<Latent-code-422>", "<Latent-code-423>", "<Latent-code-424>", "<Latent-code-425>", "<Latent-code-426>", "<Latent-code-427>", "<Latent-code-428>", "<Latent-code-429>", "<Latent-code-430>", "<Latent-code-431>", "<Latent-code-432>", "<Latent-code-433>", "<Latent-code-434>", "<Latent-code-435>", "<Latent-code-436>", "<Latent-code-437>", "<Latent-code-438>", "<Latent-code-439>", "<Latent-code-440>", "<Latent-code-441>", "<Latent-code-442>", "<Latent-code-443>", "<Latent-code-444>", "<Latent-code-445>", "<Latent-code-446>", "<Latent-code-447>", "<Latent-code-448>", "<Latent-code-449>", "<Latent-code-450>", "<Latent-code-451>", "<Latent-code-452>", "<Latent-code-453>", "<Latent-code-454>", "<Latent-code-455>", "<Latent-code-456>", "<Latent-code-457>", "<Latent-code-458>", "<Latent-code-459>", "<Latent-code-460>", "<Latent-code-461>", "<Latent-code-462>", "<Latent-code-463>", "<Latent-code-464>", "<Latent-code-465>", "<Latent-code-466>", "<Latent-code-467>", "<Latent-code-468>", "<Latent-code-469>", "<Latent-code-470>", "<Latent-code-471>", "<Latent-code-472>", "<Latent-code-473>", "<Latent-code-474>", "<Latent-code-475>", "<Latent-code-476>", "<Latent-code-477>", "<Latent-code-478>", "<Latent-code-479>", "<Latent-code-480>", "<Latent-code-481>", "<Latent-code-482>", "<Latent-code-483>", "<Latent-code-484>", "<Latent-code-485>", "<Latent-code-486>", "<Latent-code-487>", "<Latent-code-488>", "<Latent-code-489>", "<Latent-code-490>", "<Latent-code-491>", "<Latent-code-492>", "<Latent-code-493>", "<Latent-code-494>", "<Latent-code-495>", "<Latent-code-496>", "<Latent-code-497>", "<Latent-code-498>", "<Latent-code-499>", "<Latent-code-500>", "<Latent-code-501>", "<Latent-code-502>", "<Latent-code-503>", "<Latent-code-504>", "<Latent-code-505>", "<Latent-code-506>", "<Latent-code-507>", "<Latent-code-508>", "<Latent-code-509>", "<Latent-code-510>", "<Latent-code-511>", "<Latent-code-512>", "<Latent-code-513>", "<Latent-code-514>", "<Latent-code-515>", "<Latent-code-516>", "<Latent-code-517>", "<Latent-code-518>", "<Latent-code-519>", "<Latent-code-520>", "<Latent-code-521>", "<Latent-code-522>", "<Latent-code-523>", "<Latent-code-524>", "<Latent-code-525>", "<Latent-code-526>", "<Latent-code-527>", "<Latent-code-528>", "<Latent-code-529>", "<Latent-code-530>", "<Latent-code-531>", "<Latent-code-532>", "<Latent-code-533>", "<Latent-code-534>", "<Latent-code-535>", "<Latent-code-536>", "<Latent-code-537>", "<Latent-code-538>", "<Latent-code-539>", "<Latent-code-540>", "<Latent-code-541>", "<Latent-code-542>", "<Latent-code-543>", "<Latent-code-544>", "<Latent-code-545>", "<Latent-code-546>", "<Latent-code-547>", "<Latent-code-548>", "<Latent-code-549>", "<Latent-code-550>", "<Latent-code-551>", "<Latent-code-552>", "<Latent-code-553>", "<Latent-code-554>", "<Latent-code-555>", "<Latent-code-556>", "<Latent-code-557>", "<Latent-code-558>", "<Latent-code-559>", "<Latent-code-560>", "<Latent-code-561>", "<Latent-code-562>", "<Latent-code-563>", "<Latent-code-564>", "<Latent-code-565>", "<Latent-code-566>", "<Latent-code-567>", "<Latent-code-568>", "<Latent-code-569>", "<Latent-code-570>", "<Latent-code-571>", "<Latent-code-572>", "<Latent-code-573>", "<Latent-code-574>", "<Latent-code-575>", "<Latent-code-576>", "<Latent-code-577>", "<Latent-code-578>", "<Latent-code-579>", "<Latent-code-580>", "<Latent-code-581>", "<Latent-code-582>", "<Latent-code-583>", "<Latent-code-584>", "<Latent-code-585>", "<Latent-code-586>", "<Latent-code-587>", "<Latent-code-588>", "<Latent-code-589>", "<Latent-code-590>", "<Latent-code-591>", "<Latent-code-592>", "<Latent-code-593>", "<Latent-code-594>", "<Latent-code-595>", "<Latent-code-596>", "<Latent-code-597>", "<Latent-code-598>", "<Latent-code-599>", "<Latent-code-600>", "<Latent-code-601>", "<Latent-code-602>", "<Latent-code-603>", "<Latent-code-604>", "<Latent-code-605>", "<Latent-code-606>", "<Latent-code-607>", "<Latent-code-608>", "<Latent-code-609>", "<Latent-code-610>", "<Latent-code-611>", "<Latent-code-612>", "<Latent-code-613>", "<Latent-code-614>", "<Latent-code-615>", "<Latent-code-616>", "<Latent-code-617>", "<Latent-code-618>", "<Latent-code-619>", "<Latent-code-620>", "<Latent-code-621>", "<Latent-code-622>", "<Latent-code-623>", "<Latent-code-624>", "<Latent-code-625>", "<Latent-code-626>", "<Latent-code-627>", "<Latent-code-628>", "<Latent-code-629>", "<Latent-code-630>", "<Latent-code-631>", "<Latent-code-632>", "<Latent-code-633>", "<Latent-code-634>", "<Latent-code-635>", "<Latent-code-636>", "<Latent-code-637>", "<Latent-code-638>", "<Latent-code-639>", "<Latent-code-640>", "<Latent-code-641>", "<Latent-code-642>", "<Latent-code-643>", "<Latent-code-644>", "<Latent-code-645>", "<Latent-code-646>", "<Latent-code-647>", "<Latent-code-648>", "<Latent-code-649>", "<Latent-code-650>", "<Latent-code-651>", "<Latent-code-652>", "<Latent-code-653>", "<Latent-code-654>", "<Latent-code-655>", "<Latent-code-656>", "<Latent-code-657>", "<Latent-code-658>", "<Latent-code-659>", "<Latent-code-660>", "<Latent-code-661>", "<Latent-code-662>", "<Latent-code-663>", "<Latent-code-664>", "<Latent-code-665>", "<Latent-code-666>", "<Latent-code-667>", "<Latent-code-668>", "<Latent-code-669>", "<Latent-code-670>", "<Latent-code-671>", "<Latent-code-672>", "<Latent-code-673>", "<Latent-code-674>", "<Latent-code-675>", "<Latent-code-676>", "<Latent-code-677>", "<Latent-code-678>", "<Latent-code-679>", "<Latent-code-680>", "<Latent-code-681>", "<Latent-code-682>", "<Latent-code-683>", "<Latent-code-684>", "<Latent-code-685>", "<Latent-code-686>", "<Latent-code-687>", "<Latent-code-688>", "<Latent-code-689>", "<Latent-code-690>", "<Latent-code-691>", "<Latent-code-692>", "<Latent-code-693>", "<Latent-code-694>", "<Latent-code-695>", "<Latent-code-696>", "<Latent-code-697>", "<Latent-code-698>", "<Latent-code-699>", "<Latent-code-700>", "<Latent-code-701>", "<Latent-code-702>", "<Latent-code-703>", "<Latent-code-704>", "<Latent-code-705>", "<Latent-code-706>", "<Latent-code-707>", "<Latent-code-708>", "<Latent-code-709>", "<Latent-code-710>", "<Latent-code-711>", "<Latent-code-712>", "<Latent-code-713>", "<Latent-code-714>", "<Latent-code-715>", "<Latent-code-716>", "<Latent-code-717>", "<Latent-code-718>", "<Latent-code-719>", "<Latent-code-720>", "<Latent-code-721>", "<Latent-code-722>", "<Latent-code-723>", "<Latent-code-724>", "<Latent-code-725>", "<Latent-code-726>", "<Latent-code-727>", "<Latent-code-728>", "<Latent-code-729>", "<Latent-code-730>", "<Latent-code-731>", "<Latent-code-732>", "<Latent-code-733>", "<Latent-code-734>", "<Latent-code-735>", "<Latent-code-736>", "<Latent-code-737>", "<Latent-code-738>", "<Latent-code-739>", "<Latent-code-740>", "<Latent-code-741>", "<Latent-code-742>", "<Latent-code-743>", "<Latent-code-744>", "<Latent-code-745>", "<Latent-code-746>", "<Latent-code-747>", "<Latent-code-748>", "<Latent-code-749>", "<Latent-code-750>", "<Latent-code-751>", "<Latent-code-752>", "<Latent-code-753>", "<Latent-code-754>", "<Latent-code-755>", "<Latent-code-756>", "<Latent-code-757>", "<Latent-code-758>", "<Latent-code-759>", "<Latent-code-760>", "<Latent-code-761>", "<Latent-code-762>", "<Latent-code-763>", "<Latent-code-764>", "<Latent-code-765>", "<Latent-code-766>", "<Latent-code-767>", "<Latent-code-768>", "<Latent-code-769>", "<Latent-code-770>", "<Latent-code-771>", "<Latent-code-772>", "<Latent-code-773>", "<Latent-code-774>", "<Latent-code-775>", "<Latent-code-776>", "<Latent-code-777>", "<Latent-code-778>", "<Latent-code-779>", "<Latent-code-780>", "<Latent-code-781>", "<Latent-code-782>", "<Latent-code-783>", "<Latent-code-784>", "<Latent-code-785>", "<Latent-code-786>", "<Latent-code-787>", "<Latent-code-788>", "<Latent-code-789>", "<Latent-code-790>", "<Latent-code-791>", "<Latent-code-792>", "<Latent-code-793>", "<Latent-code-794>", "<Latent-code-795>", "<Latent-code-796>", "<Latent-code-797>", "<Latent-code-798>", "<Latent-code-799>", "<Latent-code-800>", "<Latent-code-801>", "<Latent-code-802>", "<Latent-code-803>", "<Latent-code-804>", "<Latent-code-805>", "<Latent-code-806>", "<Latent-code-807>", "<Latent-code-808>", "<Latent-code-809>", "<Latent-code-810>", "<Latent-code-811>", "<Latent-code-812>", "<Latent-code-813>", "<Latent-code-814>", "<Latent-code-815>", "<Latent-code-816>", "<Latent-code-817>", "<Latent-code-818>", "<Latent-code-819>", "<Latent-code-820>", "<Latent-code-821>", "<Latent-code-822>", "<Latent-code-823>", "<Latent-code-824>", "<Latent-code-825>", "<Latent-code-826>", "<Latent-code-827>", "<Latent-code-828>", "<Latent-code-829>", "<Latent-code-830>", "<Latent-code-831>", "<Latent-code-832>", "<Latent-code-833>", "<Latent-code-834>", "<Latent-code-835>", "<Latent-code-836>", "<Latent-code-837>", "<Latent-code-838>", "<Latent-code-839>", "<Latent-code-840>", "<Latent-code-841>", "<Latent-code-842>", "<Latent-code-843>", "<Latent-code-844>", "<Latent-code-845>", "<Latent-code-846>", "<Latent-code-847>", "<Latent-code-848>", "<Latent-code-849>", "<Latent-code-850>", "<Latent-code-851>", "<Latent-code-852>", "<Latent-code-853>", "<Latent-code-854>", "<Latent-code-855>", "<Latent-code-856>", "<Latent-code-857>", "<Latent-code-858>", "<Latent-code-859>", "<Latent-code-860>", "<Latent-code-861>", "<Latent-code-862>", "<Latent-code-863>", "<Latent-code-864>", "<Latent-code-865>", "<Latent-code-866>", "<Latent-code-867>", "<Latent-code-868>", "<Latent-code-869>", "<Latent-code-870>", "<Latent-code-871>", "<Latent-code-872>", "<Latent-code-873>", "<Latent-code-874>", "<Latent-code-875>", "<Latent-code-876>", "<Latent-code-877>", "<Latent-code-878>", "<Latent-code-879>", "<Latent-code-880>", "<Latent-code-881>", "<Latent-code-882>", "<Latent-code-883>", "<Latent-code-884>", "<Latent-code-885>", "<Latent-code-886>", "<Latent-code-887>", "<Latent-code-888>", "<Latent-code-889>", "<Latent-code-890>", "<Latent-code-891>", "<Latent-code-892>", "<Latent-code-893>", "<Latent-code-894>", "<Latent-code-895>", "<Latent-code-896>", "<Latent-code-897>", "<Latent-code-898>", "<Latent-code-899>", "<Latent-code-900>", "<Latent-code-901>", "<Latent-code-902>", "<Latent-code-903>", "<Latent-code-904>", "<Latent-code-905>", "<Latent-code-906>", "<Latent-code-907>", "<Latent-code-908>", "<Latent-code-909>", "<Latent-code-910>", "<Latent-code-911>", "<Latent-code-912>", "<Latent-code-913>", "<Latent-code-914>", "<Latent-code-915>", "<Latent-code-916>", "<Latent-code-917>", "<Latent-code-918>", "<Latent-code-919>", "<Latent-code-920>", "<Latent-code-921>", "<Latent-code-922>", "<Latent-code-923>", "<Latent-code-924>", "<Latent-code-925>", "<Latent-code-926>", "<Latent-code-927>", "<Latent-code-928>", "<Latent-code-929>", "<Latent-code-930>", "<Latent-code-931>", "<Latent-code-932>", "<Latent-code-933>", "<Latent-code-934>", "<Latent-code-935>", "<Latent-code-936>", "<Latent-code-937>", "<Latent-code-938>", "<Latent-code-939>", "<Latent-code-940>", "<Latent-code-941>", "<Latent-code-942>", "<Latent-code-943>", "<Latent-code-944>", "<Latent-code-945>", "<Latent-code-946>", "<Latent-code-947>", "<Latent-code-948>", "<Latent-code-949>", "<Latent-code-950>", "<Latent-code-951>", "<Latent-code-952>", "<Latent-code-953>", "<Latent-code-954>", "<Latent-code-955>", "<Latent-code-956>", "<Latent-code-957>", "<Latent-code-958>", "<Latent-code-959>", "<Latent-code-960>", "<Latent-code-961>", "<Latent-code-962>", "<Latent-code-963>", "<Latent-code-964>", "<Latent-code-965>", "<Latent-code-966>", "<Latent-code-967>", "<Latent-code-968>", "<Latent-code-969>", "<Latent-code-970>", "<Latent-code-971>", "<Latent-code-972>", "<Latent-code-973>", "<Latent-code-974>", "<Latent-code-975>", "<Latent-code-976>", "<Latent-code-977>", "<Latent-code-978>", "<Latent-code-979>", "<Latent-code-980>", "<Latent-code-981>", "<Latent-code-982>", "<Latent-code-983>", "<Latent-code-984>", "<Latent-code-985>", "<Latent-code-986>", "<Latent-code-987>", "<Latent-code-988>", "<Latent-code-989>", "<Latent-code-990>", "<Latent-code-991>", "<Latent-code-992>", "<Latent-code-993>", "<Latent-code-994>", "<Latent-code-995>", "<Latent-code-996>", "<Latent-code-997>", "<Latent-code-998>", "<Latent-code-999>", "<Latent-code-1000>", "<Latent-code-1001>", "<Latent-code-1002>", "<Latent-code-1003>", "<Latent-code-1004>", "<Latent-code-1005>", "<Latent-code-1006>", "<Latent-code-1007>", "<Latent-code-1008>", "<Latent-code-1009>", "<Latent-code-1010>", "<Latent-code-1011>", "<Latent-code-1012>", "<Latent-code-1013>", "<Latent-code-1014>", "<Latent-code-1015>", "<Latent-code-1016>", "<Latent-code-1017>", "<Latent-code-1018>", "<Latent-code-1019>", "<Latent-code-1020>", "<Latent-code-1021>", "<Latent-code-1022>", "<Latent-code-1023>" ], "is_local": false, "local_files_only": false, "model_input_names": [ "input_ids", "attention_mask" ], "model_max_length": 131072, "pad_token": "<|finetune_right_pad_id|>", "padding_side": "left", "tokenizer_class": "TokenizersBackend", "unk_token": null }
{
"backend": "tokenizers",
"bos_token": "<|begin_of_text|>",
"clean_up_tokenization_spaces": true,
"eos_token": "<|end_of_text|>",
"extra_special_tokens": [
"<boLatent>",
"<eoLatent>",
"<Latent-code-0>",
"<Latent-code-1>",
"<Latent-code-2>",
"<Latent-code-3>",
"<Latent-code-4>",
"<Latent-code-5>",
"<Latent-code-6>",
"<Latent-code-7>",
"<Latent-code-8>",
"<Latent-code-9>",
"<Latent-code-10>",
"<Latent-code-11>",
"<Latent-code-12>",
"<Latent-code-13>",
"<Latent-code-14>",
"<Latent-code-15>",
"<Latent-code-16>",
"<Latent-code-17>",
"<Latent-code-18>",
"<Latent-code-19>",
"<Latent-code-20>",
"<Latent-code-21>",
"<Latent-code-22>",
"<Latent-code-23>",
"<Latent-code-24>",
"<Latent-code-25>",
"<Latent-code-26>",
"<Latent-code-27>",
"<Latent-code-28>",
"<Latent-code-29>",
"<Latent-code-30>",
"<Latent-code-31>",
"<Latent-code-32>",
"<Latent-code-33>",
"<Latent-code-34>",
"<Latent-code-35>",
"<Latent-code-36>",
"<Latent-code-37>",
"<Latent-code-38>",
"<Latent-code-39>",
"<Latent-code-40>",
"<Latent-code-41>",
"<Latent-code-42>",
"<Latent-code-43>",
"<Latent-code-44>",
"<Latent-code-45>",
"<Latent-code-46>",
"<Latent-code-47>",
"<Latent-code-48>",
"<Latent-code-49>",
"<Latent-code-50>",
"<Latent-code-51>",
"<Latent-code-52>",
"<Latent-code-53>",
"<Latent-code-54>",
"<Latent-code-55>",
"<Latent-code-56>",
"<Latent-code-57>",
"<Latent-code-58>",
"<Latent-code-59>",
"<Latent-code-60>",
"<Latent-code-61>",
"<Latent-code-62>",
"<Latent-code-63>",
"<Latent-code-64>",
"<Latent-code-65>",
"<Latent-code-66>",
"<Latent-code-67>",
"<Latent-code-68>",
"<Latent-code-69>",
"<Latent-code-70>",
"<Latent-code-71>",
"<Latent-code-72>",
"<Latent-code-73>",
"<Latent-code-74>",
"<Latent-code-75>",
"<Latent-code-76>",
"<Latent-code-77>",
"<Latent-code-78>",
"<Latent-code-79>",
"<Latent-code-80>",
"<Latent-code-81>",
"<Latent-code-82>",
"<Latent-code-83>",
"<Latent-code-84>",
"<Latent-code-85>",
"<Latent-code-86>",
"<Latent-code-87>",
"<Latent-code-88>",
"<Latent-code-89>",
"<Latent-code-90>",
"<Latent-code-91>",