import subprocess import sys # Install tokenizer without downgrading Hugging Face Hub subprocess.check_call([ sys.executable, "-m", "pip", "install", "--no-deps", "supernova-v5==0.1.1" ]) import gradio as gr import spaces import time import ast import unicodedata from supernova import Tokenizer # Initialize tokenizer try: tokenizer = Tokenizer("v5") TOKENIZER_READY = True LOAD_ERROR = None except Exception as e: tokenizer = None TOKENIZER_READY = False LOAD_ERROR = str(e) # ZeroGPU-compatible encoding function @spaces.GPU def encode_text(text): if not TOKENIZER_READY: return "Tokenizer failed: " + LOAD_ERROR, "", "" if not text: return "Please enter some text.", "", "" try: start = time.perf_counter() tokens = tokenizer.encode(text) elapsed = time.perf_counter() - start decoded = tokenizer.decode(tokens) round_trip = decoded == text stats = f""" ### Encoding statistics - **Characters:** {len(text)} - **UTF-8 bytes:** {len(text.encode('utf-8'))} - **Token count:** {len(tokens)} - **Encoding time:** {elapsed * 1000:.3f} ms - **Round-trip:** {"PASS" if round_trip else "FAIL"} - **Exact reconstruction:** {"Yes" if round_trip else "No"} """ return stats, str(tokens), decoded except Exception as e: return f"Encoding error: {e}", "", "" def decode_tokens(token_text): if not TOKENIZER_READY: return "Tokenizer failed: " + LOAD_ERROR try: tokens = ast.literal_eval(token_text) if not isinstance(tokens, list): return "Input must be a list of token IDs." if not all(type(x) is int for x in tokens): return "Every token ID must be an integer." return tokenizer.decode(tokens) except Exception as e: return f"Decoding error: {e}" def unicode_info(text): if not text: return "Enter text to inspect." lines = [] for char in text: lines.append( f"{repr(char)} | U+{ord(char):04X} | " f"{ord(char)} | " f"{unicodedata.name(char, 'UNKNOWN')}" ) return "\n".join(lines) with gr.Blocks( title="Supernova NepaliFast V5" ) as demo: gr.Markdown(""" # ЁЯММ Supernova NepaliFast V5 ### Interactive Nepali tokenizer playground Encode, decode, inspect token IDs, and verify exact reconstruction. """) if not TOKENIZER_READY: gr.Markdown( f"тЪая╕П **Tokenizer initialization failed:** `{LOAD_ERROR}`" ) else: gr.Markdown( "ЁЯЯв **Supernova NepaliFast V5 loaded successfully**" ) with gr.Tab("Text Encoder"): input_text = gr.Textbox( label="Input text", placeholder="рдиреЗрдкрд╛рд▓ рд░рд╛рдореНрд░реЛ рдЫред", lines=5 ) encode_button = gr.Button( "Encode Text", variant="primary" ) stats_output = gr.Markdown() token_output = gr.Textbox( label="Token IDs", lines=5 ) decoded_output = gr.Textbox( label="Decoded text", lines=3 ) encode_button.click( fn=encode_text, inputs=input_text, outputs=[ stats_output, token_output, decoded_output ] ) gr.Examples( examples=[ ["рдиреЗрдкрд╛рд▓ рд░рд╛рдореНрд░реЛ рдЫред"], ["рдирдорд╕реНрдХрд╛рд░!"], ["Supernova is building language technology."], ["рдиреЗрдкрд╛рд▓ ЁЯЗ│ЁЯЗ╡"], ["AI, рд╡рд┐рдЬреНрдЮрд╛рди рд░ рдкреНрд░рд╡рд┐рдзрд┐"] ], inputs=input_text ) with gr.Tab("Token Decoder"): decoder_input = gr.Textbox( label="Token IDs", placeholder="[123, 456, 789]", lines=5 ) decode_button = gr.Button("Decode Tokens") decoder_output = gr.Textbox( label="Decoded text", lines=5 ) decode_button.click( fn=decode_tokens, inputs=decoder_input, outputs=decoder_output ) with gr.Tab("Unicode Inspector"): unicode_input = gr.Textbox( label="Text", placeholder="рдиреЗрдкрд╛рд▓", lines=3 ) unicode_button = gr.Button("Inspect Unicode") unicode_output = gr.Textbox( label="Unicode information", lines=12 ) unicode_button.click( fn=unicode_info, inputs=unicode_input, outputs=unicode_output ) with gr.Tab("About"): gr.Markdown(""" ## About Supernova NepaliFast V5 Developed by Supernova. **Playground features** - Text encoding - Token ID inspection - Text decoding - Exact reconstruction verification - Unicode inspection **Project:** [Supernova](https://huggingface.co/Supernova11c) **Tokenizer:** [NepaliFast V5](https://huggingface.co/Supernova11c/Supernova-NepaliFast-V5) """) gr.Markdown(""" ---