Spaces:
Running on Zero
Running on Zero
| import subprocess | |
| import sys | |
| # Install tokenizer without downgrading Hugging Face Hub | |
| subprocess.check_call([ | |
| sys.executable, | |
| "-m", | |
| "pip", | |
| "install", | |
| "--no-deps", | |
| "supernova-v5==0.1.1" | |
| ]) | |
| import gradio as gr | |
| import spaces | |
| import time | |
| import ast | |
| import unicodedata | |
| from supernova import Tokenizer | |
| # Initialize tokenizer | |
| try: | |
| tokenizer = Tokenizer("v5") | |
| TOKENIZER_READY = True | |
| LOAD_ERROR = None | |
| except Exception as e: | |
| tokenizer = None | |
| TOKENIZER_READY = False | |
| LOAD_ERROR = str(e) | |
| # ZeroGPU-compatible encoding function | |
| def encode_text(text): | |
| if not TOKENIZER_READY: | |
| return "Tokenizer failed: " + LOAD_ERROR, "", "" | |
| if not text: | |
| return "Please enter some text.", "", "" | |
| try: | |
| start = time.perf_counter() | |
| tokens = tokenizer.encode(text) | |
| elapsed = time.perf_counter() - start | |
| decoded = tokenizer.decode(tokens) | |
| round_trip = decoded == text | |
| stats = f""" | |
| ### Encoding statistics | |
| - **Characters:** {len(text)} | |
| - **UTF-8 bytes:** {len(text.encode('utf-8'))} | |
| - **Token count:** {len(tokens)} | |
| - **Encoding time:** {elapsed * 1000:.3f} ms | |
| - **Round-trip:** {"PASS" if round_trip else "FAIL"} | |
| - **Exact reconstruction:** {"Yes" if round_trip else "No"} | |
| """ | |
| return stats, str(tokens), decoded | |
| except Exception as e: | |
| return f"Encoding error: {e}", "", "" | |
| def decode_tokens(token_text): | |
| if not TOKENIZER_READY: | |
| return "Tokenizer failed: " + LOAD_ERROR | |
| try: | |
| tokens = ast.literal_eval(token_text) | |
| if not isinstance(tokens, list): | |
| return "Input must be a list of token IDs." | |
| if not all(type(x) is int for x in tokens): | |
| return "Every token ID must be an integer." | |
| return tokenizer.decode(tokens) | |
| except Exception as e: | |
| return f"Decoding error: {e}" | |
| def unicode_info(text): | |
| if not text: | |
| return "Enter text to inspect." | |
| lines = [] | |
| for char in text: | |
| lines.append( | |
| f"{repr(char)} | U+{ord(char):04X} | " | |
| f"{ord(char)} | " | |
| f"{unicodedata.name(char, 'UNKNOWN')}" | |
| ) | |
| return "\n".join(lines) | |
| with gr.Blocks( | |
| title="Supernova NepaliFast V5" | |
| ) as demo: | |
| gr.Markdown(""" | |
| # 🌌 Supernova NepaliFast V5 | |
| ### Interactive Nepali tokenizer playground | |
| Encode, decode, inspect token IDs, and verify exact reconstruction. | |
| """) | |
| if not TOKENIZER_READY: | |
| gr.Markdown( | |
| f"⚠️ **Tokenizer initialization failed:** `{LOAD_ERROR}`" | |
| ) | |
| else: | |
| gr.Markdown( | |
| "🟢 **Supernova NepaliFast V5 loaded successfully**" | |
| ) | |
| with gr.Tab("Text Encoder"): | |
| input_text = gr.Textbox( | |
| label="Input text", | |
| placeholder="नेपाल राम्रो छ।", | |
| lines=5 | |
| ) | |
| encode_button = gr.Button( | |
| "Encode Text", | |
| variant="primary" | |
| ) | |
| stats_output = gr.Markdown() | |
| token_output = gr.Textbox( | |
| label="Token IDs", | |
| lines=5 | |
| ) | |
| decoded_output = gr.Textbox( | |
| label="Decoded text", | |
| lines=3 | |
| ) | |
| encode_button.click( | |
| fn=encode_text, | |
| inputs=input_text, | |
| outputs=[ | |
| stats_output, | |
| token_output, | |
| decoded_output | |
| ] | |
| ) | |
| gr.Examples( | |
| examples=[ | |
| ["नेपाल राम्रो छ।"], | |
| ["नमस्कार!"], | |
| ["Supernova is building language technology."], | |
| ["नेपाल 🇳🇵"], | |
| ["AI, विज्ञान र प्रविधि"] | |
| ], | |
| inputs=input_text | |
| ) | |
| with gr.Tab("Token Decoder"): | |
| decoder_input = gr.Textbox( | |
| label="Token IDs", | |
| placeholder="[123, 456, 789]", | |
| lines=5 | |
| ) | |
| decode_button = gr.Button("Decode Tokens") | |
| decoder_output = gr.Textbox( | |
| label="Decoded text", | |
| lines=5 | |
| ) | |
| decode_button.click( | |
| fn=decode_tokens, | |
| inputs=decoder_input, | |
| outputs=decoder_output | |
| ) | |
| with gr.Tab("Unicode Inspector"): | |
| unicode_input = gr.Textbox( | |
| label="Text", | |
| placeholder="नेपाल", | |
| lines=3 | |
| ) | |
| unicode_button = gr.Button("Inspect Unicode") | |
| unicode_output = gr.Textbox( | |
| label="Unicode information", | |
| lines=12 | |
| ) | |
| unicode_button.click( | |
| fn=unicode_info, | |
| inputs=unicode_input, | |
| outputs=unicode_output | |
| ) | |
| with gr.Tab("About"): | |
| gr.Markdown(""" | |
| ## About Supernova NepaliFast V5 | |
| Developed by Supernova. | |
| **Playground features** | |
| - Text encoding | |
| - Token ID inspection | |
| - Text decoding | |
| - Exact reconstruction verification | |
| - Unicode inspection | |
| **Project:** | |
| [Supernova](https://huggingface.co/Supernova11c) | |
| **Tokenizer:** | |
| [NepaliFast V5](https://huggingface.co/Supernova11c/Supernova-NepaliFast-V5) | |
| """) | |
| gr.Markdown(""" | |
| --- | |
| <center>Built with 🌌 by Supernova</center> | |
| """) | |
| if __name__ == "__main__": | |
| demo.launch( | |
| theme=gr.themes.Soft() | |
| ) | |