ui/tab_tokenizer.py

"""
Tab 1 - Text Preprocessing (Tokenization)

Interface Gradio : comparaison visuelle de 4 méthodes de tokenisation.
NLTK word-level . SpaCy word-level . BERT WordPiece . GPT-2 BPE
"""

import gradio as gr

from core.tokenizer import tokenize_all
from core.settings import generate_random_text
from core.language_detector import format_language_badge


# Texte exemple par defaut
DEFAULT_TEXT = (
    "Artificial intelligence is transforming the way we process "
    "natural language. Deep learning models like BERT and GPT "
    "can understand context, sentiment, and meaning in unprecedented ways."
)


def _format_tokens(tokens: list[str]) -> str:
    """Formate les tokens pour affichage avec separateurs visuels."""
    return "  >  ".join(f"`{t}`" for t in tokens)


def _run_tokenization(text: str) -> str:
    """Execute les 4 tokenisations et formate les resultats."""
    if not text or not text.strip():
        return "_Enter some text to tokenize._"

    lang_badge = format_language_badge(text)
    results = tokenize_all(text)

    output_parts = []
    for r in results:
        formatted = _format_tokens(r["tokens"])
        output_parts.append(
            f"#### {r['method']}\n"
            f"**{r['count']} tokens**\n\n"
            f"{formatted}"
        )

    body = "\n\n---\n\n".join(output_parts)
    return (lang_badge + "\n\n" + body) if lang_badge else body


def _generate(char_count):
    """Genere du texte aleatoire via Gemini."""
    return generate_random_text(int(char_count))


def create_tab() -> gr.Tab:
    """Cree le tab de tokenisation comparative."""
    with gr.Tab("Tokenization", id="tokenization") as tab:
        gr.Markdown(
            "### Text Preprocessing - Tokenization\n"
            "Compare how different NLP models split text into tokens. "
            "Word-level vs. subword (WordPiece, BPE) tokenization.\n\n"
            ">  **Multilingual** — Language is auto-detected."
        )

        # Input
        text_input = gr.Textbox(
            label="Input Text",
            placeholder="Type or paste your text here...",
            value=DEFAULT_TEXT,
            lines=3,
            max_lines=6,
        )

        # Generate
        with gr.Row():
            char_count = gr.Number(
                label="Characters",
                value=200,
                minimum=20,
                maximum=2000,
                scale=1,
            )
            gen_btn = gr.Button("Generate with Gemini", variant="secondary", scale=2)

        gen_btn.click(
            fn=_generate,
            inputs=[char_count],
            outputs=[text_input],
            show_progress="minimal",
        )

        # Bouton
        btn = gr.Button(
            "Tokenize",
            variant="primary",
            size="lg",
        )

        # Resultats
        gr.Markdown("---")

        results_output = gr.Markdown(value="_Click **Tokenize** to see results._")

        # Event handler
        btn.click(
            fn=_run_tokenization,
            inputs=[text_input],
            outputs=[results_output],
            show_progress="minimal",
        )

    return tab