"""Gradio demo for exploring IndoBERT Large contextual text features.""" import spaces import torch import gradio as gr import matplotlib.pyplot as plt from transformers import AutoModel, AutoTokenizer from embedding_utils import ( cosine_similarity_matrix, parse_input_lines, principal_component_projection, ) MODEL_ID = "indobenchmark/indobert-large-p2" MAX_SAMPLES = 6 MAX_TOKENS = 128 PREVIEW_DIMS = 8 # ZeroGPU intercepts this module-scope move and packs the model for GPU workers. tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) model = AutoModel.from_pretrained( MODEL_ID, torch_dtype=torch.bfloat16, attn_implementation="sdpa", ).eval().to("cuda") @spaces.GPU(duration=60) def analyze( text_block: str, pooling: str = "Mean pooling" ) -> tuple[list[list[object]], list[list[object]], plt.Figure]: """Extract IndoBERT features and compare up to six Indonesian sentences. Args: text_block: One Indonesian sentence per line, up to six lines. pooling: Choose "Mean pooling" or the first-token "[CLS]" vector. Returns: Pairwise cosine scores, a preview of pooled vectors, and a 2D projection. """ texts = parse_input_lines(text_block, max_samples=MAX_SAMPLES) encoded = tokenizer( texts, return_tensors="pt", padding=True, truncation=True, max_length=MAX_TOKENS, ) encoded = {key: value.to("cuda") for key, value in encoded.items()} with torch.inference_mode(): hidden = model(**encoded).last_hidden_state if pooling == "[CLS] token": pooled = hidden[:, 0, :] else: mask = encoded["attention_mask"].unsqueeze(-1).to(hidden.dtype) pooled = (hidden * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1) cpu_vectors = pooled.float().cpu() vectors = cpu_vectors.tolist() similarities = cosine_similarity_matrix(vectors) similarity_rows = [ [f"S{index + 1}"] + row + [None] * (MAX_SAMPLES - len(texts)) for index, row in enumerate(similarities) ] vector_rows = [] for index, (text, vector) in enumerate(zip(texts, vectors)): norm = sum(value * value for value in vector) ** 0.5 preview = [round(value, 5) for value in vector[:PREVIEW_DIMS]] preview.extend([None] * (PREVIEW_DIMS - len(preview))) vector_rows.append([f"S{index + 1}", text, round(norm, 5), *preview]) coordinates = principal_component_projection(vectors) figure, axis = plt.subplots(figsize=(7, 4.5)) axis.scatter( [point[0] for point in coordinates], [point[1] for point in coordinates], s=110, color="#e96b4b", edgecolors="#fff4ed", linewidths=1.5, ) for index, (x_value, y_value) in enumerate(coordinates): axis.annotate(f"S{index + 1}", (x_value, y_value), xytext=(7, 7), textcoords="offset points") axis.set_title("PCA · pooled IndoBERT features") axis.set_xlabel("Principal component 1") axis.set_ylabel("Principal component 2") axis.grid(alpha=0.2) figure.tight_layout() plt.close(figure) return similarity_rows, vector_rows, figure CSS = """ #col-container { max-width: 1120px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); } #hero { padding: 1.4rem 0 .6rem; } #hero h1 { letter-spacing: -0.04em; } """ with gr.Blocks(theme=gr.themes.Citrus(), css=CSS, title="IndoBERT Embedding Explorer") as demo: with gr.Column(elem_id="col-container"): gr.Markdown( """

IndoBERT · Embedding Explorer

Inspect contextual representations from IndoBERT Large P2. Compare sentences, explore pooled vectors, and view a compact projection.

""" ) with gr.Row(): with gr.Column(scale=5): texts = gr.Textbox( label="Kalimat Bahasa Indonesia", placeholder="Satu kalimat per baris…", lines=6, max_lines=8, value="Saya suka membaca buku.\nMembaca buku adalah hobi saya.\nBesok pagi saya pergi ke pasar.", ) pooling = gr.Radio( choices=["Mean pooling", "[CLS] token"], value="Mean pooling", label="Pooling method", ) analyze_button = gr.Button("Explore embeddings", variant="primary") with gr.Column(scale=4): gr.Markdown( "**Quick examples**\n\nTry paraphrases, a different topic, or a short news sentence. Each non-empty line is treated as one sample." ) gr.Examples( examples=[ ["Saya suka membaca buku.\nMembaca buku adalah hobi saya."], ["Harga cabai naik di pasar tradisional."], ], inputs=[texts], label="Load an example", ) gr.Markdown("### Pairwise cosine similarity\nExploratory similarity between pooled features; this model was not fine-tuned as a sentence-embedding model.") similarity = gr.Dataframe( headers=["Sample", "S1", "S2", "S3", "S4", "S5", "S6"], datatype=["str"] + ["number"] * MAX_SAMPLES, label="Cosine similarity (−1 to 1)", interactive=False, ) with gr.Row(): vectors = gr.Dataframe( headers=["Sample", "Text", "L2 norm"] + [f"d{index}" for index in range(PREVIEW_DIMS)], datatype=["str", "str", "number"] + ["number"] * PREVIEW_DIMS, label="Pooled vector preview (first 8 of 1024 dimensions)", interactive=False, wrap=True, ) projection = gr.Plot(label="2D projection") analyze_button.click( fn=analyze, inputs=[texts, pooling], outputs=[similarity, vectors, projection], api_name="analyze", ) gr.Markdown( "Model: [indobenchmark/indobert-large-p2](https://huggingface.co/indobenchmark/indobert-large-p2) · " "The 2D chart uses PCA over pooled contextual features; axes are not semantic dimensions." ) if __name__ == "__main__": demo.launch(mcp_server=True)