ExperienceNow's picture
Upload folder using huggingface_hub
bd69aa2 verified
Raw History Blame Contribute Delete
6.46 kB
"""Gradio demo for exploring IndoBERT Large contextual text features."""
import spaces
import torch
import gradio as gr
import matplotlib.pyplot as plt
from transformers import AutoModel, AutoTokenizer
from embedding_utils import (
cosine_similarity_matrix,
parse_input_lines,
principal_component_projection,
)
MODEL_ID = "indobenchmark/indobert-large-p2"
MAX_SAMPLES = 6
MAX_TOKENS = 128
PREVIEW_DIMS = 8
# ZeroGPU intercepts this module-scope move and packs the model for GPU workers.
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
model = AutoModel.from_pretrained(
MODEL_ID,
torch_dtype=torch.bfloat16,
attn_implementation="sdpa",
).eval().to("cuda")
@spaces.GPU(duration=60)
def analyze(
text_block: str, pooling: str = "Mean pooling"
) -> tuple[list[list[object]], list[list[object]], plt.Figure]:
"""Extract IndoBERT features and compare up to six Indonesian sentences.
Args:
text_block: One Indonesian sentence per line, up to six lines.
pooling: Choose "Mean pooling" or the first-token "[CLS]" vector.
Returns:
Pairwise cosine scores, a preview of pooled vectors, and a 2D projection.
"""
texts = parse_input_lines(text_block, max_samples=MAX_SAMPLES)
encoded = tokenizer(
texts,
return_tensors="pt",
padding=True,
truncation=True,
max_length=MAX_TOKENS,
)
encoded = {key: value.to("cuda") for key, value in encoded.items()}
with torch.inference_mode():
hidden = model(**encoded).last_hidden_state
if pooling == "[CLS] token":
pooled = hidden[:, 0, :]
else:
mask = encoded["attention_mask"].unsqueeze(-1).to(hidden.dtype)
pooled = (hidden * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1)
cpu_vectors = pooled.float().cpu()
vectors = cpu_vectors.tolist()
similarities = cosine_similarity_matrix(vectors)
similarity_rows = [
[f"S{index + 1}"] + row + [None] * (MAX_SAMPLES - len(texts))
for index, row in enumerate(similarities)
]
vector_rows = []
for index, (text, vector) in enumerate(zip(texts, vectors)):
norm = sum(value * value for value in vector) ** 0.5
preview = [round(value, 5) for value in vector[:PREVIEW_DIMS]]
preview.extend([None] * (PREVIEW_DIMS - len(preview)))
vector_rows.append([f"S{index + 1}", text, round(norm, 5), *preview])
coordinates = principal_component_projection(vectors)
figure, axis = plt.subplots(figsize=(7, 4.5))
axis.scatter(
[point[0] for point in coordinates],
[point[1] for point in coordinates],
s=110,
color="#e96b4b",
edgecolors="#fff4ed",
linewidths=1.5,
)
for index, (x_value, y_value) in enumerate(coordinates):
axis.annotate(f"S{index + 1}", (x_value, y_value), xytext=(7, 7), textcoords="offset points")
axis.set_title("PCA · pooled IndoBERT features")
axis.set_xlabel("Principal component 1")
axis.set_ylabel("Principal component 2")
axis.grid(alpha=0.2)
figure.tight_layout()
plt.close(figure)
return similarity_rows, vector_rows, figure
CSS = """
#col-container { max-width: 1120px; margin: 0 auto; }
.dark .gradio-container { color: var(--body-text-color); }
#hero { padding: 1.4rem 0 .6rem; }
#hero h1 { letter-spacing: -0.04em; }
"""
with gr.Blocks(theme=gr.themes.Citrus(), css=CSS, title="IndoBERT Embedding Explorer") as demo:
with gr.Column(elem_id="col-container"):
gr.Markdown(
"""<div id="hero">
<h1>IndoBERT · Embedding Explorer</h1>
<p>Inspect contextual representations from IndoBERT Large P2. Compare sentences, explore pooled vectors, and view a compact projection.</p>
</div>
"""
)
with gr.Row():
with gr.Column(scale=5):
texts = gr.Textbox(
label="Kalimat Bahasa Indonesia",
placeholder="Satu kalimat per baris…",
lines=6,
max_lines=8,
value="Saya suka membaca buku.\nMembaca buku adalah hobi saya.\nBesok pagi saya pergi ke pasar.",
)
pooling = gr.Radio(
choices=["Mean pooling", "[CLS] token"],
value="Mean pooling",
label="Pooling method",
)
analyze_button = gr.Button("Explore embeddings", variant="primary")
with gr.Column(scale=4):
gr.Markdown(
"**Quick examples**\n\nTry paraphrases, a different topic, or a short news sentence. Each non-empty line is treated as one sample."
)
gr.Examples(
examples=[
["Saya suka membaca buku.\nMembaca buku adalah hobi saya."],
["Harga cabai naik di pasar tradisional."],
],
inputs=[texts],
label="Load an example",
)
gr.Markdown("### Pairwise cosine similarity\nExploratory similarity between pooled features; this model was not fine-tuned as a sentence-embedding model.")
similarity = gr.Dataframe(
headers=["Sample", "S1", "S2", "S3", "S4", "S5", "S6"],
datatype=["str"] + ["number"] * MAX_SAMPLES,
label="Cosine similarity (−1 to 1)",
interactive=False,
)
with gr.Row():
vectors = gr.Dataframe(
headers=["Sample", "Text", "L2 norm"] + [f"d{index}" for index in range(PREVIEW_DIMS)],
datatype=["str", "str", "number"] + ["number"] * PREVIEW_DIMS,
label="Pooled vector preview (first 8 of 1024 dimensions)",
interactive=False,
wrap=True,
)
projection = gr.Plot(label="2D projection")
analyze_button.click(
fn=analyze,
inputs=[texts, pooling],
outputs=[similarity, vectors, projection],
api_name="analyze",
)
gr.Markdown(
"Model: [indobenchmark/indobert-large-p2](https://huggingface.co/indobenchmark/indobert-large-p2) · "
"The 2D chart uses PCA over pooled contextual features; axes are not semantic dimensions."
)
if __name__ == "__main__":
demo.launch(mcp_server=True)