Spaces:
Running on Zero
Running on Zero
Download app.py from jrhode666/embeddinggemma-2-explorer: direct link, hf CLI and curl.
- Browser
- Download file 27.5 kB
-
https://huggingface.co/spaces/jrhode666/embeddinggemma-2-explorer/resolve/main/app.py
- Command line
-
hf download hf://spaces/jrhode666/embeddinggemma-2-explorer/app.py
-
curl -L -o app.py https://huggingface.co/spaces/jrhode666/embeddinggemma-2-explorer/resolve/main/app.py
27.5 kB
| import spaces # must be imported before torch on ZeroGPU | |
| import base64 | |
| import html | |
| import io | |
| import os | |
| import time | |
| import av | |
| import gradio as gr | |
| import librosa | |
| import numpy as np | |
| import torch | |
| from PIL import Image | |
| from sentence_transformers import SentenceTransformer | |
| MODEL_ID = "google/embeddinggemma-2" | |
| HERE = os.path.dirname(os.path.abspath(__file__)) | |
| ASSETS = os.path.join(HERE, "assets") | |
| # bfloat16 on GPU (float16 overflows this model). Loaded once at module scope. | |
| model = SentenceTransformer(MODEL_ID, device="cuda", model_kwargs={"dtype": torch.bfloat16}) | |
| model.max_seq_length = 4096 | |
| DIMS = [768, 512, 256, 128] | |
| TEXT_TASKS = { | |
| "Web search": "SearchQuery", | |
| "Question answering": "QuestionAnswering", | |
| "Fact checking": "FactChecking", | |
| "Code search": "CodeRetrieval", | |
| } | |
| SYMMETRIC_TASKS = { | |
| "Sentence similarity": "SentenceSimilarity", | |
| "Classification": "Classification", | |
| "Clustering": "Clustering", | |
| } | |
| MAX_VIDEO_FRAMES = 24 | |
| SAMPLE_IMAGES = [ | |
| ("cat.jpg", "Pallas's cat walking in snow"), | |
| ("cats_couch.jpg", "Two cats asleep on a pink couch"), | |
| ("bee.jpg", "Bee on a pink cosmos flower"), | |
| ("baklava.jpg", "Baklava on a plate"), | |
| ("mosque.jpg", "Domed mosque with minarets"), | |
| ("squirrel.jpg", "Cubist squirrel painting"), | |
| ("tiger.jpg", "Screenshot: classifier says 'tiger'"), | |
| ] | |
| SAMPLE_AUDIO = [ | |
| ("jfk.wav", "JFK speech clip"), | |
| ("mlk.flac", "MLK speech clip"), | |
| ("kungfu.mp3", "Movie line: 'I know kung fu'"), | |
| ("musicgen.wav", "Short generated music clip"), | |
| ] | |
| SAMPLE_TEXTS = """A wild cat with thick fur prowls through a snowy forest. | |
| Bees pollinate flowers while collecting nectar. | |
| Layers of filo pastry, chopped nuts and honey syrup. | |
| Ask not what your country can do for you. | |
| The orchestra played a lively upbeat tune. | |
| Abstract geometric art inspired by Picasso. | |
| Tourists gather outside a historic building in Istanbul.""" | |
| DEFAULT_DOCS = """Northern lights | Auroras happen when charged particles from the sun collide with gases high in Earth's atmosphere. | |
| Ocean currents | Large-scale currents move heat around the planet and shape regional climates. | |
| Eiffel Tower | The wrought-iron tower was built in Paris for the 1889 World's Fair. | |
| Sleep | During deep sleep the brain consolidates memories formed during the day. | |
| Photosynthesis | Plants convert sunlight, water and carbon dioxide into glucose and oxygen. | |
| Auroras (ES) | Las auroras boreales se producen cuando el viento solar choca con la atmósfera. | |
| binary_search.py | def search(xs, t):\\n lo, hi = 0, len(xs) - 1\\n while lo <= hi:\\n mid = (lo + hi) // 2 | |
| Markets | The stock market closed higher on Friday after strong earnings reports.""" | |
| DEFAULT_SENTENCES = """The cat is sleeping on the sofa. | |
| Le chat dort sur le canapé. | |
| Die Katze schläft auf dem Sofa. | |
| A dog is barking in the garden. | |
| Quarterly revenue grew by 12 percent. | |
| Los ingresos trimestrales crecieron un 12 por ciento.""" | |
| # ----------------------------------------------------------------------------- helpers | |
| def _load_audio(path: str) -> dict: | |
| """Decode any audio file to 16 kHz mono float32, as the model expects.""" | |
| wav, _ = librosa.load(path, sr=16000, mono=True, duration=300) | |
| return {"array": wav.astype(np.float32), "sampling_rate": 16000} | |
| def _load_video(path: str) -> list[Image.Image]: | |
| """Sample a video at ~1 fps (the model's default) into a list of PIL frames.""" | |
| frames: list[Image.Image] = [] | |
| with av.open(path) as container: | |
| stream = container.streams.video[0] | |
| fps = float(stream.average_rate or 25) | |
| step = max(1, round(fps)) | |
| for i, frame in enumerate(container.decode(stream)): | |
| if i % step == 0: | |
| img = frame.to_image().convert("RGB") | |
| img.thumbnail((640, 640)) | |
| frames.append(img) | |
| if len(frames) >= MAX_VIDEO_FRAMES: | |
| break | |
| if not frames: | |
| raise gr.Error("Could not read any frames from that video.") | |
| return frames | |
| def _to_model_input(item: dict): | |
| kind, value = item["kind"], item["value"] | |
| if kind == "text": | |
| return value | |
| if kind == "image": | |
| return {"image": Image.open(value).convert("RGB")} | |
| if kind == "audio": | |
| return {"audio": _load_audio(value)} | |
| if kind == "video": | |
| return {"video": _load_video(value)} | |
| raise ValueError(kind) | |
| def _encode(items: list[dict], prompt_name: str | None, dim: int) -> np.ndarray: | |
| """Embed a mixed list of items. Task prefixes apply to text only; media gets none.""" | |
| out = np.zeros((len(items), int(dim)), dtype=np.float32) | |
| text_idx = [i for i, it in enumerate(items) if it["kind"] == "text"] | |
| media_idx = [i for i, it in enumerate(items) if it["kind"] != "text"] | |
| if text_idx: | |
| vecs = model.encode( | |
| [items[i]["value"] for i in text_idx], | |
| prompt_name=prompt_name, | |
| truncate_dim=int(dim), | |
| normalize_embeddings=True, | |
| convert_to_numpy=True, | |
| batch_size=16, | |
| show_progress_bar=False, | |
| ) | |
| out[text_idx] = vecs.astype(np.float32) | |
| # Media is encoded one at a time: modalities and audio lengths differ per item. | |
| for i in media_idx: | |
| vec = model.encode( | |
| [_to_model_input(items[i])], | |
| truncate_dim=int(dim), | |
| normalize_embeddings=True, | |
| convert_to_numpy=True, | |
| show_progress_bar=False, | |
| )[0] | |
| out[i] = vec.astype(np.float32) | |
| return out | |
| def _thumb_b64(path: str, size: int = 120) -> str: | |
| img = Image.open(path).convert("RGB") | |
| img.thumbnail((size, size)) | |
| buf = io.BytesIO() | |
| img.save(buf, format="JPEG", quality=80) | |
| return "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode() | |
| def _label(path: str, fallback: str | None = None) -> str: | |
| return fallback or os.path.basename(path) | |
| def _gallery_paths(gallery) -> list[tuple[str, str]]: | |
| out = [] | |
| for item in gallery or []: | |
| img, cap = item, None | |
| if isinstance(item, (list, tuple)): | |
| img, cap = item[0], (item[1] if len(item) > 1 else None) | |
| if isinstance(img, dict): | |
| img = img.get("path") or img.get("name") | |
| if isinstance(img, str) and os.path.isfile(img): | |
| out.append((img, _label(img, cap))) | |
| return out | |
| def _file_paths(files) -> list[str]: | |
| out = [] | |
| for f in files or []: | |
| p = f if isinstance(f, str) else getattr(f, "name", None) or (f.get("path") if isinstance(f, dict) else None) | |
| if p and os.path.isfile(p): | |
| out.append(p) | |
| return out | |
| ICONS = {"text": "📝", "image": "🖼️", "audio": "🔊", "video": "🎬"} | |
| def _results_html(ranked: list[tuple[dict, float]], note: str) -> str: | |
| rows = [] | |
| top = max((s for _, s in ranked), default=1.0) or 1.0 | |
| for rank, (item, score) in enumerate(ranked, start=1): | |
| if item["kind"] == "image": | |
| preview = f'<img src="{_thumb_b64(item["value"])}" alt="">' | |
| else: | |
| preview = f'<div class="eg-icon">{ICONS[item["kind"]]}</div>' | |
| width = max(2.0, 100.0 * max(score, 0.0) / top) | |
| rows.append( | |
| f"""<div class="eg-row"> | |
| <div class="eg-rank">{rank}</div> | |
| <div class="eg-prev">{preview}</div> | |
| <div class="eg-body"> | |
| <div class="eg-label"><span class="eg-kind">{item['kind']}</span>{html.escape(item['label'])}</div> | |
| <div class="eg-bar"><span style="width:{width:.1f}%"></span></div> | |
| </div> | |
| <div class="eg-score">{score:.3f}</div> | |
| </div>""" | |
| ) | |
| return f'<div class="eg-note">{html.escape(note)}</div><div class="eg-list">{"".join(rows)}</div>' | |
| # ----------------------------------------------------------------------------- tab 1: any-to-any search | |
| def _build_corpus(texts: str, gallery, audio_files) -> list[dict]: | |
| corpus: list[dict] = [] | |
| for line in (texts or "").splitlines(): | |
| if line.strip(): | |
| corpus.append({"kind": "text", "value": line.strip(), "label": line.strip()}) | |
| for path, label in _gallery_paths(gallery): | |
| corpus.append({"kind": "image", "value": path, "label": label}) | |
| audio_labels = {os.path.join(ASSETS, f): lbl for f, lbl in SAMPLE_AUDIO} | |
| for path in _file_paths(audio_files): | |
| base = os.path.basename(path) | |
| label = next((lbl for f, lbl in SAMPLE_AUDIO if base == f), None) | |
| corpus.append({"kind": "audio", "value": path, "label": audio_labels.get(path, label or base)}) | |
| return corpus | |
| def cross_modal_search( | |
| query_type: str, | |
| query_text: str, | |
| query_image: str | None, | |
| query_audio: str | None, | |
| query_video: str | None, | |
| corpus_texts: str, | |
| corpus_images: list | None, | |
| corpus_audio: list | None, | |
| dim: int = 768, | |
| ) -> str: | |
| """Search a mixed corpus of text, images and audio with a text, image, audio or video query. | |
| Every item is embedded into EmbeddingGemma 2's single shared vector space, | |
| truncated to `dim` (Matryoshka) and L2-normalized, then ranked by cosine similarity. | |
| Args: | |
| query_type: One of "Text", "Image", "Audio", "Video". | |
| query_text: Query string (used when query_type is "Text"). | |
| query_image: Path to a query image (used when query_type is "Image"). | |
| query_audio: Path to a query audio file (used when query_type is "Audio"). | |
| query_video: Path to a query video file (used when query_type is "Video"). | |
| corpus_texts: Text items in the corpus, one per line. | |
| corpus_images: Images in the corpus. | |
| corpus_audio: Audio files in the corpus. | |
| dim: Embedding dimension: 768, 512, 256 or 128. | |
| Returns: | |
| HTML listing the corpus ranked by similarity to the query. | |
| """ | |
| t0 = time.perf_counter() | |
| dim = int(dim) | |
| if query_type == "Text": | |
| if not (query_text or "").strip(): | |
| raise gr.Error("Type a text query.") | |
| query = {"kind": "text", "value": query_text.strip(), "label": query_text.strip()} | |
| elif query_type == "Image": | |
| if not query_image: | |
| raise gr.Error("Upload a query image.") | |
| query = {"kind": "image", "value": query_image, "label": "query image"} | |
| elif query_type == "Audio": | |
| if not query_audio: | |
| raise gr.Error("Upload or record query audio.") | |
| query = {"kind": "audio", "value": query_audio, "label": "query audio"} | |
| else: | |
| if not query_video: | |
| raise gr.Error("Upload a query video.") | |
| query = {"kind": "video", "value": query_video, "label": "query video"} | |
| corpus = _build_corpus(corpus_texts, corpus_images, corpus_audio) | |
| if not corpus: | |
| raise gr.Error("The corpus is empty — add some text, images or audio.") | |
| if len(corpus) > 48: | |
| raise gr.Error("Keep the corpus to 48 items or fewer.") | |
| # Text queries use the search-query prefix; text corpus items use the document format. | |
| q_vec = _encode([query], "SearchQuery", dim)[0] | |
| docs = [ | |
| dict(it, value=f"title: none | text: {it['value']}") if it["kind"] == "text" else it for it in corpus | |
| ] | |
| c_vecs = _encode(docs, None, dim) | |
| scores = np.clip(c_vecs @ q_vec, -1.0, 1.0) # bf16 can nudge self-matches past 1 | |
| order = np.argsort(-scores) | |
| ranked = [(corpus[i], float(scores[i])) for i in order] | |
| counts = {k: sum(1 for c in corpus if c["kind"] == k) for k in ("text", "image", "audio")} | |
| note = ( | |
| f"{query_type} query vs {counts['text']} texts · {counts['image']} images · {counts['audio']} audio clips" | |
| f" · {dim}-d · {time.perf_counter() - t0:.1f}s" | |
| ) | |
| return _results_html(ranked, note) | |
| # ----------------------------------------------------------------------------- tab 2: text retrieval + matryoshka | |
| def _parse_docs(raw: str) -> list[tuple[str, str]]: | |
| docs = [] | |
| for line in (raw or "").splitlines(): | |
| line = line.strip() | |
| if not line: | |
| continue | |
| title, content = ("none", line) | |
| if " | " in line: | |
| title, content = (s.strip() for s in line.split(" | ", 1)) | |
| content = content.replace("\\n", "\n") | |
| if content: | |
| docs.append((title or "none", content)) | |
| return docs | |
| def text_search(query: str, documents: str, task: str = "Web search") -> list[list]: | |
| """Rank documents for a query at every Matryoshka size (768, 512, 256, 128 dims). | |
| The query gets the task prefix for the chosen task; each document is formatted as | |
| `title: {title} | text: {content}`. Shows how ranking holds up as vectors shrink. | |
| Args: | |
| query: The search query. | |
| documents: One document per line, optionally `Title | text`. | |
| task: One of "Web search", "Question answering", "Fact checking", "Code search". | |
| Returns: | |
| Rows of [title, document, cosine@768, rank@768, rank@512, rank@256, rank@128]. | |
| """ | |
| query = (query or "").strip() | |
| docs = _parse_docs(documents) | |
| if not query: | |
| raise gr.Error("Enter a query.") | |
| if not docs: | |
| raise gr.Error("Add at least one document, one per line.") | |
| if len(docs) > 40: | |
| raise gr.Error("Use at most 40 documents.") | |
| prompt = TEXT_TASKS[task] | |
| # Embed once at full size, then truncate + re-normalize for each Matryoshka size. | |
| q = model.encode(query, prompt_name=prompt, convert_to_numpy=True, show_progress_bar=False).astype(np.float32) | |
| d = model.encode( | |
| [f"title: {t} | text: {c}" for t, c in docs], convert_to_numpy=True, show_progress_bar=False | |
| ).astype(np.float32) | |
| ranks, full_scores = {}, None | |
| for dim in DIMS: | |
| qd = q[:dim] / np.linalg.norm(q[:dim]) | |
| dd = d[:, :dim] / np.linalg.norm(d[:, :dim], axis=1, keepdims=True) | |
| s = np.clip(dd @ qd, -1.0, 1.0) | |
| if dim == 768: | |
| full_scores = s | |
| order = np.argsort(-s) | |
| r = np.empty(len(docs), dtype=int) | |
| r[order] = np.arange(1, len(docs) + 1) | |
| ranks[dim] = r | |
| rows = [] | |
| for i in np.argsort(-full_scores): | |
| title, content = docs[i] | |
| snippet = content if len(content) <= 140 else content[:137] + "…" | |
| rows.append([title, snippet, round(float(full_scores[i]), 4)] + [int(ranks[d_][i]) for d_ in DIMS]) | |
| return rows | |
| # ----------------------------------------------------------------------------- tab 3: similarity matrix | |
| def _heatmap_html(labels: list[str], sim: np.ndarray) -> str: | |
| n = len(labels) | |
| lo, hi = float(sim[~np.eye(n, dtype=bool)].min()) if n > 1 else 0.0, 1.0 | |
| head = "".join(f'<th title="{html.escape(l)}">#{j + 1}</th>' for j, l in enumerate(labels)) | |
| body = [] | |
| for i, l in enumerate(labels): | |
| cells = [] | |
| for j in range(n): | |
| v = float(sim[i, j]) | |
| t = 0.0 if hi == lo else (v - lo) / (hi - lo) | |
| alpha = 0.08 + 0.82 * max(0.0, min(1.0, t)) | |
| cells.append(f'<td style="background:rgba(99,102,241,{alpha:.2f})">{v:.2f}</td>') | |
| body.append(f'<tr><th class="eg-rowlab">#{i + 1} {html.escape(l[:60])}</th>{"".join(cells)}</tr>') | |
| return f'<div class="eg-heat"><table><tr><th></th>{head}</tr>{"".join(body)}</table></div>' | |
| def similarity_matrix(sentences: str, task: str = "Sentence similarity", dim: int = 768) -> str: | |
| """Pairwise cosine similarity between lines of text (one per line), as a heatmap. | |
| Every line gets the same symmetric task prefix. Works across languages. | |
| Args: | |
| sentences: Two to sixteen lines of text. | |
| task: One of "Sentence similarity", "Classification", "Clustering". | |
| dim: Embedding dimension: 768, 512, 256 or 128. | |
| Returns: | |
| HTML heatmap of the similarity matrix. | |
| """ | |
| lines = [s.strip() for s in (sentences or "").splitlines() if s.strip()] | |
| if len(lines) < 2: | |
| raise gr.Error("Enter at least two lines.") | |
| if len(lines) > 16: | |
| raise gr.Error("Use at most 16 lines.") | |
| vecs = _encode([{"kind": "text", "value": s} for s in lines], SYMMETRIC_TASKS[task], int(dim)) | |
| return _heatmap_html(lines, np.clip(vecs @ vecs.T, -1.0, 1.0)) | |
| # ----------------------------------------------------------------------------- tab 4: raw embedding (API / MCP) | |
| def embed(text: str, prompt_name: str = "SearchQuery", dim: int = 768) -> list[float]: | |
| """Return the EmbeddingGemma 2 embedding for a piece of text. | |
| Args: | |
| text: Text to embed. | |
| prompt_name: Task prefix name, e.g. SearchQuery, Document, QuestionAnswering, FactChecking, | |
| CodeRetrieval, Classification, Clustering, SentenceSimilarity. | |
| dim: Output dimension: 768, 512, 256 or 128 (L2-normalized). | |
| Returns: | |
| The embedding as a list of floats. | |
| """ | |
| if not (text or "").strip(): | |
| raise gr.Error("Enter some text.") | |
| if prompt_name not in model.prompts: | |
| raise gr.Error(f"Unknown prompt_name. Choose from: {', '.join(sorted(model.prompts))}") | |
| v = model.encode( | |
| text, prompt_name=prompt_name, truncate_dim=int(dim), normalize_embeddings=True, convert_to_numpy=True | |
| ) | |
| return [round(float(x), 6) for x in v] | |
| # ----------------------------------------------------------------------------- UI | |
| CSS = """ | |
| .gradio-container { max-width: 1180px !important; margin: 0 auto !important; } | |
| .dark .gradio-container { color: var(--body-text-color); } | |
| .eg-note { font-size: .85rem; opacity: .75; margin: 2px 0 10px; } | |
| .eg-list { display: flex; flex-direction: column; gap: 6px; } | |
| .eg-row { display: grid; grid-template-columns: 28px 64px 1fr 64px; align-items: center; gap: 10px; | |
| padding: 6px 10px; border: 1px solid var(--border-color-primary); border-radius: 10px; | |
| background: var(--block-background-fill); } | |
| .eg-row:first-child { border-color: var(--color-accent); } | |
| .eg-rank { font-weight: 700; text-align: center; opacity: .7; } | |
| .eg-prev img { width: 64px; height: 48px; object-fit: cover; border-radius: 6px; display: block; } | |
| .eg-icon { width: 64px; height: 48px; display: grid; place-items: center; font-size: 22px; | |
| border-radius: 6px; background: var(--background-fill-secondary); } | |
| .eg-label { font-size: .92rem; line-height: 1.3; } | |
| .eg-kind { font-size: .7rem; text-transform: uppercase; letter-spacing: .04em; opacity: .6; margin-right: 8px; } | |
| .eg-bar { height: 6px; border-radius: 3px; background: var(--background-fill-secondary); margin-top: 6px; overflow: hidden; } | |
| .eg-bar span { display: block; height: 100%; background: var(--color-accent); border-radius: 3px; } | |
| .eg-score { font-variant-numeric: tabular-nums; text-align: right; font-weight: 600; } | |
| .eg-heat { overflow-x: auto; } | |
| .eg-heat table { border-collapse: collapse; font-size: .82rem; } | |
| .eg-heat th, .eg-heat td { padding: 6px 8px; text-align: center; border: 1px solid var(--border-color-primary); } | |
| .eg-heat td { font-variant-numeric: tabular-nums; min-width: 44px; } | |
| .eg-heat .eg-rowlab { text-align: left; font-weight: 500; white-space: nowrap; } | |
| """ | |
| INTRO = f"""# EmbeddingGemma 2 — one vector space for text, images, audio & video | |
| [{MODEL_ID}](https://huggingface.co/{MODEL_ID}) is Google DeepMind's 740M multimodal embedding model: 768-d vectors, | |
| 100+ languages, task prefixes, and Matryoshka truncation to 512 / 256 / 128 dims. | |
| """ | |
| with gr.Blocks(title="EmbeddingGemma 2 Explorer") as demo: | |
| gr.Markdown(INTRO) | |
| with gr.Tabs(): | |
| # ---------------------------------------------------------------- any-to-any | |
| with gr.Tab("🔀 Any-to-any search"): | |
| gr.Markdown( | |
| "Query with **text, an image, audio or a video** and rank a mixed corpus of text, images and audio. " | |
| "Everything lives in the same embedding space." | |
| ) | |
| with gr.Row(): | |
| with gr.Column(scale=5): | |
| q_type = gr.Radio(["Text", "Image", "Audio", "Video"], value="Text", label="Query type") | |
| q_text = gr.Textbox( | |
| label="Text query", value="a fluffy animal in the snow", lines=2, visible=True | |
| ) | |
| q_image = gr.Image(label="Image query", type="filepath", height=220, visible=False) | |
| q_audio = gr.Audio(label="Audio query", type="filepath", visible=False) | |
| q_video = gr.Video(label="Video query (sampled at 1 fps, ≤24 frames)", visible=False) | |
| dim1 = gr.Radio(DIMS, value=768, label="Dimensions (Matryoshka)") | |
| go1 = gr.Button("Search", variant="primary") | |
| with gr.Accordion("Corpus", open=False): | |
| c_texts = gr.Textbox(label="Texts (one per line)", value=SAMPLE_TEXTS, lines=8) | |
| c_images = gr.Gallery( | |
| label="Images", | |
| value=[(os.path.join(ASSETS, f), c) for f, c in SAMPLE_IMAGES], | |
| type="filepath", | |
| columns=4, | |
| height=260, | |
| interactive=True, | |
| ) | |
| c_audio = gr.File( | |
| label="Audio clips", | |
| value=[os.path.join(ASSETS, f) for f, _ in SAMPLE_AUDIO], | |
| file_count="multiple", | |
| file_types=["audio"], | |
| ) | |
| with gr.Column(scale=6): | |
| out1 = gr.HTML(label="Ranked results") | |
| def _toggle(kind): | |
| return [gr.update(visible=kind == k) for k in ("Text", "Image", "Audio", "Video")] | |
| q_type.change(_toggle, q_type, [q_text, q_image, q_audio, q_video], api_visibility="private") | |
| inputs1 = [q_type, q_text, q_image, q_audio, q_video, c_texts, c_images, c_audio, dim1] | |
| go1.click(cross_modal_search, inputs1, out1, api_name="cross_modal_search") | |
| q_text.submit(cross_modal_search, inputs1, out1, api_visibility="private") | |
| gr.Examples( | |
| label="Try", | |
| examples=[ | |
| ["Text", "a fluffy animal in the snow", None, None], | |
| ["Text", "something sweet to eat", None, None], | |
| ["Text", "a famous political speech", None, None], | |
| ["Text", "un insecto sobre una flor", None, None], | |
| ["Text", "a user interface showing model predictions", None, None], | |
| ["Image", "", os.path.join(ASSETS, "cats_couch.jpg"), None], | |
| ["Audio", "", None, os.path.join(ASSETS, "jfk.wav")], | |
| ], | |
| inputs=[q_type, q_text, q_image, q_audio], | |
| ) | |
| # ---------------------------------------------------------------- text retrieval | |
| with gr.Tab("📚 Text retrieval & Matryoshka"): | |
| gr.Markdown( | |
| "Task-prefixed retrieval. The table shows each document's rank at **768 / 512 / 256 / 128** dims " | |
| "— a quick look at how much you can shrink vectors before ranking shifts." | |
| ) | |
| with gr.Row(): | |
| with gr.Column(scale=5): | |
| q2 = gr.Textbox(label="Query", value="What causes the northern lights?", lines=2) | |
| task2 = gr.Dropdown(list(TEXT_TASKS), value="Web search", label="Task prefix") | |
| docs2 = gr.Textbox( | |
| label="Documents", info="One per line, optional `Title | text`.", value=DEFAULT_DOCS, lines=12 | |
| ) | |
| go2 = gr.Button("Rank", variant="primary") | |
| with gr.Column(scale=7): | |
| out2 = gr.Dataframe( | |
| headers=["Title", "Document", "Cosine @768", "Rank @768", "@512", "@256", "@128"], | |
| datatype=["str", "str", "number", "number", "number", "number", "number"], | |
| interactive=False, | |
| wrap=True, | |
| label="Results", | |
| ) | |
| go2.click(text_search, [q2, docs2, task2], out2, api_name="text_search") | |
| q2.submit(text_search, [q2, docs2, task2], out2, api_visibility="private") | |
| gr.Examples( | |
| label="Try", | |
| examples=[ | |
| ["What causes the northern lights?", DEFAULT_DOCS, "Web search"], | |
| ["When was the Eiffel Tower built?", DEFAULT_DOCS, "Question answering"], | |
| ["Solar wind creates auroras.", DEFAULT_DOCS, "Fact checking"], | |
| ["find an item in a sorted list", DEFAULT_DOCS, "Code search"], | |
| ], | |
| inputs=[q2, docs2, task2], | |
| outputs=out2, | |
| fn=text_search, | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| ) | |
| # ---------------------------------------------------------------- similarity | |
| with gr.Tab("🌐 Similarity matrix"): | |
| gr.Markdown("Pairwise cosine similarity — note how translations cluster across languages.") | |
| with gr.Row(): | |
| with gr.Column(scale=4): | |
| s3 = gr.Textbox(label="Lines", value=DEFAULT_SENTENCES, lines=10) | |
| task3 = gr.Dropdown(list(SYMMETRIC_TASKS), value="Sentence similarity", label="Task prefix") | |
| dim3 = gr.Radio(DIMS, value=768, label="Dimensions") | |
| go3 = gr.Button("Compare", variant="primary") | |
| with gr.Column(scale=6): | |
| out3 = gr.HTML() | |
| go3.click(similarity_matrix, [s3, task3, dim3], out3, api_name="similarity_matrix") | |
| gr.Examples( | |
| label="Try", | |
| examples=[ | |
| [DEFAULT_SENTENCES, "Sentence similarity", 768], | |
| [ | |
| "I love this phone, the battery lasts forever.\nThe screen cracked after one day.\n" | |
| "Ce téléphone est génial.\nPantalla rota, muy decepcionado.\nBest purchase I made this year.", | |
| "Classification", | |
| 256, | |
| ], | |
| ], | |
| inputs=[s3, task3, dim3], | |
| outputs=out3, | |
| fn=similarity_matrix, | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| ) | |
| # ---------------------------------------------------------------- raw embedding | |
| with gr.Tab("🧮 Embed"): | |
| gr.Markdown("Get the raw vector — also exposed via the API and as an MCP tool.") | |
| with gr.Row(): | |
| with gr.Column(): | |
| t4 = gr.Textbox(label="Text", value="What causes the northern lights?", lines=3) | |
| p4 = gr.Dropdown(sorted(model.prompts), value="SearchQuery", label="prompt_name") | |
| d4 = gr.Radio(DIMS, value=256, label="Dimensions") | |
| go4 = gr.Button("Embed", variant="primary") | |
| with gr.Column(): | |
| out4 = gr.JSON(label="Embedding") | |
| go4.click(embed, [t4, p4, d4], out4, api_name="embed") | |
| gr.Markdown( | |
| "Runs in bfloat16 on ZeroGPU. Text uses the model's task prefixes; images, audio (16 kHz mono) and video " | |
| "(1 fps) are embedded without a prefix. Sample media from Hugging Face documentation datasets." | |
| ) | |
| if __name__ == "__main__": | |
| demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True) | |