app.py

from functools import lru_cache
from itertools import combinations
import os
import re

import gensim.downloader as api
import gradio as gr
import pandas as pd
from sentence_transformers import SentenceTransformer, util


MODEL_NAME = "Alibaba-NLP/gte-modernbert-base"
GLOVE_NAME = "glove-wiki-gigaword-50"
MAX_WORDS = 20


@lru_cache(maxsize=1)
def load_glove_model():
    return api.load(GLOVE_NAME)


@lru_cache(maxsize=1)
def load_contextual_model():
    return SentenceTransformer(MODEL_NAME)


def parse_words(words_text):
    words = [
        item.strip()
        for item in re.split(r"[\n,]+", words_text)
        if item.strip()
    ]
    return list(dict.fromkeys(words))


def get_token_span(tokens, word):
    cleaned = [
        token.replace("##", "").replace("\u0120", "").replace("\u2581", "")
        for token in tokens
    ]
    word_clean = word.lower()

    for start in range(len(cleaned)):
        current = ""
        for end in range(start, len(cleaned)):
            current += cleaned[end].lower()

            if current == word_clean:
                return list(range(start, end + 1))

            if len(current) > len(word_clean):
                break

    return None


def contextual_similarity(sent_a, sent_b, word_a, word_b, model):
    token_embeddings = model.encode(
        [sent_a, sent_b],
        output_value="token_embeddings",
        show_progress_bar=False,
    )
    tokens_a = model.tokenizer.tokenize(sent_a)
    tokens_b = model.tokenizer.tokenize(sent_b)

    span_a = get_token_span(tokens_a, word_a)
    span_b = get_token_span(tokens_b, word_b)

    if span_a is None or span_b is None:
        return None

    vec_a = token_embeddings[0][span_a].mean(dim=0)
    vec_b = token_embeddings[1][span_b].mean(dim=0)
    return util.cos_sim(vec_a, vec_b).item()


def static_similarity(word_a, word_b, glove_model):
    try:
        return glove_model.similarity(word_a, word_b)
    except KeyError:
        return None


def compare_word_list(pre_context, words_text, post_context):
    words = parse_words(words_text)
    if len(words) < 2:
        raise gr.Error("Enter at least two words, separated by commas or new lines.")
    if len(words) > MAX_WORDS:
        raise gr.Error(f"Enter {MAX_WORDS} words or fewer.")

    glove_model = load_glove_model()
    contextual_model = load_contextual_model()

    rows = []
    for word_a, word_b in combinations(words, 2):
        sent_a = f"{pre_context}{word_a}{post_context}"
        sent_b = f"{pre_context}{word_b}{post_context}"

        static_sim = static_similarity(word_a, word_b, glove_model)
        contextual_sim = contextual_similarity(
            sent_a, sent_b, word_a, word_b, contextual_model
        )

        rows.append(
            {
                "Word A": word_a,
                "Word B": word_b,
                "Sentence A": sent_a,
                "Sentence B": sent_b,
                "Static Sim": None if static_sim is None else round(static_sim, 4),
                f"Contextual Sim ({MODEL_NAME})": (
                    None if contextual_sim is None else round(contextual_sim, 4)
                ),
            }
        )

    return pd.DataFrame(rows)


with gr.Blocks(title="Static vs. Contextual Embeddings") as demo:
    gr.Markdown(
        "# Static vs. Contextual Embeddings\n"
        "Compare every pair in a word list using static GloVe similarity and "
        "contextual token similarity from ModernBERT."
    )

    with gr.Row():
        pre_context = gr.Textbox(
            label="Pre-context",
            value="She took a firm ",
            lines=2,
        )
        post_context = gr.Textbox(
            label="Post-context",
            value=" on the issue.",
            lines=2,
        )

    words = gr.Textbox(
        label="Words to compare",
        value="stance, stand, posture, position",
        lines=4,
        info="Separate words with commas or new lines.",
    )

    compare_button = gr.Button("Compare", variant="primary")
    output = gr.Dataframe(
        label="Pairwise similarities",
        wrap=True,
        interactive=False,
    )

    compare_button.click(
        fn=compare_word_list,
        inputs=[pre_context, words, post_context],
        outputs=output,
    )

    gr.Examples(
        examples=[
            ["I really enjoy eating ", "meat, fruit, steak, apples", " for lunch."],
            ["She took a firm ", "stance, stand, posture, position", " on the issue."],
            ["The camera ", "stand, stance", " collapsed during filming."],
        ],
        inputs=[pre_context, words, post_context],
    )


if __name__ == "__main__":
    port = int(os.environ.get("PORT", os.environ.get("GRADIO_SERVER_PORT", "7860")))
    demo.launch(server_name="0.0.0.0", server_port=port, ssr_mode=False)