Spaces:
Sleeping
Sleeping
File size: 4,808 Bytes
8cf0f8d 729d4b1 8c1afca | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 | from functools import lru_cache
from itertools import combinations
import os
import re
import gensim.downloader as api
import gradio as gr
import pandas as pd
from sentence_transformers import SentenceTransformer, util
MODEL_NAME = "Alibaba-NLP/gte-modernbert-base"
GLOVE_NAME = "glove-wiki-gigaword-50"
MAX_WORDS = 20
@lru_cache(maxsize=1)
def load_glove_model():
return api.load(GLOVE_NAME)
@lru_cache(maxsize=1)
def load_contextual_model():
return SentenceTransformer(MODEL_NAME)
def parse_words(words_text):
words = [
item.strip()
for item in re.split(r"[\n,]+", words_text)
if item.strip()
]
return list(dict.fromkeys(words))
def get_token_span(tokens, word):
cleaned = [
token.replace("##", "").replace("\u0120", "").replace("\u2581", "")
for token in tokens
]
word_clean = word.lower()
for start in range(len(cleaned)):
current = ""
for end in range(start, len(cleaned)):
current += cleaned[end].lower()
if current == word_clean:
return list(range(start, end + 1))
if len(current) > len(word_clean):
break
return None
def contextual_similarity(sent_a, sent_b, word_a, word_b, model):
token_embeddings = model.encode(
[sent_a, sent_b],
output_value="token_embeddings",
show_progress_bar=False,
)
tokens_a = model.tokenizer.tokenize(sent_a)
tokens_b = model.tokenizer.tokenize(sent_b)
span_a = get_token_span(tokens_a, word_a)
span_b = get_token_span(tokens_b, word_b)
if span_a is None or span_b is None:
return None
vec_a = token_embeddings[0][span_a].mean(dim=0)
vec_b = token_embeddings[1][span_b].mean(dim=0)
return util.cos_sim(vec_a, vec_b).item()
def static_similarity(word_a, word_b, glove_model):
try:
return glove_model.similarity(word_a, word_b)
except KeyError:
return None
def compare_word_list(pre_context, words_text, post_context):
words = parse_words(words_text)
if len(words) < 2:
raise gr.Error("Enter at least two words, separated by commas or new lines.")
if len(words) > MAX_WORDS:
raise gr.Error(f"Enter {MAX_WORDS} words or fewer.")
glove_model = load_glove_model()
contextual_model = load_contextual_model()
rows = []
for word_a, word_b in combinations(words, 2):
sent_a = f"{pre_context}{word_a}{post_context}"
sent_b = f"{pre_context}{word_b}{post_context}"
static_sim = static_similarity(word_a, word_b, glove_model)
contextual_sim = contextual_similarity(
sent_a, sent_b, word_a, word_b, contextual_model
)
rows.append(
{
"Word A": word_a,
"Word B": word_b,
"Sentence A": sent_a,
"Sentence B": sent_b,
"Static Sim": None if static_sim is None else round(static_sim, 4),
f"Contextual Sim ({MODEL_NAME})": (
None if contextual_sim is None else round(contextual_sim, 4)
),
}
)
return pd.DataFrame(rows)
with gr.Blocks(title="Static vs. Contextual Embeddings") as demo:
gr.Markdown(
"# Static vs. Contextual Embeddings\n"
"Compare every pair in a word list using static GloVe similarity and "
"contextual token similarity from ModernBERT."
)
with gr.Row():
pre_context = gr.Textbox(
label="Pre-context",
value="She took a firm ",
lines=2,
)
post_context = gr.Textbox(
label="Post-context",
value=" on the issue.",
lines=2,
)
words = gr.Textbox(
label="Words to compare",
value="stance, stand, posture, position",
lines=4,
info="Separate words with commas or new lines.",
)
compare_button = gr.Button("Compare", variant="primary")
output = gr.Dataframe(
label="Pairwise similarities",
wrap=True,
interactive=False,
)
compare_button.click(
fn=compare_word_list,
inputs=[pre_context, words, post_context],
outputs=output,
)
gr.Examples(
examples=[
["I really enjoy eating ", "meat, fruit, steak, apples", " for lunch."],
["She took a firm ", "stance, stand, posture, position", " on the issue."],
["The camera ", "stand, stance", " collapsed during filming."],
],
inputs=[pre_context, words, post_context],
)
if __name__ == "__main__":
port = int(os.environ.get("PORT", os.environ.get("GRADIO_SERVER_PORT", "7860")))
demo.launch(server_name="0.0.0.0", server_port=port, ssr_mode=False)
|