edan20-lab2 / app.py
IsacEP's picture
Update app.py
d3cf9cb verified
Raw History Blame Contribute Delete
2.45 kB
import json
import regex as re
import gradio as gr
from huggingface_hub import hf_hub_download
REPO_ID = "IsacEP/selma_text"
def load_ngrams_jsonl(filename):
path = hf_hub_download(
repo_id=REPO_ID,
filename=filename,
repo_type="dataset",
)
result = {}
with open(path, "r", encoding="utf-8") as f:
for line in f:
if not line.strip():
continue
data = json.loads(line)
key = tuple(data["ngram"]) if len(data["ngram"]) > 1 else data["ngram"][0]
result[key] = data["count"]
return result
unigrams = load_ngrams_jsonl("unigrams.jsonl")
bigrams = load_ngrams_jsonl("bigrams.jsonl")
trigrams = load_ngrams_jsonl("trigrams.jsonl")
def tokenize(text):
return re.findall(r"\p{L}+", text.lower())
def top_words(candidates, n):
candidates.sort(key=lambda x: (-x[1], x[0]))
return [word for word, _ in candidates[:n]]
def predict(text, number_of_predictions=5):
tokens = tokenize(text)
# Trigram prediction when two previous words are available.
context = ["<s>"] + tokens
if len(context) >= 2:
prev2, prev1 = context[-2], context[-1]
candidates = [
(ngram[2], count)
for ngram, count in trigrams.items()
if ngram[0] == prev2 and ngram[1] == prev1
]
words = top_words(candidates, number_of_predictions)
if words:
return ", ".join(words)
# Bigram backoff.
if context:
prev = context[-1]
candidates = [
(ngram[1], count)
for ngram, count in bigrams.items()
if ngram[0] == prev
]
words = top_words(candidates, number_of_predictions)
if words:
return ", ".join(words)
# Unigram backoff.
candidates = [
(word, count)
for word, count in unigrams.items()
if word not in {"<s>", "</s>"}
]
return ", ".join(top_words(candidates, number_of_predictions))
demo = gr.Interface(
fn=predict,
inputs=[
gr.Textbox(label="Input text", placeholder="Write Swedish text..."),
gr.Slider(1, 10, value=5, step=1, label="Number of predictions"),
],
outputs=gr.Textbox(label="Predicted next words"),
title="Selma Lagerlöf n-gram predictor",
description="Trigram next-word prediction with bigram and unigram backoff.",
)
if __name__ == "__main__":
demo.launch()