import json import regex as re import gradio as gr from huggingface_hub import hf_hub_download REPO_ID = "IsacEP/selma_text" def load_ngrams_jsonl(filename): path = hf_hub_download( repo_id=REPO_ID, filename=filename, repo_type="dataset", ) result = {} with open(path, "r", encoding="utf-8") as f: for line in f: if not line.strip(): continue data = json.loads(line) key = tuple(data["ngram"]) if len(data["ngram"]) > 1 else data["ngram"][0] result[key] = data["count"] return result unigrams = load_ngrams_jsonl("unigrams.jsonl") bigrams = load_ngrams_jsonl("bigrams.jsonl") trigrams = load_ngrams_jsonl("trigrams.jsonl") def tokenize(text): return re.findall(r"\p{L}+", text.lower()) def top_words(candidates, n): candidates.sort(key=lambda x: (-x[1], x[0])) return [word for word, _ in candidates[:n]] def predict(text, number_of_predictions=5): tokens = tokenize(text) # Trigram prediction when two previous words are available. context = [""] + tokens if len(context) >= 2: prev2, prev1 = context[-2], context[-1] candidates = [ (ngram[2], count) for ngram, count in trigrams.items() if ngram[0] == prev2 and ngram[1] == prev1 ] words = top_words(candidates, number_of_predictions) if words: return ", ".join(words) # Bigram backoff. if context: prev = context[-1] candidates = [ (ngram[1], count) for ngram, count in bigrams.items() if ngram[0] == prev ] words = top_words(candidates, number_of_predictions) if words: return ", ".join(words) # Unigram backoff. candidates = [ (word, count) for word, count in unigrams.items() if word not in {"", ""} ] return ", ".join(top_words(candidates, number_of_predictions)) demo = gr.Interface( fn=predict, inputs=[ gr.Textbox(label="Input text", placeholder="Write Swedish text..."), gr.Slider(1, 10, value=5, step=1, label="Number of predictions"), ], outputs=gr.Textbox(label="Predicted next words"), title="Selma Lagerlöf n-gram predictor", description="Trigram next-word prediction with bigram and unigram backoff.", ) if __name__ == "__main__": demo.launch()