Spaces:
Running
Running
Download app.py from IsacEP/edan20-lab2: direct link, hf CLI and curl.
- Browser
- Download file 2.45 kB
-
https://huggingface.co/spaces/IsacEP/edan20-lab2/resolve/main/app.py
- Command line
-
hf download hf://spaces/IsacEP/edan20-lab2/app.py
-
curl -L -o app.py https://huggingface.co/spaces/IsacEP/edan20-lab2/resolve/main/app.py
2.45 kB
| import json | |
| import regex as re | |
| import gradio as gr | |
| from huggingface_hub import hf_hub_download | |
| REPO_ID = "IsacEP/selma_text" | |
| def load_ngrams_jsonl(filename): | |
| path = hf_hub_download( | |
| repo_id=REPO_ID, | |
| filename=filename, | |
| repo_type="dataset", | |
| ) | |
| result = {} | |
| with open(path, "r", encoding="utf-8") as f: | |
| for line in f: | |
| if not line.strip(): | |
| continue | |
| data = json.loads(line) | |
| key = tuple(data["ngram"]) if len(data["ngram"]) > 1 else data["ngram"][0] | |
| result[key] = data["count"] | |
| return result | |
| unigrams = load_ngrams_jsonl("unigrams.jsonl") | |
| bigrams = load_ngrams_jsonl("bigrams.jsonl") | |
| trigrams = load_ngrams_jsonl("trigrams.jsonl") | |
| def tokenize(text): | |
| return re.findall(r"\p{L}+", text.lower()) | |
| def top_words(candidates, n): | |
| candidates.sort(key=lambda x: (-x[1], x[0])) | |
| return [word for word, _ in candidates[:n]] | |
| def predict(text, number_of_predictions=5): | |
| tokens = tokenize(text) | |
| # Trigram prediction when two previous words are available. | |
| context = ["<s>"] + tokens | |
| if len(context) >= 2: | |
| prev2, prev1 = context[-2], context[-1] | |
| candidates = [ | |
| (ngram[2], count) | |
| for ngram, count in trigrams.items() | |
| if ngram[0] == prev2 and ngram[1] == prev1 | |
| ] | |
| words = top_words(candidates, number_of_predictions) | |
| if words: | |
| return ", ".join(words) | |
| # Bigram backoff. | |
| if context: | |
| prev = context[-1] | |
| candidates = [ | |
| (ngram[1], count) | |
| for ngram, count in bigrams.items() | |
| if ngram[0] == prev | |
| ] | |
| words = top_words(candidates, number_of_predictions) | |
| if words: | |
| return ", ".join(words) | |
| # Unigram backoff. | |
| candidates = [ | |
| (word, count) | |
| for word, count in unigrams.items() | |
| if word not in {"<s>", "</s>"} | |
| ] | |
| return ", ".join(top_words(candidates, number_of_predictions)) | |
| demo = gr.Interface( | |
| fn=predict, | |
| inputs=[ | |
| gr.Textbox(label="Input text", placeholder="Write Swedish text..."), | |
| gr.Slider(1, 10, value=5, step=1, label="Number of predictions"), | |
| ], | |
| outputs=gr.Textbox(label="Predicted next words"), | |
| title="Selma Lagerlöf n-gram predictor", | |
| description="Trigram next-word prediction with bigram and unigram backoff.", | |
| ) | |
| if __name__ == "__main__": | |
| demo.launch() | |