Download predict.py from Mauro1455/Lab4: direct link, hf CLI and curl.
- Browser
- Download file 2.87 kB
-
https://huggingface.co/Mauro1455/Lab4/resolve/main/predict.py
- Command line
-
hf download hf://Mauro1455/Lab4/predict.py
-
curl -L -o predict.py https://huggingface.co/Mauro1455/Lab4/resolve/main/predict.py
2.87 kB
| """Nano language detector (EDAN20, Lab 4) -- inference with the scikit-learn and PyTorch models. | |
| Usage: | |
| python predict.py "Hejsan grabbar!" "Hello guys!" | |
| """ | |
| import sys | |
| import hashlib | |
| from collections import Counter | |
| import joblib | |
| import torch | |
| import torch.nn as nn | |
| from huggingface_hub import hf_hub_download | |
| REPO_ID = "Mauro1455/Lab4" | |
| MAXES = [521, 1031, 1031] # hash buckets for 1-, 2-, 3-grams | |
| MAX_SHIFT = [0, 521, 1552] # offsets so the three blocks do not overlap | |
| # ---------- Feature extraction (identical to the training notebook) ---------- | |
| def ngrams(sentence, n=1, lc=True): | |
| if lc: | |
| sentence = sentence.lower() | |
| return [sentence[i:i + n] for i in range(len(sentence) - n + 1)] | |
| def all_ngrams(sentence, max_ngram=3, lc=True): | |
| return [ngrams(sentence, n=i, lc=lc) for i in range(1, max_ngram + 1)] | |
| def reproducible_hash(string): | |
| h = hashlib.md5(string.encode("utf-8"), usedforsecurity=False) | |
| return int.from_bytes(h.digest()[0:8], 'big', signed=True) | |
| def hash_ngrams(ngram_lists, modulos): | |
| return [[reproducible_hash(ng) % m for ng in ngram_list] | |
| for ngram_list, m in zip(ngram_lists, modulos)] | |
| def calc_rel_freq(codes): | |
| cnt = Counter(codes) | |
| total = len(codes) | |
| for code in cnt: | |
| cnt[code] /= total | |
| return cnt | |
| def shift_keys(dicts, shifts): | |
| return {key + shift: value | |
| for d, shift in zip(dicts, shifts) | |
| for key, value in d.items()} | |
| def build_freq_dict(sentence): | |
| hngrams = hash_ngrams(all_ngrams(sentence), MAXES) | |
| return shift_keys(map(calc_rel_freq, hngrams), MAX_SHIFT) | |
| # ---------- Load the models ---------- | |
| def load(filename): | |
| return hf_hub_download(repo_id=REPO_ID, filename=filename) | |
| clf = joblib.load(load("nld.joblib")) | |
| vectorizer = joblib.load(load("nld_vectorizer.joblib")) | |
| idx2lang = joblib.load(load("nld_lang_codes.joblib")) | |
| model = nn.Sequential( | |
| nn.Linear(len(vectorizer.feature_names_), 50), | |
| nn.ReLU(), | |
| nn.Linear(50, len(idx2lang)), | |
| ) | |
| model.load_state_dict(torch.load(load("nld.pth"), map_location="cpu")) | |
| model.eval() | |
| # ---------- Predict ---------- | |
| def predict(sentences): | |
| X = vectorizer.transform([build_freq_dict(s) for s in sentences]) | |
| pred_sklearn = [idx2lang[i] for i in clf.predict(X)] | |
| with torch.no_grad(): | |
| proba = torch.softmax(model(torch.Tensor(X)), dim=-1) | |
| pred_pytorch = [idx2lang[i.item()] for i in torch.argmax(proba, dim=-1)] | |
| return pred_sklearn, pred_pytorch, proba | |
| if __name__ == "__main__": | |
| sentences = sys.argv[1:] or ["Salut les gars !", "Hejsan grabbar!", "Hello guys!", "Hejsan tjejer!"] | |
| pred_sklearn, pred_pytorch, proba = predict(sentences) | |
| for s, p_sk, p_pt, p in zip(sentences, pred_sklearn, pred_pytorch, proba): | |
| print(f"{s!r:25} sklearn: {p_sk} PyTorch: {p_pt} ({p.max().item():.2f})") | |