Lab4 / predict.py
Mauro1455's picture
Update predict.py
d5afbaf verified
Raw History Blame Contribute Delete
2.87 kB
"""Nano language detector (EDAN20, Lab 4) -- inference with the scikit-learn and PyTorch models.
Usage:
python predict.py "Hejsan grabbar!" "Hello guys!"
"""
import sys
import hashlib
from collections import Counter
import joblib
import torch
import torch.nn as nn
from huggingface_hub import hf_hub_download
REPO_ID = "Mauro1455/Lab4"
MAXES = [521, 1031, 1031] # hash buckets for 1-, 2-, 3-grams
MAX_SHIFT = [0, 521, 1552] # offsets so the three blocks do not overlap
# ---------- Feature extraction (identical to the training notebook) ----------
def ngrams(sentence, n=1, lc=True):
if lc:
sentence = sentence.lower()
return [sentence[i:i + n] for i in range(len(sentence) - n + 1)]
def all_ngrams(sentence, max_ngram=3, lc=True):
return [ngrams(sentence, n=i, lc=lc) for i in range(1, max_ngram + 1)]
def reproducible_hash(string):
h = hashlib.md5(string.encode("utf-8"), usedforsecurity=False)
return int.from_bytes(h.digest()[0:8], 'big', signed=True)
def hash_ngrams(ngram_lists, modulos):
return [[reproducible_hash(ng) % m for ng in ngram_list]
for ngram_list, m in zip(ngram_lists, modulos)]
def calc_rel_freq(codes):
cnt = Counter(codes)
total = len(codes)
for code in cnt:
cnt[code] /= total
return cnt
def shift_keys(dicts, shifts):
return {key + shift: value
for d, shift in zip(dicts, shifts)
for key, value in d.items()}
def build_freq_dict(sentence):
hngrams = hash_ngrams(all_ngrams(sentence), MAXES)
return shift_keys(map(calc_rel_freq, hngrams), MAX_SHIFT)
# ---------- Load the models ----------
def load(filename):
return hf_hub_download(repo_id=REPO_ID, filename=filename)
clf = joblib.load(load("nld.joblib"))
vectorizer = joblib.load(load("nld_vectorizer.joblib"))
idx2lang = joblib.load(load("nld_lang_codes.joblib"))
model = nn.Sequential(
nn.Linear(len(vectorizer.feature_names_), 50),
nn.ReLU(),
nn.Linear(50, len(idx2lang)),
)
model.load_state_dict(torch.load(load("nld.pth"), map_location="cpu"))
model.eval()
# ---------- Predict ----------
def predict(sentences):
X = vectorizer.transform([build_freq_dict(s) for s in sentences])
pred_sklearn = [idx2lang[i] for i in clf.predict(X)]
with torch.no_grad():
proba = torch.softmax(model(torch.Tensor(X)), dim=-1)
pred_pytorch = [idx2lang[i.item()] for i in torch.argmax(proba, dim=-1)]
return pred_sklearn, pred_pytorch, proba
if __name__ == "__main__":
sentences = sys.argv[1:] or ["Salut les gars !", "Hejsan grabbar!", "Hello guys!", "Hejsan tjejer!"]
pred_sklearn, pred_pytorch, proba = predict(sentences)
for s, p_sk, p_pt, p in zip(sentences, pred_sklearn, pred_pytorch, proba):
print(f"{s!r:25} sklearn: {p_sk} PyTorch: {p_pt} ({p.max().item():.2f})")