"""Nano language detector (EDAN20, Lab 4) -- inference with the scikit-learn and PyTorch models. Usage: python predict.py "Hejsan grabbar!" "Hello guys!" """ import sys import hashlib from collections import Counter import joblib import torch import torch.nn as nn from huggingface_hub import hf_hub_download REPO_ID = "Mauro1455/Lab4" MAXES = [521, 1031, 1031] # hash buckets for 1-, 2-, 3-grams MAX_SHIFT = [0, 521, 1552] # offsets so the three blocks do not overlap # ---------- Feature extraction (identical to the training notebook) ---------- def ngrams(sentence, n=1, lc=True): if lc: sentence = sentence.lower() return [sentence[i:i + n] for i in range(len(sentence) - n + 1)] def all_ngrams(sentence, max_ngram=3, lc=True): return [ngrams(sentence, n=i, lc=lc) for i in range(1, max_ngram + 1)] def reproducible_hash(string): h = hashlib.md5(string.encode("utf-8"), usedforsecurity=False) return int.from_bytes(h.digest()[0:8], 'big', signed=True) def hash_ngrams(ngram_lists, modulos): return [[reproducible_hash(ng) % m for ng in ngram_list] for ngram_list, m in zip(ngram_lists, modulos)] def calc_rel_freq(codes): cnt = Counter(codes) total = len(codes) for code in cnt: cnt[code] /= total return cnt def shift_keys(dicts, shifts): return {key + shift: value for d, shift in zip(dicts, shifts) for key, value in d.items()} def build_freq_dict(sentence): hngrams = hash_ngrams(all_ngrams(sentence), MAXES) return shift_keys(map(calc_rel_freq, hngrams), MAX_SHIFT) # ---------- Load the models ---------- def load(filename): return hf_hub_download(repo_id=REPO_ID, filename=filename) clf = joblib.load(load("nld.joblib")) vectorizer = joblib.load(load("nld_vectorizer.joblib")) idx2lang = joblib.load(load("nld_lang_codes.joblib")) model = nn.Sequential( nn.Linear(len(vectorizer.feature_names_), 50), nn.ReLU(), nn.Linear(50, len(idx2lang)), ) model.load_state_dict(torch.load(load("nld.pth"), map_location="cpu")) model.eval() # ---------- Predict ---------- def predict(sentences): X = vectorizer.transform([build_freq_dict(s) for s in sentences]) pred_sklearn = [idx2lang[i] for i in clf.predict(X)] with torch.no_grad(): proba = torch.softmax(model(torch.Tensor(X)), dim=-1) pred_pytorch = [idx2lang[i.item()] for i in torch.argmax(proba, dim=-1)] return pred_sklearn, pred_pytorch, proba if __name__ == "__main__": sentences = sys.argv[1:] or ["Salut les gars !", "Hejsan grabbar!", "Hello guys!", "Hejsan tjejer!"] pred_sklearn, pred_pytorch, proba = predict(sentences) for s, p_sk, p_pt, p in zip(sentences, pred_sklearn, pred_pytorch, proba): print(f"{s!r:25} sklearn: {p_sk} PyTorch: {p_pt} ({p.max().item():.2f})")