File size: 2,870 Bytes
a9363f7
 
 
 
 
 
 
 
 
 
 
 
 
 
d5afbaf
a9363f7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
"""Nano language detector (EDAN20, Lab 4) -- inference with the scikit-learn and PyTorch models.

Usage:
    python predict.py "Hejsan grabbar!" "Hello guys!"
"""
import sys
import hashlib
from collections import Counter

import joblib
import torch
import torch.nn as nn
from huggingface_hub import hf_hub_download

REPO_ID = "Mauro1455/Lab4"

MAXES = [521, 1031, 1031]                  # hash buckets for 1-, 2-, 3-grams
MAX_SHIFT = [0, 521, 1552]                 # offsets so the three blocks do not overlap


# ---------- Feature extraction (identical to the training notebook) ----------
def ngrams(sentence, n=1, lc=True):
    if lc:
        sentence = sentence.lower()
    return [sentence[i:i + n] for i in range(len(sentence) - n + 1)]


def all_ngrams(sentence, max_ngram=3, lc=True):
    return [ngrams(sentence, n=i, lc=lc) for i in range(1, max_ngram + 1)]


def reproducible_hash(string):
    h = hashlib.md5(string.encode("utf-8"), usedforsecurity=False)
    return int.from_bytes(h.digest()[0:8], 'big', signed=True)


def hash_ngrams(ngram_lists, modulos):
    return [[reproducible_hash(ng) % m for ng in ngram_list]
            for ngram_list, m in zip(ngram_lists, modulos)]


def calc_rel_freq(codes):
    cnt = Counter(codes)
    total = len(codes)
    for code in cnt:
        cnt[code] /= total
    return cnt


def shift_keys(dicts, shifts):
    return {key + shift: value
            for d, shift in zip(dicts, shifts)
            for key, value in d.items()}


def build_freq_dict(sentence):
    hngrams = hash_ngrams(all_ngrams(sentence), MAXES)
    return shift_keys(map(calc_rel_freq, hngrams), MAX_SHIFT)


# ---------- Load the models ----------
def load(filename):
    return hf_hub_download(repo_id=REPO_ID, filename=filename)


clf = joblib.load(load("nld.joblib"))
vectorizer = joblib.load(load("nld_vectorizer.joblib"))
idx2lang = joblib.load(load("nld_lang_codes.joblib"))

model = nn.Sequential(
    nn.Linear(len(vectorizer.feature_names_), 50),
    nn.ReLU(),
    nn.Linear(50, len(idx2lang)),
)
model.load_state_dict(torch.load(load("nld.pth"), map_location="cpu"))
model.eval()


# ---------- Predict ----------
def predict(sentences):
    X = vectorizer.transform([build_freq_dict(s) for s in sentences])
    pred_sklearn = [idx2lang[i] for i in clf.predict(X)]
    with torch.no_grad():
        proba = torch.softmax(model(torch.Tensor(X)), dim=-1)
    pred_pytorch = [idx2lang[i.item()] for i in torch.argmax(proba, dim=-1)]
    return pred_sklearn, pred_pytorch, proba


if __name__ == "__main__":
    sentences = sys.argv[1:] or ["Salut les gars !", "Hejsan grabbar!", "Hello guys!", "Hejsan tjejer!"]
    pred_sklearn, pred_pytorch, proba = predict(sentences)
    for s, p_sk, p_pt, p in zip(sentences, pred_sklearn, pred_pytorch, proba):
        print(f"{s!r:25} sklearn: {p_sk}  PyTorch: {p_pt} ({p.max().item():.2f})")