File size: 2,010 Bytes
eb587ad
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
"""Run the scikit-learn and PyTorch Assignment 4 language detectors."""

from __future__ import annotations

import hashlib
import sys
from collections import Counter

import joblib
import torch
from torch import nn


MAXES = [521, 1031, 1031]
SHIFTS = [0, 521, 1552]


def reproducible_hash(text: str) -> int:
    digest = hashlib.md5(text.encode("utf-8"), usedforsecurity=False).digest()
    return int.from_bytes(digest[:8], "big", signed=True)


def features(text: str) -> dict[int, float]:
    text = text.lower()
    result: dict[int, float] = {}
    for n, (modulo, shift) in enumerate(zip(MAXES, SHIFTS), start=1):
        grams = [text[i : i + n] for i in range(len(text) - n + 1)]
        if not grams:
            continue
        codes = Counter(reproducible_hash(gram) % modulo for gram in grams)
        result.update({code + shift: count / len(grams) for code, count in codes.items()})
    return result


def load_models():
    sklearn_model = joblib.load("nld.joblib")
    vectorizer = joblib.load("nld_vectorizer.joblib")
    idx2lang = joblib.load("nld_lang_codes.joblib")
    pytorch_model = nn.Sequential(nn.Linear(2583, 50), nn.ReLU(), nn.Linear(50, 8))
    pytorch_model.load_state_dict(torch.load("nld.pth", map_location="cpu", weights_only=True))
    pytorch_model.eval()
    return sklearn_model, vectorizer, idx2lang, pytorch_model


def predict(text: str) -> tuple[str, str]:
    sklearn_model, vectorizer, idx2lang, pytorch_model = load_models()
    vector = vectorizer.transform([features(text)])
    sklearn_label = idx2lang[int(sklearn_model.predict(vector)[0])]
    with torch.no_grad():
        pytorch_label = idx2lang[int(pytorch_model(torch.tensor(vector)).argmax(dim=1)[0])]
    return sklearn_label, pytorch_label


if __name__ == "__main__":
    sentence = " ".join(sys.argv[1:]) or "Bonjour tout le monde"
    sklearn_prediction, pytorch_prediction = predict(sentence)
    print(f"scikit-learn: {sklearn_prediction}")
    print(f"PyTorch:       {pytorch_prediction}")