Austinyu7's picture
Add trained language detector models
eb587ad verified
Raw History Blame Contribute Delete
2.01 kB
"""Run the scikit-learn and PyTorch Assignment 4 language detectors."""
from __future__ import annotations
import hashlib
import sys
from collections import Counter
import joblib
import torch
from torch import nn
MAXES = [521, 1031, 1031]
SHIFTS = [0, 521, 1552]
def reproducible_hash(text: str) -> int:
digest = hashlib.md5(text.encode("utf-8"), usedforsecurity=False).digest()
return int.from_bytes(digest[:8], "big", signed=True)
def features(text: str) -> dict[int, float]:
text = text.lower()
result: dict[int, float] = {}
for n, (modulo, shift) in enumerate(zip(MAXES, SHIFTS), start=1):
grams = [text[i : i + n] for i in range(len(text) - n + 1)]
if not grams:
continue
codes = Counter(reproducible_hash(gram) % modulo for gram in grams)
result.update({code + shift: count / len(grams) for code, count in codes.items()})
return result
def load_models():
sklearn_model = joblib.load("nld.joblib")
vectorizer = joblib.load("nld_vectorizer.joblib")
idx2lang = joblib.load("nld_lang_codes.joblib")
pytorch_model = nn.Sequential(nn.Linear(2583, 50), nn.ReLU(), nn.Linear(50, 8))
pytorch_model.load_state_dict(torch.load("nld.pth", map_location="cpu", weights_only=True))
pytorch_model.eval()
return sklearn_model, vectorizer, idx2lang, pytorch_model
def predict(text: str) -> tuple[str, str]:
sklearn_model, vectorizer, idx2lang, pytorch_model = load_models()
vector = vectorizer.transform([features(text)])
sklearn_label = idx2lang[int(sklearn_model.predict(vector)[0])]
with torch.no_grad():
pytorch_label = idx2lang[int(pytorch_model(torch.tensor(vector)).argmax(dim=1)[0])]
return sklearn_label, pytorch_label
if __name__ == "__main__":
sentence = " ".join(sys.argv[1:]) or "Bonjour tout le monde"
sklearn_prediction, pytorch_prediction = predict(sentence)
print(f"scikit-learn: {sklearn_prediction}")
print(f"PyTorch: {pytorch_prediction}")