"""Run the scikit-learn and PyTorch Assignment 4 language detectors.""" from __future__ import annotations import hashlib import sys from collections import Counter import joblib import torch from torch import nn MAXES = [521, 1031, 1031] SHIFTS = [0, 521, 1552] def reproducible_hash(text: str) -> int: digest = hashlib.md5(text.encode("utf-8"), usedforsecurity=False).digest() return int.from_bytes(digest[:8], "big", signed=True) def features(text: str) -> dict[int, float]: text = text.lower() result: dict[int, float] = {} for n, (modulo, shift) in enumerate(zip(MAXES, SHIFTS), start=1): grams = [text[i : i + n] for i in range(len(text) - n + 1)] if not grams: continue codes = Counter(reproducible_hash(gram) % modulo for gram in grams) result.update({code + shift: count / len(grams) for code, count in codes.items()}) return result def load_models(): sklearn_model = joblib.load("nld.joblib") vectorizer = joblib.load("nld_vectorizer.joblib") idx2lang = joblib.load("nld_lang_codes.joblib") pytorch_model = nn.Sequential(nn.Linear(2583, 50), nn.ReLU(), nn.Linear(50, 8)) pytorch_model.load_state_dict(torch.load("nld.pth", map_location="cpu", weights_only=True)) pytorch_model.eval() return sklearn_model, vectorizer, idx2lang, pytorch_model def predict(text: str) -> tuple[str, str]: sklearn_model, vectorizer, idx2lang, pytorch_model = load_models() vector = vectorizer.transform([features(text)]) sklearn_label = idx2lang[int(sklearn_model.predict(vector)[0])] with torch.no_grad(): pytorch_label = idx2lang[int(pytorch_model(torch.tensor(vector)).argmax(dim=1)[0])] return sklearn_label, pytorch_label if __name__ == "__main__": sentence = " ".join(sys.argv[1:]) or "Bonjour tout le monde" sklearn_prediction, pytorch_prediction = predict(sentence) print(f"scikit-learn: {sklearn_prediction}") print(f"PyTorch: {pytorch_prediction}")