Download predict.py from Austinyu7/edan20-assignment4-language-detector: direct link, hf CLI and curl.
- Browser
- Download file 2.01 kB
-
https://huggingface.co/Austinyu7/edan20-assignment4-language-detector/resolve/main/predict.py
- Command line
-
hf download hf://Austinyu7/edan20-assignment4-language-detector/predict.py
-
curl -L -o predict.py https://huggingface.co/Austinyu7/edan20-assignment4-language-detector/resolve/main/predict.py
2.01 kB
| """Run the scikit-learn and PyTorch Assignment 4 language detectors.""" | |
| from __future__ import annotations | |
| import hashlib | |
| import sys | |
| from collections import Counter | |
| import joblib | |
| import torch | |
| from torch import nn | |
| MAXES = [521, 1031, 1031] | |
| SHIFTS = [0, 521, 1552] | |
| def reproducible_hash(text: str) -> int: | |
| digest = hashlib.md5(text.encode("utf-8"), usedforsecurity=False).digest() | |
| return int.from_bytes(digest[:8], "big", signed=True) | |
| def features(text: str) -> dict[int, float]: | |
| text = text.lower() | |
| result: dict[int, float] = {} | |
| for n, (modulo, shift) in enumerate(zip(MAXES, SHIFTS), start=1): | |
| grams = [text[i : i + n] for i in range(len(text) - n + 1)] | |
| if not grams: | |
| continue | |
| codes = Counter(reproducible_hash(gram) % modulo for gram in grams) | |
| result.update({code + shift: count / len(grams) for code, count in codes.items()}) | |
| return result | |
| def load_models(): | |
| sklearn_model = joblib.load("nld.joblib") | |
| vectorizer = joblib.load("nld_vectorizer.joblib") | |
| idx2lang = joblib.load("nld_lang_codes.joblib") | |
| pytorch_model = nn.Sequential(nn.Linear(2583, 50), nn.ReLU(), nn.Linear(50, 8)) | |
| pytorch_model.load_state_dict(torch.load("nld.pth", map_location="cpu", weights_only=True)) | |
| pytorch_model.eval() | |
| return sklearn_model, vectorizer, idx2lang, pytorch_model | |
| def predict(text: str) -> tuple[str, str]: | |
| sklearn_model, vectorizer, idx2lang, pytorch_model = load_models() | |
| vector = vectorizer.transform([features(text)]) | |
| sklearn_label = idx2lang[int(sklearn_model.predict(vector)[0])] | |
| with torch.no_grad(): | |
| pytorch_label = idx2lang[int(pytorch_model(torch.tensor(vector)).argmax(dim=1)[0])] | |
| return sklearn_label, pytorch_label | |
| if __name__ == "__main__": | |
| sentence = " ".join(sys.argv[1:]) or "Bonjour tout le monde" | |
| sklearn_prediction, pytorch_prediction = predict(sentence) | |
| print(f"scikit-learn: {sklearn_prediction}") | |
| print(f"PyTorch: {pytorch_prediction}") | |