Spaces:
Sleeping
Sleeping
Download translator.py from ME22B191/Neural_Translation: direct link, hf CLI and curl.
- Browser
- Download file 2.16 kB
-
https://huggingface.co/spaces/ME22B191/Neural_Translation/resolve/main/translator.py
- Command line
-
hf download hf://spaces/ME22B191/Neural_Translation/translator.py
-
curl -L -o translator.py https://huggingface.co/spaces/ME22B191/Neural_Translation/resolve/main/translator.py
2.16 kB
| """ | |
| translator.py — loads your already-uploaded checkpoint plus the small | |
| vocab.json and assembles the model for serving. Imports NO training | |
| libraries: no datasets, no gdown. | |
| """ | |
| import json | |
| import spacy | |
| import torch | |
| from huggingface_hub import hf_hub_download | |
| from model import Transformer # safe: model.py imports only torch at the top | |
| UNK_IDX = 0 | |
| class _Consts: | |
| """Stand-in for the dataset module that infer() reads token indices from, | |
| so we never import dataset.py (which would pull in `datasets`).""" | |
| UNK_IDX = 0 | |
| PAD_IDX = 1 | |
| SOS_IDX = 2 | |
| EOS_IDX = 3 | |
| class Vocab: | |
| """Minimal copy of dataset.Vocab — only the methods infer() calls. | |
| Rebuilt from the saved itos list; stoi is its inverse.""" | |
| def __init__(self, itos): | |
| self.itos = itos | |
| self.stoi = {tok: i for i, tok in enumerate(itos)} | |
| def encode(self, tokens): | |
| return [self.stoi.get(t, UNK_IDX) for t in tokens] | |
| def lookup_token(self, idx): | |
| return self.itos[idx] | |
| def load_translator(repo_id: str, | |
| checkpoint_file: str = "checkpoint.pt", | |
| vocab_file: str = "vocab.json", | |
| device: str = "cpu"): | |
| """Download weights + vocab from your model repo and assemble the model.""" | |
| ckpt_path = hf_hub_download(repo_id=repo_id, filename=checkpoint_file) | |
| vocab_path = hf_hub_download(repo_id=repo_id, filename=vocab_file) | |
| # Weights + config come straight from the checkpoint you already uploaded. | |
| # Real vocab sizes (not 1) mean __init__ does NOT auto-trigger the old | |
| # gdown/datasets path. | |
| ckpt = torch.load(ckpt_path, map_location=device) | |
| model = Transformer(**ckpt["model_config"]) | |
| model.load_state_dict(ckpt["model_state_dict"]) | |
| model.to(device) | |
| model.eval() | |
| # Vocab from JSON; tokenizer + token-index constants are built in. | |
| with open(vocab_path, encoding="utf-8") as f: | |
| vocab = json.load(f) | |
| model.src_vocab = Vocab(vocab["src_itos"]) | |
| model.tgt_vocab = Vocab(vocab["tgt_itos"]) | |
| model._nlp_de = spacy.blank("de") # tokenizer-only: no model download | |
| model._ds = _Consts() | |
| return model |