File size: 6,569 Bytes
2d52135
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2491489
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2d52135
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2491489
2d52135
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
"""Programmatic inference helper for mini_transformer."""

from __future__ import annotations

import warnings
from pathlib import Path

import torch
from hydra.utils import to_absolute_path
from omegaconf import DictConfig, OmegaConf
from transformers import PreTrainedTokenizerFast

from .configs import InferAppCfg, ModelCfg, TokenizerCfg
from .transformer import BasicEncoderDecoderTransformer
from .utils import check_tokenizer_model_compatibility, set_global_seed


def _resolve_device(device_str: str | None) -> torch.device:
    """Pick a device honoring configs while falling back to CPU if unavailable."""

    if not device_str:
        return torch.device("cpu")
    try:
        device = torch.device(device_str)
    except (TypeError, ValueError) as exc:
        raise ValueError(f"Invalid runtime.device value: {device_str!r}") from exc

    if device.type == "cuda":
        if not torch.cuda.is_available():
            warnings.warn(
                "CUDA device requested but torch.cuda.is_available() is False; falling back to CPU.",
                RuntimeWarning,
                stacklevel=2,
            )
            return torch.device("cpu")
        if device.index is not None and device.index >= torch.cuda.device_count():
            warnings.warn(
                f"CUDA device index {device.index} out of range; using cuda:0 instead.",
                RuntimeWarning,
                stacklevel=2,
            )
            return torch.device("cuda:0")
    return device


_SESSION_CACHE: dict[
    tuple[str, ...], tuple[BasicEncoderDecoderTransformer, PreTrainedTokenizerFast]
] = {}


def _session_cache_key(
    model_cfg: ModelCfg,
    tokenizer_cfg: TokenizerCfg,
    tokenizer_path: Path,
    checkpoint_path: Path | None,
    device: torch.device,
) -> tuple[str, ...]:
    special_tokens = ",".join(tokenizer_cfg.special_tokens) if tokenizer_cfg.special_tokens else ""
    return (
        model_cfg.name,
        str(tokenizer_path),
        str(checkpoint_path) if checkpoint_path else "",
        str(device),
        str(model_cfg.vocab_size),
        str(model_cfg.d_model),
        str(model_cfg.num_heads),
        str(model_cfg.num_layers),
        str(model_cfg.d_ff),
        f"{model_cfg.dropout_rate}",
        str(model_cfg.max_seq_len),
        str(model_cfg.pad_id),
        str(model_cfg.bos_id),
        str(model_cfg.eos_id),
        tokenizer_cfg.name,
        str(tokenizer_cfg.vocab_size),
        str(tokenizer_cfg.max_seq_len),
        tokenizer_cfg.pad_token or "",
        tokenizer_cfg.bos_token or "",
        tokenizer_cfg.eos_token or "",
        tokenizer_cfg.unk_token or "",
        special_tokens,
    )


def _get_or_create_session(
    *,
    model_cfg: ModelCfg,
    tokenizer_cfg: TokenizerCfg,
    tokenizer_path: Path,
    checkpoint_path: Path | None,
    device: torch.device,
) -> tuple[BasicEncoderDecoderTransformer, PreTrainedTokenizerFast]:
    key = _session_cache_key(model_cfg, tokenizer_cfg, tokenizer_path, checkpoint_path, device)
    if key in _SESSION_CACHE:
        return _SESSION_CACHE[key]

    transformer = BasicEncoderDecoderTransformer(model_cfg)
    if checkpoint_path:
        ckpt = torch.load(checkpoint_path, map_location=device)
        state = ckpt.get("model_state_dict", ckpt)
        transformer.load_state_dict(state, strict=True)
    transformer.to(device)
    transformer.eval()

    tokenizer = PreTrainedTokenizerFast(
        tokenizer_file=str(tokenizer_path),
        bos_token=tokenizer_cfg.bos_token,
        eos_token=tokenizer_cfg.eos_token,
        unk_token=tokenizer_cfg.unk_token,
        pad_token=tokenizer_cfg.pad_token,
        model_max_length=tokenizer_cfg.max_seq_len,
    )

    _SESSION_CACHE[key] = (transformer, tokenizer)
    return transformer, tokenizer


def _preprocess_text(text: str) -> str:
    """Lowercase and ensure sentence ends with punctuation.

    Simple preprocessing used before tokenization during inference:
    - Lowercase the entire input.
    - If the last non-space character is not one of '.!?', append a period.
    """
    s = text.strip()
    if not s:
        return s
    s = s.lower()
    last = s[-1]
    if last not in ".!?":
        s += "."
    return s


def run_inference(cfg: DictConfig) -> list[str]:
    """Run inference using a composed Hydra configuration."""

    scfg_temp = OmegaConf.merge(OmegaConf.structured(InferAppCfg), cfg)
    scfg: InferAppCfg = OmegaConf.to_object(scfg_temp)
    model_cfg = ModelCfg(**vars(scfg.model))
    tokenizer_cfg = TokenizerCfg(**vars(scfg.tokenizer))

    set_global_seed(scfg.runtime.seed)
    check_tokenizer_model_compatibility(model_cfg, tokenizer_cfg)

    device = _resolve_device(getattr(scfg.runtime, "device", None))

    if not tokenizer_cfg.path:
        raise FileNotFoundError(
            "Tokenizer path is empty. Update your config or set MINI_TRANSFORMER_TOKENIZER_PATH."
        )

    tokenizer_path = Path(to_absolute_path(tokenizer_cfg.path))
    if not tokenizer_path.is_file():
        raise FileNotFoundError(
            f"Tokenizer file not found: {tokenizer_path}. Check your model's config or environment variables."
        )

    checkpoint_path: Path | None = None
    if scfg.model.best_checkpoint_path:
        checkpoint_path = Path(to_absolute_path(scfg.model.best_checkpoint_path))

    transformer, tokenizer = _get_or_create_session(
        model_cfg=model_cfg,
        tokenizer_cfg=tokenizer_cfg,
        tokenizer_path=tokenizer_path,
        checkpoint_path=checkpoint_path,
        device=device,
    )

    text_input = _preprocess_text(scfg.input_text)
    if not text_input:
        raise SystemExit('Pass text like: input_text="hello world"')

    encoded = tokenizer(text_input, padding=True, truncation=True, return_tensors="pt")
    src_ids = encoded["input_ids"].to(device)
    src_padd_mask = (encoded["attention_mask"] == 0).to(device)

    tgt_ids = transformer.generate(
        src_ids,
        src_padd_mask,
        max_new_tokens=scfg.generation.max_new_tokens,
        temperature=scfg.generation.temperature,
        top_k=scfg.generation.top_k,
        top_p=scfg.generation.top_p,
        do_sample=scfg.generation.do_sample,
        presence_penalty=scfg.generation.presence_penalty,
        frequency_penalty=scfg.generation.frequency_penalty,
        no_repeat_ngram=scfg.generation.no_repeat_ngram,
        min_steps_before_eos=scfg.generation.min_steps_before_eos,
        seed=scfg.runtime.seed,
    )
    return tokenizer.batch_decode(tgt_ids.cpu(), skip_special_tokens=True)