harshitv804 commited on
Commit
ced6373
·
verified ·
1 Parent(s): ee8a284

Upload 2 files

Browse files
Files changed (2) hide show
  1. onnx_inference.py +310 -0
  2. transliterate.py +158 -0
onnx_inference.py ADDED
@@ -0,0 +1,310 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """CPU-only ONNX inference for the transliteration model.
2
+
3
+ Uses beam search to decode with :class:`OnnxTransliterator`.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import json
9
+ from collections.abc import Sequence
10
+ from pathlib import Path
11
+
12
+ import numpy as np
13
+ import onnxruntime as ort
14
+ from tokenizers import Tokenizer
15
+
16
+ DEFAULT_BEAM_SIZE = 4
17
+ DEFAULT_LENGTH_PENALTY = 1.0
18
+
19
+ __all__ = ["OnnxTransliterator", "normalize_direction"]
20
+
21
+
22
+ def normalize_direction(direction: str) -> str:
23
+ """Normalize ``en-ta`` / ``<|en-ta|>`` to ``en-ta``."""
24
+ direction = direction.strip()
25
+ if direction.startswith("<|") and direction.endswith("|>"):
26
+ direction = direction[2:-2]
27
+ return direction
28
+
29
+
30
+ def _log_softmax(x: np.ndarray, axis: int = -1) -> np.ndarray:
31
+ shifted = x - np.max(x, axis=axis, keepdims=True)
32
+ return shifted - np.log(np.sum(np.exp(shifted), axis=axis, keepdims=True))
33
+
34
+
35
+ def _validate_beam_options(beam_size: int, length_penalty: float) -> None:
36
+ if beam_size < 1:
37
+ raise ValueError("beam_size must be >= 1")
38
+ if length_penalty < 0.0:
39
+ raise ValueError("length_penalty must be >= 0")
40
+
41
+
42
+ class OnnxTransliterator:
43
+ """Decoder around the exported ONNX encoder/decoder."""
44
+
45
+ def __init__(
46
+ self,
47
+ model_dir: str | Path,
48
+ *,
49
+ max_src_len: int | None = None,
50
+ max_new_tokens: int | None = None,
51
+ ):
52
+ self.model_dir = Path(model_dir)
53
+ config_path = self.model_dir / "config.json"
54
+ if not config_path.exists():
55
+ raise FileNotFoundError(
56
+ f"{config_path} not found; run export_onnx.py first"
57
+ )
58
+
59
+ self.meta = json.loads(config_path.read_text(encoding="utf-8"))
60
+ self.tokenizer = Tokenizer.from_file(str(self.model_dir / "tokenizer.json"))
61
+ self.tokenizer.no_padding()
62
+ self.tokenizer.no_truncation()
63
+
64
+ options = ort.SessionOptions()
65
+ options.log_severity_level = 3
66
+ providers = ["CPUExecutionProvider"]
67
+ self.encoder = ort.InferenceSession(
68
+ str(self.model_dir / "encoder.onnx"), options, providers=providers
69
+ )
70
+ self.decoder = ort.InferenceSession(
71
+ str(self.model_dir / "decoder.onnx"), options, providers=providers
72
+ )
73
+
74
+ self.direction_ids = {k: int(v) for k, v in self.meta["direction_ids"].items()}
75
+ specials = self.meta["special_tokens"]
76
+ self.bos_id = int(specials["bos"])
77
+ self.eos_id = int(specials["eos"])
78
+ self.pad_id = int(specials["pad"])
79
+
80
+ model_max_len = int(self.meta["model"]["max_seq_len"])
81
+ self.max_src_len = min(
82
+ max_src_len or int(self.meta["max_src_len"]), model_max_len
83
+ )
84
+ self.max_new_tokens = min(
85
+ max_new_tokens or int(self.meta["max_new_tokens"]), model_max_len - 1
86
+ )
87
+ self._suppress_ids = np.array(
88
+ [
89
+ token_id
90
+ for token, token_id in self.tokenizer.get_vocab().items()
91
+ if token.startswith("<|") and token_id != self.eos_id
92
+ ],
93
+ dtype=np.int64,
94
+ )
95
+
96
+ # ------------------------------------------------------------------ #
97
+ # Public API
98
+ # ------------------------------------------------------------------ #
99
+ def translate(
100
+ self,
101
+ text: str,
102
+ direction: str,
103
+ *,
104
+ beam_size: int = DEFAULT_BEAM_SIZE,
105
+ length_penalty: float = DEFAULT_LENGTH_PENALTY,
106
+ ) -> str:
107
+ return self.translate_batch(
108
+ [text],
109
+ direction,
110
+ beam_size=beam_size,
111
+ length_penalty=length_penalty,
112
+ )[0]
113
+
114
+ def translate_batch(
115
+ self,
116
+ texts: Sequence[str],
117
+ direction: str | Sequence[str],
118
+ *,
119
+ beam_size: int = DEFAULT_BEAM_SIZE,
120
+ length_penalty: float = DEFAULT_LENGTH_PENALTY,
121
+ ) -> list[str]:
122
+ _validate_beam_options(beam_size, length_penalty)
123
+ texts = list(texts)
124
+ if not texts:
125
+ return []
126
+
127
+ directions = self._expand_directions(len(texts), direction)
128
+ source_ids = [
129
+ self._encode_source(text, tag) for text, tag in zip(texts, directions)
130
+ ]
131
+ output_ids = self._generate_ids(
132
+ source_ids,
133
+ beam_size=beam_size,
134
+ length_penalty=length_penalty,
135
+ )
136
+ return [self._decode(ids) for ids in output_ids]
137
+
138
+ # ------------------------------------------------------------------ #
139
+ # Internals
140
+ # ------------------------------------------------------------------ #
141
+ def _expand_directions(
142
+ self, n: int, direction: str | Sequence[str]
143
+ ) -> list[str]:
144
+ if isinstance(direction, str):
145
+ directions = [normalize_direction(direction)] * n
146
+ else:
147
+ directions = [normalize_direction(d) for d in direction]
148
+ if len(directions) != n:
149
+ raise ValueError(
150
+ f"got {n} texts but {len(directions)} directions"
151
+ )
152
+
153
+ unknown = sorted({d for d in directions if d not in self.direction_ids})
154
+ if unknown:
155
+ raise ValueError(
156
+ f"unknown direction(s) {unknown}; expected "
157
+ f"{sorted(self.direction_ids)}"
158
+ )
159
+ return directions
160
+
161
+ def _encode_source(self, text: str, direction: str) -> list[int]:
162
+ body = self.tokenizer.encode(text).ids[: self.max_src_len - 2]
163
+ return [self.direction_ids[direction]] + body + [self.eos_id]
164
+
165
+ def _prepare_source(
166
+ self, source_ids: list[list[int]]
167
+ ) -> tuple[np.ndarray, np.ndarray]:
168
+ batch = len(source_ids)
169
+ width = max(len(ids) for ids in source_ids)
170
+
171
+ src_ids = np.full((batch, width), self.pad_id, dtype=np.int64)
172
+ src_mask = np.zeros((batch, width), dtype=np.bool_)
173
+ for row, ids in enumerate(source_ids):
174
+ src_ids[row, : len(ids)] = ids
175
+ src_mask[row, : len(ids)] = True
176
+ return src_ids, src_mask
177
+
178
+ def _suppress_specials(self, logits: np.ndarray) -> np.ndarray:
179
+ if self._suppress_ids.size == 0:
180
+ return logits
181
+ logits = logits.copy()
182
+ logits[..., self._suppress_ids] = -np.inf
183
+ return logits
184
+
185
+ def _generate_ids(
186
+ self,
187
+ source_ids: list[list[int]],
188
+ *,
189
+ beam_size: int = DEFAULT_BEAM_SIZE,
190
+ length_penalty: float = DEFAULT_LENGTH_PENALTY,
191
+ ) -> list[list[int]]:
192
+ src_ids, src_mask = self._prepare_source(source_ids)
193
+ memory = self.encoder.run(
194
+ None,
195
+ {"src_ids": src_ids, "src_mask": src_mask},
196
+ )[0]
197
+ return self._generate_beam(
198
+ memory,
199
+ src_mask,
200
+ beam_size=beam_size,
201
+ length_penalty=length_penalty,
202
+ )
203
+
204
+ def _generate_beam(
205
+ self,
206
+ memory: np.ndarray,
207
+ src_mask: np.ndarray,
208
+ *,
209
+ beam_size: int,
210
+ length_penalty: float,
211
+ ) -> list[list[int]]:
212
+ batch = memory.shape[0]
213
+ beams = beam_size
214
+
215
+ memory = np.repeat(memory, beams, axis=0)
216
+ cross_mask = np.repeat(src_mask, beams, axis=0)
217
+ generated = np.full((batch * beams, 1), self.bos_id, dtype=np.int64)
218
+ scores = np.full((batch, beams), -np.inf, dtype=np.float32)
219
+ scores[:, 0] = 0.0
220
+ finished = np.zeros((batch, beams), dtype=np.bool_)
221
+
222
+ for _ in range(self.max_new_tokens):
223
+ logits = self.decoder.run(
224
+ None,
225
+ {
226
+ "tgt_ids": generated,
227
+ "memory": memory,
228
+ "src_mask": cross_mask,
229
+ },
230
+ )[0]
231
+ logits = logits[:, -1, :]
232
+ vocab_size = logits.shape[-1]
233
+ logits = logits.reshape(batch, beams, vocab_size)
234
+ logits = self._suppress_specials(logits)
235
+ log_probs = _log_softmax(logits, axis=-1)
236
+
237
+ candidate_scores = scores[:, :, None] + log_probs
238
+ candidate_scores = np.where(
239
+ finished[:, :, None], -np.inf, candidate_scores
240
+ )
241
+
242
+ flat_scores = candidate_scores.reshape(batch, beams * vocab_size)
243
+ frozen_scores = np.where(finished, scores, -np.inf)
244
+ combined = np.concatenate((flat_scores, frozen_scores), axis=1)
245
+
246
+ top_indices = np.argpartition(-combined, beams - 1, axis=1)[:, :beams]
247
+ top_scores = np.take_along_axis(combined, top_indices, axis=1)
248
+ order = np.argsort(-top_scores, axis=1)
249
+ top_indices = np.take_along_axis(top_indices, order, axis=1)
250
+ top_scores = np.take_along_axis(top_scores, order, axis=1)
251
+
252
+ frozen = top_indices >= beams * vocab_size
253
+ frozen_beam = np.clip(top_indices - beams * vocab_size, 0, beams - 1)
254
+ active_beam = np.clip(top_indices // vocab_size, 0, beams - 1)
255
+ token_ids = top_indices % vocab_size
256
+ selected_beam = np.where(frozen, frozen_beam, active_beam)
257
+
258
+ flat_generated = generated.reshape(batch, beams, -1)
259
+ gather_index = np.broadcast_to(
260
+ selected_beam[:, :, None],
261
+ (batch, beams, flat_generated.shape[-1]),
262
+ )
263
+ next_generated = np.take_along_axis(
264
+ flat_generated, gather_index, axis=1
265
+ )
266
+ next_token = np.where(frozen, self.pad_id, token_ids)
267
+ next_generated = np.concatenate(
268
+ (next_generated, next_token[:, :, None]), axis=-1
269
+ )
270
+
271
+ old_finished = np.take_along_axis(
272
+ finished, selected_beam, axis=1
273
+ )
274
+ next_finished = old_finished | (
275
+ (~frozen) & (token_ids == self.eos_id)
276
+ )
277
+
278
+ generated = next_generated.reshape(batch * beams, -1)
279
+ scores = top_scores.astype(np.float32)
280
+ finished = next_finished
281
+
282
+ if bool(finished.all()):
283
+ break
284
+
285
+ sequences = generated.reshape(batch, beams, -1)[:, :, 1:]
286
+ eos_mask = sequences == self.eos_id
287
+ has_eos = eos_mask.any(axis=-1)
288
+ eos_pos = np.argmax(eos_mask, axis=-1)
289
+ lengths = np.where(
290
+ has_eos,
291
+ eos_pos + 1,
292
+ sequences.shape[-1],
293
+ ).astype(np.float64)
294
+ lengths = np.maximum(lengths, 1.0)
295
+
296
+ adjusted = scores.astype(np.float64) / (lengths**length_penalty)
297
+ best = np.argmax(adjusted, axis=1)
298
+ best_sequences = sequences[np.arange(batch), best]
299
+ return self._collect_results(best_sequences)
300
+
301
+ def _collect_results(self, sequences: np.ndarray) -> list[list[int]]:
302
+ results: list[list[int]] = []
303
+ for row in sequences.tolist():
304
+ if self.eos_id in row:
305
+ row = row[: row.index(self.eos_id)]
306
+ results.append(row)
307
+ return results
308
+
309
+ def _decode(self, ids: list[int]) -> str:
310
+ return self.tokenizer.decode(ids, skip_special_tokens=True).strip()
transliterate.py ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Word-by-word sentence transliteration preserving non-text runs."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import unicodedata
6
+ from collections.abc import Sequence
7
+
8
+ from onnx_inference import (
9
+ DEFAULT_BEAM_SIZE,
10
+ DEFAULT_LENGTH_PENALTY,
11
+ OnnxTransliterator,
12
+ normalize_direction,
13
+ )
14
+
15
+ __all__ = ["IndicTransliterator", "split_text_runs"]
16
+
17
+
18
+ def _is_word_char(char: str) -> bool:
19
+ """True for Unicode letters and combining marks."""
20
+ return unicodedata.category(char)[0] in ("L", "M")
21
+
22
+
23
+ def _has_letter(run: str) -> bool:
24
+ return any(unicodedata.category(char).startswith("L") for char in run)
25
+
26
+
27
+ def split_text_runs(text: str) -> list[tuple[bool, str]]:
28
+ """Split into ``(is_word, run)`` pieces; joining restores the input."""
29
+ if not text:
30
+ return []
31
+
32
+ runs: list[tuple[bool, str]] = []
33
+ start = 0
34
+ current = _is_word_char(text[0])
35
+ for index in range(1, len(text)):
36
+ is_word = _is_word_char(text[index])
37
+ if is_word != current:
38
+ runs.append((current, text[start:index]))
39
+ start = index
40
+ current = is_word
41
+ runs.append((current, text[start:]))
42
+
43
+ # Demote mark-only runs, then merge adjacent non-text runs.
44
+ cleaned: list[tuple[bool, str]] = []
45
+ for is_word, run in runs:
46
+ is_word = is_word and _has_letter(run)
47
+ if cleaned and cleaned[-1][0] == is_word:
48
+ cleaned[-1] = (is_word, cleaned[-1][1] + run)
49
+ else:
50
+ cleaned.append((is_word, run))
51
+ return cleaned
52
+
53
+
54
+ class IndicTransliterator:
55
+ """Transliterate sentences while preserving formatting.
56
+
57
+ English source text is lowercased before transliteration. Decoding options
58
+ are forwarded to the underlying :class:`OnnxTransliterator`.
59
+ """
60
+
61
+ def __init__(
62
+ self,
63
+ model_dir: str,
64
+ *,
65
+ max_src_len: int | None = None,
66
+ max_new_tokens: int | None = None,
67
+ ):
68
+ self.engine = OnnxTransliterator(
69
+ model_dir,
70
+ max_src_len=max_src_len,
71
+ max_new_tokens=max_new_tokens,
72
+ )
73
+
74
+ @property
75
+ def direction_ids(self) -> dict[str, int]:
76
+ return self.engine.direction_ids
77
+
78
+ def translate(
79
+ self,
80
+ text: str,
81
+ direction: str,
82
+ *,
83
+ beam_size: int = DEFAULT_BEAM_SIZE,
84
+ length_penalty: float = DEFAULT_LENGTH_PENALTY,
85
+ ) -> str:
86
+ """Transliterate one sentence."""
87
+ return self.translate_batch(
88
+ [text],
89
+ direction,
90
+ beam_size=beam_size,
91
+ length_penalty=length_penalty,
92
+ )[0]
93
+
94
+ def translate_batch(
95
+ self,
96
+ texts: Sequence[str],
97
+ direction: str | Sequence[str],
98
+ *,
99
+ beam_size: int = DEFAULT_BEAM_SIZE,
100
+ length_penalty: float = DEFAULT_LENGTH_PENALTY,
101
+ ) -> list[str]:
102
+ """Transliterate sentences, batching the words in each."""
103
+ texts = list(texts)
104
+ if not texts:
105
+ return []
106
+
107
+ directions = self._resolve_directions(len(texts), direction)
108
+ output: list[str] = []
109
+
110
+ for text, direction_tag in zip(texts, directions):
111
+ if direction_tag.startswith("en-"):
112
+ text = text.lower()
113
+ runs = split_text_runs(text)
114
+ words = [run for is_word, run in runs if is_word]
115
+ translated_words = (
116
+ self.engine.translate_batch(
117
+ words,
118
+ direction_tag,
119
+ beam_size=beam_size,
120
+ length_penalty=length_penalty,
121
+ )
122
+ if words
123
+ else []
124
+ )
125
+
126
+ pieces: list[str] = []
127
+ cursor = 0
128
+ for is_word, run in runs:
129
+ if is_word:
130
+ pieces.append(translated_words[cursor])
131
+ cursor += 1
132
+ else:
133
+ pieces.append(run)
134
+ output.append("".join(pieces))
135
+ return output
136
+
137
+ def __call__(self, text: str, direction: str, **kwargs) -> str:
138
+ return self.translate(text, direction, **kwargs)
139
+
140
+ def _resolve_directions(
141
+ self, count: int, direction: str | Sequence[str]
142
+ ) -> list[str]:
143
+ if isinstance(direction, str):
144
+ directions = [normalize_direction(direction)] * count
145
+ else:
146
+ directions = [normalize_direction(item) for item in direction]
147
+ if len(directions) != count:
148
+ raise ValueError(f"got {count} texts but {len(directions)} directions")
149
+
150
+ unknown = sorted(
151
+ {item for item in directions if item not in self.engine.direction_ids}
152
+ )
153
+ if unknown:
154
+ raise ValueError(
155
+ f"unknown direction(s) {unknown}; "
156
+ f"expected {sorted(self.engine.direction_ids)}"
157
+ )
158
+ return directions