Spaces:
Running
Running
Nicholas Finch
fix: default workers=0 (sequential) for Docker cache — ProcessPoolExecutor hangs on cpu-basic
d7013e6 Download src/services/tokenizer_service.py from tisaconundrum/grassmann-cache: direct link, hf CLI and curl.
- Browser
- Download file 5.79 kB
-
https://huggingface.co/spaces/tisaconundrum/grassmann-cache/resolve/main/src/services/tokenizer_service.py
- Command line
-
hf download hf://spaces/tisaconundrum/grassmann-cache/src/services/tokenizer_service.py
-
curl -L -o tokenizer_service.py https://huggingface.co/spaces/tisaconundrum/grassmann-cache/resolve/main/src/services/tokenizer_service.py
5.79 kB
| from __future__ import annotations | |
| from pathlib import Path | |
| import torch | |
| from src.interfaces import ITokenizer | |
| class CharTokenizer(ITokenizer): | |
| """ | |
| Character-level tokenizer for transcription. | |
| Vocabulary: <blank> + space + a-z + apostrophe | |
| """ | |
| def __init__(self): | |
| self._chars = " " + "abcdefghijklmnopqrstuvwxyz" + "'" | |
| self._blank = "<blank>" | |
| self._char_to_idx: dict[str, int] = {c: i + 1 for i, c in enumerate(self._chars)} | |
| self._idx_to_char: dict[int, str] = {i + 1: c for i, c in enumerate(self._chars)} | |
| # Index 0 is reserved for CTC blank | |
| self._idx_to_char[0] = "" | |
| def encode(self, text: str) -> torch.Tensor: | |
| text = text.lower().strip() | |
| indices = [] | |
| for c in text: | |
| if c in self._char_to_idx: | |
| indices.append(self._char_to_idx[c]) | |
| return torch.tensor(indices, dtype=torch.long) | |
| def decode(self, tokens: torch.Tensor) -> str: | |
| chars = [] | |
| prev = -1 | |
| for idx in tokens.tolist(): | |
| if idx == self.blank_id: | |
| prev = idx | |
| continue | |
| if idx != prev: | |
| chars.append(self._idx_to_char.get(idx, "")) | |
| prev = idx | |
| return "".join(chars) | |
| def vocab_size(self) -> int: | |
| return len(self._chars) + 1 # +1 for blank | |
| def blank_id(self) -> int: | |
| return 0 | |
| class BPETokenizer(ITokenizer): | |
| """ | |
| BPE (Byte Pair Encoding) tokenizer for transcription. | |
| Uses a simple word-level BPE trained on the fly from training transcripts. | |
| Fewer output steps = better CTC alignment. | |
| Token 0 is reserved for CTC blank. | |
| """ | |
| def __init__(self, vocab_size: int = 1000): | |
| self._target_vocab_size = vocab_size | |
| # Start with character-level vocab | |
| self._chars = list(" abcdefghijklmnopqrstuvwxyz'") | |
| # Token 0 = blank, tokens 1..N = actual symbols | |
| self._token_to_id: dict[str, int] = {c: i + 1 for i, c in enumerate(self._chars)} | |
| self._id_to_token: dict[int, str] = {i + 1: c for i, c in enumerate(self._chars)} | |
| self._id_to_token[0] = "" | |
| self._trained = False | |
| def train_from_texts(self, texts: list[str], max_merges: int | None = None) -> None: | |
| """Train BPE merges from a list of transcripts.""" | |
| max_merges = max_merges or (self._target_vocab_size - len(self._chars) - 1) | |
| # Tokenize all texts into characters | |
| corpus: list[list[str]] = [] | |
| for text in texts: | |
| text = text.lower().strip() | |
| chars = [c for c in text if c in self._token_to_id] | |
| if chars: | |
| corpus.append(chars) | |
| # BPE merge loop | |
| merges: list[tuple[str, str]] = [] | |
| for _ in range(max_merges): | |
| # Count adjacent pairs | |
| pair_counts: dict[tuple[str, str], int] = {} | |
| for tokens in corpus: | |
| for i in range(len(tokens) - 1): | |
| pair = (tokens[i], tokens[i + 1]) | |
| pair_counts[pair] = pair_counts.get(pair, 0) + 1 | |
| if not pair_counts: | |
| break | |
| # Find most frequent pair | |
| best_pair = max(pair_counts, key=pair_counts.get) # type: ignore | |
| merged = best_pair[0] + best_pair[1] | |
| merges.append(best_pair) | |
| # Add to vocab | |
| new_id = len(self._id_to_token) | |
| self._token_to_id[merged] = new_id | |
| self._id_to_token[new_id] = merged | |
| # Apply merge to corpus | |
| for idx, tokens in enumerate(corpus): | |
| new_tokens = [] | |
| i = 0 | |
| while i < len(tokens): | |
| if i < len(tokens) - 1 and (tokens[i], tokens[i + 1]) == best_pair: | |
| new_tokens.append(merged) | |
| i += 2 | |
| else: | |
| new_tokens.append(tokens[i]) | |
| i += 1 | |
| corpus[idx] = new_tokens | |
| self._merges = merges | |
| self._trained = True | |
| print(f" BPE trained: {len(merges)} merges, vocab size {self.vocab_size}", flush=True) | |
| def encode(self, text: str) -> torch.Tensor: | |
| text = text.lower().strip() | |
| tokens = [c for c in text if c in self._token_to_id] | |
| # Apply merges in order | |
| if self._trained: | |
| for a, b in self._merges: | |
| merged = a + b | |
| new_tokens = [] | |
| i = 0 | |
| while i < len(tokens): | |
| if i < len(tokens) - 1 and tokens[i] == a and tokens[i + 1] == b: | |
| new_tokens.append(merged) | |
| i += 2 | |
| else: | |
| new_tokens.append(tokens[i]) | |
| i += 1 | |
| tokens = new_tokens | |
| indices = [self._token_to_id[t] for t in tokens if t in self._token_to_id] | |
| return torch.tensor(indices, dtype=torch.long) | |
| def decode(self, tokens: torch.Tensor) -> str: | |
| pieces = [] | |
| prev = -1 | |
| for idx in tokens.tolist(): | |
| if idx == self.blank_id: | |
| prev = idx | |
| continue | |
| if idx != prev: | |
| pieces.append(self._id_to_token.get(idx, "")) | |
| prev = idx | |
| return "".join(pieces) | |
| def vocab_size(self) -> int: | |
| return len(self._id_to_token) | |
| def blank_id(self) -> int: | |
| return 0 | |
| def create_tokenizer(config_type: str = "char", **kwargs) -> ITokenizer: | |
| """Factory function to create the right tokenizer.""" | |
| if config_type == "bpe": | |
| return BPETokenizer(vocab_size=kwargs.get("vocab_size", 1000)) | |
| return CharTokenizer() | |