Spaces:
Running
Running
File size: 5,787 Bytes
d7013e6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 | from __future__ import annotations
from pathlib import Path
import torch
from src.interfaces import ITokenizer
class CharTokenizer(ITokenizer):
"""
Character-level tokenizer for transcription.
Vocabulary: <blank> + space + a-z + apostrophe
"""
def __init__(self):
self._chars = " " + "abcdefghijklmnopqrstuvwxyz" + "'"
self._blank = "<blank>"
self._char_to_idx: dict[str, int] = {c: i + 1 for i, c in enumerate(self._chars)}
self._idx_to_char: dict[int, str] = {i + 1: c for i, c in enumerate(self._chars)}
# Index 0 is reserved for CTC blank
self._idx_to_char[0] = ""
def encode(self, text: str) -> torch.Tensor:
text = text.lower().strip()
indices = []
for c in text:
if c in self._char_to_idx:
indices.append(self._char_to_idx[c])
return torch.tensor(indices, dtype=torch.long)
def decode(self, tokens: torch.Tensor) -> str:
chars = []
prev = -1
for idx in tokens.tolist():
if idx == self.blank_id:
prev = idx
continue
if idx != prev:
chars.append(self._idx_to_char.get(idx, ""))
prev = idx
return "".join(chars)
@property
def vocab_size(self) -> int:
return len(self._chars) + 1 # +1 for blank
@property
def blank_id(self) -> int:
return 0
class BPETokenizer(ITokenizer):
"""
BPE (Byte Pair Encoding) tokenizer for transcription.
Uses a simple word-level BPE trained on the fly from training transcripts.
Fewer output steps = better CTC alignment.
Token 0 is reserved for CTC blank.
"""
def __init__(self, vocab_size: int = 1000):
self._target_vocab_size = vocab_size
# Start with character-level vocab
self._chars = list(" abcdefghijklmnopqrstuvwxyz'")
# Token 0 = blank, tokens 1..N = actual symbols
self._token_to_id: dict[str, int] = {c: i + 1 for i, c in enumerate(self._chars)}
self._id_to_token: dict[int, str] = {i + 1: c for i, c in enumerate(self._chars)}
self._id_to_token[0] = ""
self._trained = False
def train_from_texts(self, texts: list[str], max_merges: int | None = None) -> None:
"""Train BPE merges from a list of transcripts."""
max_merges = max_merges or (self._target_vocab_size - len(self._chars) - 1)
# Tokenize all texts into characters
corpus: list[list[str]] = []
for text in texts:
text = text.lower().strip()
chars = [c for c in text if c in self._token_to_id]
if chars:
corpus.append(chars)
# BPE merge loop
merges: list[tuple[str, str]] = []
for _ in range(max_merges):
# Count adjacent pairs
pair_counts: dict[tuple[str, str], int] = {}
for tokens in corpus:
for i in range(len(tokens) - 1):
pair = (tokens[i], tokens[i + 1])
pair_counts[pair] = pair_counts.get(pair, 0) + 1
if not pair_counts:
break
# Find most frequent pair
best_pair = max(pair_counts, key=pair_counts.get) # type: ignore
merged = best_pair[0] + best_pair[1]
merges.append(best_pair)
# Add to vocab
new_id = len(self._id_to_token)
self._token_to_id[merged] = new_id
self._id_to_token[new_id] = merged
# Apply merge to corpus
for idx, tokens in enumerate(corpus):
new_tokens = []
i = 0
while i < len(tokens):
if i < len(tokens) - 1 and (tokens[i], tokens[i + 1]) == best_pair:
new_tokens.append(merged)
i += 2
else:
new_tokens.append(tokens[i])
i += 1
corpus[idx] = new_tokens
self._merges = merges
self._trained = True
print(f" BPE trained: {len(merges)} merges, vocab size {self.vocab_size}", flush=True)
def encode(self, text: str) -> torch.Tensor:
text = text.lower().strip()
tokens = [c for c in text if c in self._token_to_id]
# Apply merges in order
if self._trained:
for a, b in self._merges:
merged = a + b
new_tokens = []
i = 0
while i < len(tokens):
if i < len(tokens) - 1 and tokens[i] == a and tokens[i + 1] == b:
new_tokens.append(merged)
i += 2
else:
new_tokens.append(tokens[i])
i += 1
tokens = new_tokens
indices = [self._token_to_id[t] for t in tokens if t in self._token_to_id]
return torch.tensor(indices, dtype=torch.long)
def decode(self, tokens: torch.Tensor) -> str:
pieces = []
prev = -1
for idx in tokens.tolist():
if idx == self.blank_id:
prev = idx
continue
if idx != prev:
pieces.append(self._id_to_token.get(idx, ""))
prev = idx
return "".join(pieces)
@property
def vocab_size(self) -> int:
return len(self._id_to_token)
@property
def blank_id(self) -> int:
return 0
def create_tokenizer(config_type: str = "char", **kwargs) -> ITokenizer:
"""Factory function to create the right tokenizer."""
if config_type == "bpe":
return BPETokenizer(vocab_size=kwargs.get("vocab_size", 1000))
return CharTokenizer()
|