Spaces:
Running
Running
File size: 606 Bytes
d7013e6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | from abc import ABC, abstractmethod
import torch
class ITokenizer(ABC):
"""Interface for text tokenization (transcription labels)."""
@abstractmethod
def encode(self, text: str) -> torch.Tensor:
"""Encode text string to token indices."""
...
@abstractmethod
def decode(self, tokens: torch.Tensor) -> str:
"""Decode token indices back to text string."""
...
@property
@abstractmethod
def vocab_size(self) -> int:
...
@property
@abstractmethod
def blank_id(self) -> int:
"""CTC blank token id."""
...
|