File size: 606 Bytes
d7013e6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
from abc import ABC, abstractmethod

import torch


class ITokenizer(ABC):
    """Interface for text tokenization (transcription labels)."""

    @abstractmethod
    def encode(self, text: str) -> torch.Tensor:
        """Encode text string to token indices."""
        ...

    @abstractmethod
    def decode(self, tokens: torch.Tensor) -> str:
        """Decode token indices back to text string."""
        ...

    @property
    @abstractmethod
    def vocab_size(self) -> int:
        ...

    @property
    @abstractmethod
    def blank_id(self) -> int:
        """CTC blank token id."""
        ...