Spaces:
Running on T4
Running on T4
Download src/interfaces/i_tokenizer.py from tisaconundrum/grassmann-ablation: direct link, hf CLI and curl.
- Browser
- Download file 606 Bytes
-
https://huggingface.co/spaces/tisaconundrum/grassmann-ablation/resolve/main/src/interfaces/i_tokenizer.py
- Command line
-
hf download hf://spaces/tisaconundrum/grassmann-ablation/src/interfaces/i_tokenizer.py
-
curl -L -o i_tokenizer.py https://huggingface.co/spaces/tisaconundrum/grassmann-ablation/resolve/main/src/interfaces/i_tokenizer.py
606 Bytes
| from abc import ABC, abstractmethod | |
| import torch | |
| class ITokenizer(ABC): | |
| """Interface for text tokenization (transcription labels).""" | |
| def encode(self, text: str) -> torch.Tensor: | |
| """Encode text string to token indices.""" | |
| ... | |
| def decode(self, tokens: torch.Tensor) -> str: | |
| """Decode token indices back to text string.""" | |
| ... | |
| def vocab_size(self) -> int: | |
| ... | |
| def blank_id(self) -> int: | |
| """CTC blank token id.""" | |
| ... | |