| # Install Hugging Face tokenizers (hard dependency of transformers>=4.57). | |
| set -euo pipefail | |
| cd "$(dirname "$0")/.." | |
| if python -c "import tokenizers" 2>/dev/null; then | |
| python -c "import tokenizers; print(f'tokenizers already installed: {tokenizers.__version__}')" | |
| exit 0 | |
| fi | |
| echo "Installing tokenizers (required by transformers)..." | |
| pip install 'tokenizers>=0.21.0,<0.23.0' | |
| python -c "import tokenizers; print(f'OK: tokenizers {tokenizers.__version__}')" | |