Update Dockerfile

Dockerfile

@@ -4,16 +4,27 @@ WORKDIR /app
 
 COPY app.py .
 
-# Use stable compatible versions
+# Install rust (needed for tokenizers)
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    curl build-essential \
+    && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \
+    && . $HOME/.cargo/env \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Install Python packages
 RUN pip install --no-cache-dir \
     gradio==4.36.1 \
     transformers==4.44.0 \
-    tokenizers==0.19.1 \
     torch==2.1.0 \
     accelerate==0.33.0 \
     sentencepiece==0.2.0 \
     protobuf==3.20.3
 
+# Install tokenizers separately to ensure it builds
+RUN pip install --no-cache-dir tokenizers==0.19.1
+
 EXPOSE 7860
 
 ENV GRADIO_SERVER_NAME="0.0.0.0"