Model_Zoo / Dockerfile
Prashanth Ramakrishna
Remove authentication for ungated Qwen model
5846074
Raw History Blame Contribute Delete
2.11 kB
# Text Generation Inference Docker for HuggingFace Spaces
# Deployment for Qwen2.5-72B with 4xL4 GPUs
FROM ghcr.io/huggingface/text-generation-inference:1.4
# Switch to root to set up directories
USER root
# Create writable directories for caching with proper permissions
RUN mkdir -p /tmp/numba_cache /tmp/huggingface /tmp/transformers && \
chmod -R 777 /tmp
# Install any missing dependencies if needed
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential \
&& rm -rf /var/lib/apt/lists/*
# Switch back to user
USER 1000
# HuggingFace Spaces configuration
ENV PORT=7860
# Fix Numba and caching issues
ENV NUMBA_CACHE_DIR=/tmp/numba_cache
ENV MPLCONFIGDIR=/tmp/matplotlib
ENV HF_HOME=/tmp/huggingface
ENV TRANSFORMERS_CACHE=/tmp/transformers
ENV HUGGINGFACE_HUB_CACHE=/tmp
ENV HOME=/tmp
# Numba specific settings to avoid caching issues
ENV NUMBA_DISABLE_PERFORMANCE_WARNINGS=1
ENV NUMBA_DISABLE_INTEL_SVML=1
# Model configuration
# Using Qwen2.5-7B - Ungated, fast, reliable XML formatting
# Switched from Mistral due to authentication issues
ENV MODEL_ID=Qwen/Qwen2.5-7B-Instruct
# HuggingFace authentication support (for gated models)
# Token should be stored as a secret in HF Space settings, not in code
# Commented out for ungated models - can cause authentication issues
# ARG HUGGING_FACE_HUB_TOKEN
# ENV HUGGING_FACE_HUB_TOKEN=${HUGGING_FACE_HUB_TOKEN}
# ENV HF_TOKEN=${HUGGING_FACE_HUB_TOKEN}
ENV HF_HOME=/tmp/huggingface
# TGI Configuration optimized for speed
ENV MAX_INPUT_LENGTH=4096
ENV MAX_TOTAL_TOKENS=8192
ENV MAX_BATCH_PREFILL_TOKENS=8192
ENV MAX_BATCH_TOTAL_TOKENS=32768
# Single GPU is enough for 7B model (much faster than sharding)
ENV NUM_SHARD=1
# No quantization needed for 7B model on L4 (24GB VRAM)
# Trust remote code (required for some models)
ENV TRUST_REMOTE_CODE=true
# Disable custom kernels that might cause issues
ENV DISABLE_CUSTOM_KERNELS=true
# Force usage of /tmp for all operations
WORKDIR /tmp
# The entrypoint is already set in the base image
# TGI will automatically start serving on the specified PORT