# Text Generation Inference Docker for HuggingFace Spaces # Deployment for Qwen2.5-72B with 4xL4 GPUs FROM ghcr.io/huggingface/text-generation-inference:1.4 # Switch to root to set up directories USER root # Create writable directories for caching with proper permissions RUN mkdir -p /tmp/numba_cache /tmp/huggingface /tmp/transformers && \ chmod -R 777 /tmp # Install any missing dependencies if needed RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ && rm -rf /var/lib/apt/lists/* # Switch back to user USER 1000 # HuggingFace Spaces configuration ENV PORT=7860 # Fix Numba and caching issues ENV NUMBA_CACHE_DIR=/tmp/numba_cache ENV MPLCONFIGDIR=/tmp/matplotlib ENV HF_HOME=/tmp/huggingface ENV TRANSFORMERS_CACHE=/tmp/transformers ENV HUGGINGFACE_HUB_CACHE=/tmp ENV HOME=/tmp # Numba specific settings to avoid caching issues ENV NUMBA_DISABLE_PERFORMANCE_WARNINGS=1 ENV NUMBA_DISABLE_INTEL_SVML=1 # Model configuration # Using Qwen2.5-7B - Ungated, fast, reliable XML formatting # Switched from Mistral due to authentication issues ENV MODEL_ID=Qwen/Qwen2.5-7B-Instruct # HuggingFace authentication support (for gated models) # Token should be stored as a secret in HF Space settings, not in code # Commented out for ungated models - can cause authentication issues # ARG HUGGING_FACE_HUB_TOKEN # ENV HUGGING_FACE_HUB_TOKEN=${HUGGING_FACE_HUB_TOKEN} # ENV HF_TOKEN=${HUGGING_FACE_HUB_TOKEN} ENV HF_HOME=/tmp/huggingface # TGI Configuration optimized for speed ENV MAX_INPUT_LENGTH=4096 ENV MAX_TOTAL_TOKENS=8192 ENV MAX_BATCH_PREFILL_TOKENS=8192 ENV MAX_BATCH_TOTAL_TOKENS=32768 # Single GPU is enough for 7B model (much faster than sharding) ENV NUM_SHARD=1 # No quantization needed for 7B model on L4 (24GB VRAM) # Trust remote code (required for some models) ENV TRUST_REMOTE_CODE=true # Disable custom kernels that might cause issues ENV DISABLE_CUSTOM_KERNELS=true # Force usage of /tmp for all operations WORKDIR /tmp # The entrypoint is already set in the base image # TGI will automatically start serving on the specified PORT