Spaces:
Runtime error
Runtime error
Prashanth Ramakrishna commited on
Commit ·
82e4ccc
1
Parent(s): ab82ad3
Switch to Qwen2.5-72B-Instruct with 4xL4 GPU tensor parallelism and NF4 quantization
Browse files- Dockerfile +11 -7
- Dockerfile.simple +16 -0
Dockerfile
CHANGED
|
@@ -32,16 +32,20 @@ ENV NUMBA_DISABLE_PERFORMANCE_WARNINGS=1
|
|
| 32 |
ENV NUMBA_DISABLE_INTEL_SVML=1
|
| 33 |
|
| 34 |
# Model configuration
|
| 35 |
-
|
|
|
|
| 36 |
|
| 37 |
-
# TGI Configuration for
|
| 38 |
-
ENV MAX_INPUT_LENGTH=
|
| 39 |
-
ENV MAX_TOTAL_TOKENS=
|
| 40 |
ENV MAX_BATCH_PREFILL_TOKENS=4096
|
| 41 |
-
ENV MAX_BATCH_TOTAL_TOKENS=
|
| 42 |
|
| 43 |
-
#
|
| 44 |
-
ENV NUM_SHARD=
|
|
|
|
|
|
|
|
|
|
| 45 |
|
| 46 |
# Trust remote code (required for some models)
|
| 47 |
ENV TRUST_REMOTE_CODE=true
|
|
|
|
| 32 |
ENV NUMBA_DISABLE_INTEL_SVML=1
|
| 33 |
|
| 34 |
# Model configuration
|
| 35 |
+
# Using Qwen2.5-72B-Instruct - state-of-the-art open model
|
| 36 |
+
ENV MODEL_ID=Qwen/Qwen2.5-72B-Instruct
|
| 37 |
|
| 38 |
+
# TGI Configuration for large model
|
| 39 |
+
ENV MAX_INPUT_LENGTH=4096
|
| 40 |
+
ENV MAX_TOTAL_TOKENS=8192
|
| 41 |
ENV MAX_BATCH_PREFILL_TOKENS=4096
|
| 42 |
+
ENV MAX_BATCH_TOTAL_TOKENS=16384
|
| 43 |
|
| 44 |
+
# Use all 4 GPUs for tensor parallelism (required for 72B model)
|
| 45 |
+
ENV NUM_SHARD=4
|
| 46 |
+
|
| 47 |
+
# Enable quantization to fit in 96GB VRAM (4xL4)
|
| 48 |
+
ENV QUANTIZE=bitsandbytes-nf4
|
| 49 |
|
| 50 |
# Trust remote code (required for some models)
|
| 51 |
ENV TRUST_REMOTE_CODE=true
|
Dockerfile.simple
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Simplified TGI deployment - if main Dockerfile still has issues
|
| 2 |
+
FROM ghcr.io/huggingface/text-generation-inference:1.4
|
| 3 |
+
|
| 4 |
+
# Completely disable JIT compilation to avoid all caching issues
|
| 5 |
+
ENV NUMBA_DISABLE_JIT=1
|
| 6 |
+
|
| 7 |
+
# Basic configuration
|
| 8 |
+
ENV PORT=7860
|
| 9 |
+
ENV MODEL_ID=mistralai/Mistral-7B-Instruct-v0.2
|
| 10 |
+
ENV MAX_INPUT_LENGTH=2048
|
| 11 |
+
ENV MAX_TOTAL_TOKENS=4096
|
| 12 |
+
ENV TRUST_REMOTE_CODE=true
|
| 13 |
+
ENV HUGGINGFACE_HUB_CACHE=/tmp
|
| 14 |
+
|
| 15 |
+
# Run from /tmp
|
| 16 |
+
WORKDIR /tmp
|