Prashanth Ramakrishna commited on
Commit
82e4ccc
·
1 Parent(s): ab82ad3

Switch to Qwen2.5-72B-Instruct with 4xL4 GPU tensor parallelism and NF4 quantization

Browse files
Files changed (2) hide show
  1. Dockerfile +11 -7
  2. Dockerfile.simple +16 -0
Dockerfile CHANGED
@@ -32,16 +32,20 @@ ENV NUMBA_DISABLE_PERFORMANCE_WARNINGS=1
32
  ENV NUMBA_DISABLE_INTEL_SVML=1
33
 
34
  # Model configuration
35
- ENV MODEL_ID=mistralai/Mistral-7B-Instruct-v0.2
 
36
 
37
- # TGI Configuration for optimal performance
38
- ENV MAX_INPUT_LENGTH=2048
39
- ENV MAX_TOTAL_TOKENS=4096
40
  ENV MAX_BATCH_PREFILL_TOKENS=4096
41
- ENV MAX_BATCH_TOTAL_TOKENS=8192
42
 
43
- # Number of shards (for model parallelism, set to 1 for single GPU)
44
- ENV NUM_SHARD=1
 
 
 
45
 
46
  # Trust remote code (required for some models)
47
  ENV TRUST_REMOTE_CODE=true
 
32
  ENV NUMBA_DISABLE_INTEL_SVML=1
33
 
34
  # Model configuration
35
+ # Using Qwen2.5-72B-Instruct - state-of-the-art open model
36
+ ENV MODEL_ID=Qwen/Qwen2.5-72B-Instruct
37
 
38
+ # TGI Configuration for large model
39
+ ENV MAX_INPUT_LENGTH=4096
40
+ ENV MAX_TOTAL_TOKENS=8192
41
  ENV MAX_BATCH_PREFILL_TOKENS=4096
42
+ ENV MAX_BATCH_TOTAL_TOKENS=16384
43
 
44
+ # Use all 4 GPUs for tensor parallelism (required for 72B model)
45
+ ENV NUM_SHARD=4
46
+
47
+ # Enable quantization to fit in 96GB VRAM (4xL4)
48
+ ENV QUANTIZE=bitsandbytes-nf4
49
 
50
  # Trust remote code (required for some models)
51
  ENV TRUST_REMOTE_CODE=true
Dockerfile.simple ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Simplified TGI deployment - if main Dockerfile still has issues
2
+ FROM ghcr.io/huggingface/text-generation-inference:1.4
3
+
4
+ # Completely disable JIT compilation to avoid all caching issues
5
+ ENV NUMBA_DISABLE_JIT=1
6
+
7
+ # Basic configuration
8
+ ENV PORT=7860
9
+ ENV MODEL_ID=mistralai/Mistral-7B-Instruct-v0.2
10
+ ENV MAX_INPUT_LENGTH=2048
11
+ ENV MAX_TOTAL_TOKENS=4096
12
+ ENV TRUST_REMOTE_CODE=true
13
+ ENV HUGGINGFACE_HUB_CACHE=/tmp
14
+
15
+ # Run from /tmp
16
+ WORKDIR /tmp