Spaces:
Running on Zero
Running on Zero
Upload 5 files
Browse files- .env.example +9 -8
- .gitignore +1 -1
- README.md +120 -98
- app.py +416 -75
- requirements.txt +5 -3
.env.example
CHANGED
|
@@ -1,10 +1,11 @@
|
|
| 1 |
-
# Required
|
| 2 |
LLAMA_API_KEY=replace-with-a-long-random-secret
|
| 3 |
|
| 4 |
-
# Optional
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
|
|
|
|
|
| 1 |
+
# Required in Hugging Face Space Settings > Variables and secrets > Secrets
|
| 2 |
LLAMA_API_KEY=replace-with-a-long-random-secret
|
| 3 |
|
| 4 |
+
# Optional tuning variables (set these under Space Variables)
|
| 5 |
+
LLAMA_N_CTX=4096
|
| 6 |
+
LLAMA_N_BATCH=1024
|
| 7 |
+
LLAMA_N_UBATCH=512
|
| 8 |
+
LLAMA_N_THREADS=4
|
| 9 |
+
LLAMA_MAX_TOKENS=2048
|
| 10 |
+
LLAMA_GPU_DURATION_MAX=60
|
| 11 |
+
LOG_LEVEL=INFO
|
.gitignore
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
.env
|
| 2 |
-
.venv/
|
| 3 |
__pycache__/
|
| 4 |
*.py[cod]
|
| 5 |
.cache/
|
| 6 |
models/
|
|
|
|
|
|
| 1 |
.env
|
|
|
|
| 2 |
__pycache__/
|
| 3 |
*.py[cod]
|
| 4 |
.cache/
|
| 5 |
models/
|
| 6 |
+
*.gguf
|
README.md
CHANGED
|
@@ -1,168 +1,190 @@
|
|
| 1 |
---
|
| 2 |
-
title: Qwen2.5 Coder 0.5B API
|
| 3 |
emoji: ⚡
|
| 4 |
colorFrom: blue
|
| 5 |
-
colorTo:
|
| 6 |
sdk: gradio
|
| 7 |
-
python_version:
|
| 8 |
app_file: app.py
|
|
|
|
| 9 |
---
|
| 10 |
|
| 11 |
-
# Qwen2.5-Coder-0.5B-Instruct REST API
|
| 12 |
|
| 13 |
-
|
| 14 |
|
| 15 |
-
|
| 16 |
-
- Quantization: `Q4_K_M` (about 491 MB on the Hub)
|
| 17 |
-
- Runtime: `llama-cpp-python` (Python bindings to llama.cpp)
|
| 18 |
-
- Default hardware target: Hugging Face CPU Basic (2 vCPU / 16 GB RAM)
|
| 19 |
-
- Authentication: one required bearer API key from the Space Secret `LLAMA_API_KEY`
|
| 20 |
-
- Streaming: supported by `/v1/chat/completions`
|
| 21 |
-
- Docker: **not required**
|
| 22 |
|
| 23 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
|
| 25 |
-
|
| 26 |
-
2. Upload or push all files from this project to the Space repository.
|
| 27 |
-
3. Open **Settings → Variables and secrets**.
|
| 28 |
-
4. Add a **Secret** named `LLAMA_API_KEY` and set a long, random value. Do not put the key in this repository or in a normal variable.
|
| 29 |
-
5. Select **CPU Basic** hardware if it is not already selected, then let the Space build and start.
|
| 30 |
-
6. Once it is running, the API base URL is:
|
| 31 |
|
| 32 |
-
|
| 33 |
|
| 34 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
|
| 36 |
-
|
| 37 |
|
| 38 |
-
|
|
|
|
|
|
|
| 39 |
|
| 40 |
-
|
| 41 |
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
``
|
|
|
|
|
|
|
|
|
|
| 45 |
|
| 46 |
-
The
|
| 47 |
|
| 48 |
-
##
|
| 49 |
|
| 50 |
-
|
| 51 |
-
|---|---|---|---|
|
| 52 |
-
| `GET` | `/health` | No | Health check |
|
| 53 |
-
| `GET` | `/v1/models` | Bearer key | List the served model |
|
| 54 |
-
| `POST` | `/v1/chat/completions` | Bearer key | Chat Completions; supports streaming |
|
| 55 |
-
| `POST` | `/v1/completions` | Bearer key | Text Completions |
|
| 56 |
-
| `GET` | `/docs` | No | OpenAPI documentation |
|
| 57 |
-
| `GET` | `/console` | No | Small deployment/API information page |
|
| 58 |
|
| 59 |
-
|
|
|
|
|
|
|
| 60 |
|
| 61 |
-
|
| 62 |
|
| 63 |
-
```
|
| 64 |
-
|
| 65 |
```
|
| 66 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
```bash
|
| 68 |
-
curl https://YOUR-
|
| 69 |
-H "Authorization: Bearer YOUR_LLAMA_API_KEY" \
|
| 70 |
-H "Content-Type: application/json" \
|
| 71 |
-d '{
|
| 72 |
"model": "qwen2.5-coder-0.5b-instruct",
|
| 73 |
"messages": [
|
| 74 |
-
{"role": "system", "content": "You are
|
| 75 |
-
{"role": "user", "content": "Write a JavaScript function
|
| 76 |
],
|
| 77 |
"temperature": 0.2,
|
| 78 |
-
"max_tokens":
|
| 79 |
"stream": false
|
| 80 |
}'
|
| 81 |
```
|
| 82 |
|
| 83 |
-
|
| 84 |
|
| 85 |
```bash
|
| 86 |
-
curl -N https://YOUR-
|
| 87 |
-H "Authorization: Bearer YOUR_LLAMA_API_KEY" \
|
| 88 |
-H "Content-Type: application/json" \
|
| 89 |
-d '{
|
| 90 |
"model": "qwen2.5-coder-0.5b-instruct",
|
| 91 |
-
"messages": [{"role": "user", "content": "Explain JavaScript promises
|
| 92 |
-
"
|
|
|
|
| 93 |
"stream": true
|
| 94 |
}'
|
| 95 |
```
|
| 96 |
|
| 97 |
-
##
|
| 98 |
|
| 99 |
```bash
|
| 100 |
-
|
| 101 |
```
|
| 102 |
|
| 103 |
-
```
|
| 104 |
-
import
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
|
| 106 |
-
|
| 107 |
-
baseURL: "https://YOUR-SPACE-NAME.hf.space/v1",
|
| 108 |
-
apiKey: process.env.LLAMA_API_KEY,
|
| 109 |
-
});
|
| 110 |
|
| 111 |
-
|
| 112 |
-
model: "qwen2.5-coder-0.5b-instruct",
|
| 113 |
-
messages: [{ role: "user", content: "Write a TypeScript debounce function." }],
|
| 114 |
-
max_tokens: 256,
|
| 115 |
-
});
|
| 116 |
|
| 117 |
-
|
| 118 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 119 |
|
| 120 |
-
|
| 121 |
|
| 122 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 123 |
|
| 124 |
-
|
| 125 |
|
| 126 |
-
|
| 127 |
-
- Uses two inference threads by default to match CPU Basic's two vCPUs.
|
| 128 |
-
- Uses memory-mapped model loading, a 512-token logical batch, and a 256-token physical micro-batch.
|
| 129 |
-
- Starts with a 2,048-token context to reduce per-request memory and attention work. Increase it when larger prompts are more important than speed.
|
| 130 |
-
- Uses Q4_K_M rather than a larger quantization to keep the model compact.
|
| 131 |
|
| 132 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 133 |
|
| 134 |
-
##
|
| 135 |
|
| 136 |
-
|
| 137 |
|
| 138 |
-
|
| 139 |
-
|---|---:|---|
|
| 140 |
-
| `MODEL_REPO_ID` | `Qwen/Qwen2.5-Coder-0.5B-Instruct-GGUF` | Hugging Face model repository |
|
| 141 |
-
| `MODEL_FILENAME` | `qwen2.5-coder-0.5b-instruct-q4_k_m.gguf` | Exact GGUF file to download |
|
| 142 |
-
| `MODEL_ALIAS` | `qwen2.5-coder-0.5b-instruct` | Model name accepted in API requests |
|
| 143 |
-
| `LLAMA_N_THREADS` | `2` | Generation CPU threads |
|
| 144 |
-
| `LLAMA_N_THREADS_BATCH` | `2` | Prompt-processing CPU threads |
|
| 145 |
-
| `LLAMA_N_CTX` | `2048` | Context window size |
|
| 146 |
-
| `LLAMA_N_BATCH` | `512` | Logical prompt batch size |
|
| 147 |
-
| `LLAMA_N_UBATCH` | `256` | Physical prompt micro-batch size |
|
| 148 |
|
| 149 |
-
|
| 150 |
|
| 151 |
-
|
| 152 |
|
| 153 |
-
|
| 154 |
|
| 155 |
-
|
| 156 |
-
python app.py
|
| 157 |
-
```
|
| 158 |
|
| 159 |
-
|
| 160 |
|
| 161 |
-
|
| 162 |
|
| 163 |
-
|
| 164 |
-
- [llama-cpp-python](https://github.com/abetlen/llama-cpp-python)
|
| 165 |
-
- [llama.cpp server documentation](https://github.com/ggml-org/llama.cpp/tree/master/tools/server)
|
| 166 |
-
- [Hugging Face Spaces overview](https://huggingface.co/docs/hub/spaces-overview)
|
| 167 |
|
| 168 |
-
The
|
|
|
|
| 1 |
---
|
| 2 |
+
title: Qwen2.5 Coder 0.5B Instruct API
|
| 3 |
emoji: ⚡
|
| 4 |
colorFrom: blue
|
| 5 |
+
colorTo: indigo
|
| 6 |
sdk: gradio
|
| 7 |
+
python_version: 3.12
|
| 8 |
app_file: app.py
|
| 9 |
+
pinned: false
|
| 10 |
---
|
| 11 |
|
| 12 |
+
# Qwen2.5-Coder-0.5B-Instruct — ZeroGPU REST API
|
| 13 |
|
| 14 |
+
An OpenAI-compatible REST API for the official Qwen2.5-Coder-0.5B-Instruct **Q4_K_M GGUF** model, powered by `llama.cpp` with CUDA GPU offload on Hugging Face ZeroGPU.
|
| 15 |
|
| 16 |
+
## What changed from the CPU project
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
|
| 18 |
+
- Uses `@spaces.GPU` to request ZeroGPU for inference calls.
|
| 19 |
+
- FastAPI routes submit inference jobs through the internal Gradio queue; this is important because ZeroGPU schedules registered Gradio GPU callbacks.
|
| 20 |
+
- Forwards Hugging Face's incoming `X-IP-Token` to the internal queue when present, so requests can be attributed to the caller instead of automatically falling into the anonymous pool.
|
| 21 |
+
- Uses `n_gpu_layers=-1` to offload all model layers to the GPU.
|
| 22 |
+
- Uses CUDA 13.0 prebuilt `llama-cpp-python` wheels (`0.3.36`) instead of building llama.cpp from source during Space startup.
|
| 23 |
+
- `--only-binary=llama-cpp-python` prevents pip from silently falling back to a source build.
|
| 24 |
+
- Uses Flash Attention and larger batches for throughput.
|
| 25 |
+
- Keeps one API credential: the `LLAMA_API_KEY` Space Secret.
|
| 26 |
+
- The model file is downloaded to the Space cache at startup; it is not included in this repository.
|
| 27 |
|
| 28 |
+
> **Compatibility note:** Hugging Face documents ZeroGPU as primarily compatible with PyTorch-based Spaces and warns that some other GPU workloads may have limited compatibility. This project uses native CUDA through llama.cpp inside a ZeroGPU worker; that exact combination is not guaranteed by Hugging Face. It deliberately refuses CPU fallback. The first real inference request is the definitive compatibility test. See [ZeroGPU docs](https://huggingface.co/docs/hub/spaces-zerogpu) and [llama-cpp-python CUDA installation](https://github.com/abetlen/llama-cpp-python#cuda).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
|
| 30 |
+
## Create / update the Space
|
| 31 |
|
| 32 |
+
1. Create a Space with **SDK: Gradio**.
|
| 33 |
+
2. Keep the Space hardware set to **ZeroGPU**.
|
| 34 |
+
3. Upload the contents of this folder to the Space's `main` branch (replace the old `app.py`, `README.md`, and `requirements.txt`).
|
| 35 |
+
4. Open **Settings → Variables and secrets → New secret**.
|
| 36 |
+
5. Add `LLAMA_API_KEY` with a long random value. The value is not stored in this repository.
|
| 37 |
+
6. Let the Space rebuild. The Space uses Python 3.12. The dependency step should install the prebuilt CUDA 13.0 wheel; it should not print `Building wheel for llama-cpp-python`. The `gradio`, `spaces`, and CUDA-enabled `torch` packages are supplied by the Hugging Face ZeroGPU Gradio runtime and are intentionally not overridden in `requirements.txt`.
|
| 38 |
|
| 39 |
+
Generate a key locally:
|
| 40 |
|
| 41 |
+
```bash
|
| 42 |
+
python -c "import secrets; print(secrets.token_urlsafe(48))"
|
| 43 |
+
```
|
| 44 |
|
| 45 |
+
Do not put the API key in a regular Variable or commit it to source control. Use a **Secret**.
|
| 46 |
|
| 47 |
+
## Model
|
| 48 |
+
|
| 49 |
+
- Repository: `Qwen/Qwen2.5-Coder-0.5B-Instruct-GGUF`
|
| 50 |
+
- File: `qwen2.5-coder-0.5b-instruct-q4_k_m.gguf`
|
| 51 |
+
- Quantization: `Q4_K_M`
|
| 52 |
+
- Approximate download size: 491 MB
|
| 53 |
|
| 54 |
+
The model is downloaded automatically on startup with `huggingface_hub`.
|
| 55 |
|
| 56 |
+
## API
|
| 57 |
|
| 58 |
+
Your Space URL looks like:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 59 |
|
| 60 |
+
```text
|
| 61 |
+
https://YOUR-USERNAME-qwen25-coder-api.hf.space
|
| 62 |
+
```
|
| 63 |
|
| 64 |
+
All `/v1` endpoints require:
|
| 65 |
|
| 66 |
+
```http
|
| 67 |
+
Authorization: Bearer YOUR_LLAMA_API_KEY
|
| 68 |
```
|
| 69 |
|
| 70 |
+
| Method | Endpoint | Auth |
|
| 71 |
+
|---|---|---|
|
| 72 |
+
| `GET` | `/health` | No |
|
| 73 |
+
| `GET` | `/v1/models` | Yes |
|
| 74 |
+
| `POST` | `/v1/chat/completions` | Yes |
|
| 75 |
+
| `POST` | `/v1/completions` | Yes |
|
| 76 |
+
| `GET` | `/docs` | No |
|
| 77 |
+
|
| 78 |
+
### Chat completion
|
| 79 |
+
|
| 80 |
```bash
|
| 81 |
+
curl https://YOUR-USERNAME-qwen25-coder-api.hf.space/v1/chat/completions \
|
| 82 |
-H "Authorization: Bearer YOUR_LLAMA_API_KEY" \
|
| 83 |
-H "Content-Type: application/json" \
|
| 84 |
-d '{
|
| 85 |
"model": "qwen2.5-coder-0.5b-instruct",
|
| 86 |
"messages": [
|
| 87 |
+
{"role": "system", "content": "You are an expert programming assistant."},
|
| 88 |
+
{"role": "user", "content": "Write a JavaScript function to debounce another function."}
|
| 89 |
],
|
| 90 |
"temperature": 0.2,
|
| 91 |
+
"max_tokens": 512,
|
| 92 |
"stream": false
|
| 93 |
}'
|
| 94 |
```
|
| 95 |
|
| 96 |
+
### Streaming
|
| 97 |
|
| 98 |
```bash
|
| 99 |
+
curl -N https://YOUR-USERNAME-qwen25-coder-api.hf.space/v1/chat/completions \
|
| 100 |
-H "Authorization: Bearer YOUR_LLAMA_API_KEY" \
|
| 101 |
-H "Content-Type: application/json" \
|
| 102 |
-d '{
|
| 103 |
"model": "qwen2.5-coder-0.5b-instruct",
|
| 104 |
+
"messages": [{"role": "user", "content": "Explain JavaScript promises with a short example."}],
|
| 105 |
+
"temperature": 0.2,
|
| 106 |
+
"max_tokens": 512,
|
| 107 |
"stream": true
|
| 108 |
}'
|
| 109 |
```
|
| 110 |
|
| 111 |
+
### Python OpenAI SDK
|
| 112 |
|
| 113 |
```bash
|
| 114 |
+
pip install openai
|
| 115 |
```
|
| 116 |
|
| 117 |
+
```python
|
| 118 |
+
import os
|
| 119 |
+
from openai import OpenAI
|
| 120 |
+
|
| 121 |
+
client = OpenAI(
|
| 122 |
+
base_url="https://YOUR-USERNAME-qwen25-coder-api.hf.space/v1",
|
| 123 |
+
api_key=os.environ["LLAMA_API_KEY"],
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
result = client.chat.completions.create(
|
| 127 |
+
model="qwen2.5-coder-0.5b-instruct",
|
| 128 |
+
messages=[{"role": "user", "content": "Write a Python binary search."}],
|
| 129 |
+
temperature=0.2,
|
| 130 |
+
max_tokens=512,
|
| 131 |
+
)
|
| 132 |
+
print(result.choices[0].message.content)
|
| 133 |
+
```
|
| 134 |
|
| 135 |
+
## Performance settings
|
|
|
|
|
|
|
|
|
|
| 136 |
|
| 137 |
+
The defaults are tuned for the small 0.5B model and ZeroGPU's RTX PRO 6000 Blackwell allocation:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 138 |
|
| 139 |
+
- Full layer offload (`n_gpu_layers=-1`)
|
| 140 |
+
- Flash Attention enabled
|
| 141 |
+
- `n_ctx=4096`
|
| 142 |
+
- `n_batch=1024`, `n_ubatch=512`
|
| 143 |
+
- Memory mapping enabled
|
| 144 |
+
- Generation defaults to 512 tokens; maximum accepted output is 2048 tokens
|
| 145 |
|
| 146 |
+
Optional Space **Variables** (not Secrets):
|
| 147 |
|
| 148 |
+
| Variable | Default | Meaning |
|
| 149 |
+
|---|---:|---|
|
| 150 |
+
| `LLAMA_N_CTX` | `4096` | Context window |
|
| 151 |
+
| `LLAMA_N_BATCH` | `1024` | Prompt batch size |
|
| 152 |
+
| `LLAMA_N_UBATCH` | `512` | Physical micro-batch size |
|
| 153 |
+
| `LLAMA_N_THREADS` | `4` | CPU threads for sampling / host work |
|
| 154 |
+
| `LLAMA_MAX_TOKENS` | `2048` | API maximum output limit |
|
| 155 |
+
| `LLAMA_GPU_DURATION_MAX` | `60` | Maximum reserved ZeroGPU seconds per inference |
|
| 156 |
+
| `LOG_LEVEL` | `INFO` | Logging verbosity |
|
| 157 |
|
| 158 |
+
Don't set `n_gpu_layers` to zero. This build is intended to use the GPU only.
|
| 159 |
|
| 160 |
+
## ZeroGPU behavior and limits
|
|
|
|
|
|
|
|
|
|
|
|
|
| 161 |
|
| 162 |
+
- ZeroGPU is dynamically allocated. The model is loaded in the GPU worker on its first request and may remain cached while that worker stays warm. A cold worker needs to load it again.
|
| 163 |
+
- GPU quota and queue priority depend on the caller's Hugging Face account tier and authentication context. A custom `LLAMA_API_KEY` authenticates this API, but it is **not** a Hugging Face login token and does not itself grant the caller a personal ZeroGPU quota.
|
| 164 |
+
- FastAPI does not call the decorated GPU functions directly. It submits each request to the registered Gradio event queue, where ZeroGPU can allocate the GPU.
|
| 165 |
+
- `@spaces.GPU` reserves GPU time for inference. Requests are capped to keep reservations bounded.
|
| 166 |
+
- If Hugging Face provides an `X-IP-Token` header on the incoming request, the API forwards it to the internal Gradio event call. A caller that does not receive this header may be treated as unauthenticated by ZeroGPU.
|
| 167 |
+
- A successful package install does not prove CUDA works. Check the logs from the first inference call for a positive GPU offload message (for example, `offloaded ... layers to GPU`).
|
| 168 |
+
- If the CUDA wheel cannot load, the CUDA runtime is missing, the GPU worker cannot access native CUDA, or the Blackwell kernel is unsupported, inference returns an error. It will not continue on CPU.
|
| 169 |
|
| 170 |
+
## Troubleshooting
|
| 171 |
|
| 172 |
+
### The build still says `Building wheel for llama-cpp-python`
|
| 173 |
|
| 174 |
+
This project pins a binary-only CUDA wheel. Make sure the old `requirements.txt` was replaced and the new requirements were committed. The binary-only flag should make pip fail with `No matching distribution found` rather than compile from source if the wheel is unavailable for that Python/platform combination.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 175 |
|
| 176 |
+
### `llama_supports_gpu_offload()` is false
|
| 177 |
|
| 178 |
+
The installed wheel is not exposing a usable GPU backend. Check that the `cu130` index was used and that the CUDA libraries are present in the ZeroGPU runtime. Do not remove the GPU check or set `n_gpu_layers=0` to hide the issue.
|
| 179 |
|
| 180 |
+
### `libcudart.so` / `libcublas.so` cannot be found
|
| 181 |
|
| 182 |
+
The CUDA runtime libraries needed by the prebuilt wheel are missing or not visible to the dynamic linker. Check the ZeroGPU base runtime and the installed PyTorch CUDA runtime. The project does not compile a local replacement.
|
|
|
|
|
|
|
| 183 |
|
| 184 |
+
### `no kernel image is available`
|
| 185 |
|
| 186 |
+
The wheel's CUDA kernels do not match the allocated GPU architecture. Do not fall back to CPU and claim GPU acceleration. A wheel built for the active GPU architecture is required.
|
| 187 |
|
| 188 |
+
## Local development
|
|
|
|
|
|
|
|
|
|
| 189 |
|
| 190 |
+
The app is designed for Hugging Face ZeroGPU. A local CPU environment is intentionally not treated as a valid production test because this build requires a working CUDA backend and refuses CPU inference.
|
app.py
CHANGED
|
@@ -1,22 +1,40 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
import logging
|
| 4 |
import os
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
from pathlib import Path
|
|
|
|
| 6 |
|
| 7 |
import gradio as gr
|
|
|
|
| 8 |
import uvicorn
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
from huggingface_hub import hf_hub_download
|
| 10 |
-
from
|
| 11 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
|
| 13 |
logging.basicConfig(
|
| 14 |
level=os.getenv("LOG_LEVEL", "INFO").upper(),
|
| 15 |
-
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
| 16 |
)
|
| 17 |
-
logger = logging.getLogger("qwen-
|
| 18 |
|
| 19 |
-
|
| 20 |
MODEL_REPO_ID = os.getenv(
|
| 21 |
"MODEL_REPO_ID", "Qwen/Qwen2.5-Coder-0.5B-Instruct-GGUF"
|
| 22 |
)
|
|
@@ -24,113 +42,436 @@ MODEL_FILENAME = os.getenv(
|
|
| 24 |
"MODEL_FILENAME", "qwen2.5-coder-0.5b-instruct-q4_k_m.gguf"
|
| 25 |
)
|
| 26 |
MODEL_ALIAS = os.getenv("MODEL_ALIAS", "qwen2.5-coder-0.5b-instruct")
|
|
|
|
| 27 |
|
| 28 |
|
| 29 |
-
def
|
| 30 |
raw = os.getenv(name, str(default))
|
| 31 |
try:
|
| 32 |
value = int(raw)
|
| 33 |
except ValueError as exc:
|
| 34 |
-
raise RuntimeError(f"{name} must be an integer
|
| 35 |
if value < 1:
|
| 36 |
-
raise RuntimeError(f"{name} must be >= 1
|
|
|
|
|
|
|
| 37 |
return value
|
| 38 |
|
| 39 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
def download_model() -> str:
|
| 41 |
-
"
|
| 42 |
-
|
| 43 |
-
model_path = hf_hub_download(
|
| 44 |
repo_id=MODEL_REPO_ID,
|
| 45 |
filename=MODEL_FILENAME,
|
| 46 |
-
cache_dir=os.getenv("HF_HUB_CACHE") or None,
|
| 47 |
token=os.getenv("HF_TOKEN") or None,
|
| 48 |
)
|
| 49 |
-
size_mb = Path(
|
| 50 |
-
logger.info("Model
|
| 51 |
-
return
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
|
| 53 |
|
| 54 |
-
def
|
| 55 |
-
|
| 56 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
raise RuntimeError(
|
| 58 |
-
"
|
| 59 |
-
"
|
|
|
|
| 60 |
)
|
| 61 |
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
n_batch=_read_positive_int("LLAMA_N_BATCH", 512),
|
| 79 |
-
n_ubatch=_read_positive_int("LLAMA_N_UBATCH", 256),
|
| 80 |
-
n_gpu_layers=0,
|
| 81 |
use_mmap=True,
|
| 82 |
use_mlock=False,
|
| 83 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 84 |
verbose=False,
|
|
|
|
|
|
|
| 85 |
)
|
| 86 |
-
api_app = create_app(settings=settings)
|
| 87 |
|
| 88 |
-
# A lightweight public health route for Space monitoring. Inference routes
|
| 89 |
-
# remain protected by the server's built-in Bearer-key dependency.
|
| 90 |
-
@api_app.get("/health", include_in_schema=False)
|
| 91 |
-
async def health():
|
| 92 |
-
return {"status": "ok", "model": MODEL_ALIAS}
|
| 93 |
|
| 94 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
|
| 96 |
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 102 |
|
| 103 |
-
The model is running through llama.cpp. This Space exposes an
|
| 104 |
-
OpenAI-compatible REST API; inference endpoints require your
|
| 105 |
-
configured Bearer API key.
|
| 106 |
|
| 107 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 108 |
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 112 |
|
| 113 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 114 |
|
| 115 |
-
- `GET /health` — public service health check
|
| 116 |
-
- `GET /v1/models` — model list (Bearer key required)
|
| 117 |
-
- `POST /v1/chat/completions` — chat and streaming completions
|
| 118 |
-
- `POST /v1/completions` — text completions
|
| 119 |
-
- `GET /docs` — OpenAPI schema
|
| 120 |
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
|
|
|
|
|
|
|
|
|
| 124 |
)
|
| 125 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 126 |
|
| 127 |
|
| 128 |
-
|
| 129 |
-
api_app = build_api_app()
|
| 130 |
-
app = gr.mount_gradio_app(api_app, build_console(), path="/console")
|
| 131 |
-
logger.info("Starting API on 0.0.0.0:%s", SPACE_PORT)
|
| 132 |
-
uvicorn.run(app, host="0.0.0.0", port=SPACE_PORT, log_level="info")
|
| 133 |
|
| 134 |
|
| 135 |
if __name__ == "__main__":
|
| 136 |
-
|
|
|
|
|
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
+
# ZeroGPU must be imported before torch or any module that can touch CUDA.
|
| 4 |
+
import spaces
|
| 5 |
+
|
| 6 |
+
import json
|
| 7 |
import logging
|
| 8 |
import os
|
| 9 |
+
import secrets
|
| 10 |
+
import threading
|
| 11 |
+
import time
|
| 12 |
+
from functools import lru_cache
|
| 13 |
from pathlib import Path
|
| 14 |
+
from typing import Any, Iterator, Optional, Union
|
| 15 |
|
| 16 |
import gradio as gr
|
| 17 |
+
import torch # ZeroGPU's Gradio image provides its matching CUDA-enabled PyTorch.
|
| 18 |
import uvicorn
|
| 19 |
+
from fastapi import Depends, FastAPI, Header, HTTPException, Request
|
| 20 |
+
from fastapi.middleware.cors import CORSMiddleware
|
| 21 |
+
from fastapi.responses import RedirectResponse, StreamingResponse
|
| 22 |
+
from gradio_client import Client
|
| 23 |
from huggingface_hub import hf_hub_download
|
| 24 |
+
from pydantic import BaseModel, ConfigDict, Field
|
| 25 |
+
|
| 26 |
+
# Import llama.cpp after the ZeroGPU bootstrap and the platform CUDA runtime.
|
| 27 |
+
import llama_cpp
|
| 28 |
+
from llama_cpp import Llama, llama_supports_gpu_offload
|
| 29 |
+
|
| 30 |
|
| 31 |
logging.basicConfig(
|
| 32 |
level=os.getenv("LOG_LEVEL", "INFO").upper(),
|
| 33 |
+
format="%(asctime)s %(levelname)s %(process)d %(name)s: %(message)s",
|
| 34 |
)
|
| 35 |
+
logger = logging.getLogger("qwen-zerogpu-api")
|
| 36 |
|
| 37 |
+
PORT = int(os.getenv("PORT", "7860"))
|
| 38 |
MODEL_REPO_ID = os.getenv(
|
| 39 |
"MODEL_REPO_ID", "Qwen/Qwen2.5-Coder-0.5B-Instruct-GGUF"
|
| 40 |
)
|
|
|
|
| 42 |
"MODEL_FILENAME", "qwen2.5-coder-0.5b-instruct-q4_k_m.gguf"
|
| 43 |
)
|
| 44 |
MODEL_ALIAS = os.getenv("MODEL_ALIAS", "qwen2.5-coder-0.5b-instruct")
|
| 45 |
+
API_KEY = os.getenv("LLAMA_API_KEY", "").strip()
|
| 46 |
|
| 47 |
|
| 48 |
+
def positive_int(name: str, default: int, maximum: Optional[int] = None) -> int:
|
| 49 |
raw = os.getenv(name, str(default))
|
| 50 |
try:
|
| 51 |
value = int(raw)
|
| 52 |
except ValueError as exc:
|
| 53 |
+
raise RuntimeError(f"{name} must be an integer, got {raw!r}") from exc
|
| 54 |
if value < 1:
|
| 55 |
+
raise RuntimeError(f"{name} must be >= 1")
|
| 56 |
+
if maximum is not None and value > maximum:
|
| 57 |
+
raise RuntimeError(f"{name} must be <= {maximum}")
|
| 58 |
return value
|
| 59 |
|
| 60 |
|
| 61 |
+
N_CTX = positive_int("LLAMA_N_CTX", 4096, 32768)
|
| 62 |
+
N_BATCH = positive_int("LLAMA_N_BATCH", 1024, 4096)
|
| 63 |
+
N_UBATCH = positive_int("LLAMA_N_UBATCH", 512, N_BATCH)
|
| 64 |
+
N_THREADS = positive_int("LLAMA_N_THREADS", 4, 64)
|
| 65 |
+
MAX_TOKENS = positive_int("LLAMA_MAX_TOKENS", 2048, 8192)
|
| 66 |
+
GPU_DURATION_MAX = positive_int("LLAMA_GPU_DURATION_MAX", 60, 120)
|
| 67 |
+
|
| 68 |
+
# This cache is created only inside a ZeroGPU worker. It is not initialized in
|
| 69 |
+
# the main web process, which has no physical GPU allocation.
|
| 70 |
+
_MODEL: Optional[Llama] = None
|
| 71 |
+
_MODEL_PID: Optional[int] = None
|
| 72 |
+
|
| 73 |
+
|
| 74 |
def download_model() -> str:
|
| 75 |
+
logger.info("Downloading/checking model: %s / %s", MODEL_REPO_ID, MODEL_FILENAME)
|
| 76 |
+
path = hf_hub_download(
|
|
|
|
| 77 |
repo_id=MODEL_REPO_ID,
|
| 78 |
filename=MODEL_FILENAME,
|
|
|
|
| 79 |
token=os.getenv("HF_TOKEN") or None,
|
| 80 |
)
|
| 81 |
+
size_mb = Path(path).stat().st_size / (1024 * 1024)
|
| 82 |
+
logger.info("Model file available: %s (%.1f MiB)", path, size_mb)
|
| 83 |
+
return path
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
# Download the model once into the Hub cache. This step does not request a GPU.
|
| 87 |
+
MODEL_PATH = download_model()
|
| 88 |
|
| 89 |
|
| 90 |
+
def get_model() -> Llama:
|
| 91 |
+
"""Load/cache llama.cpp in the active ZeroGPU worker process."""
|
| 92 |
+
global _MODEL, _MODEL_PID
|
| 93 |
+
|
| 94 |
+
pid = os.getpid()
|
| 95 |
+
if _MODEL is not None and _MODEL_PID == pid:
|
| 96 |
+
return _MODEL
|
| 97 |
+
|
| 98 |
+
if not llama_supports_gpu_offload():
|
| 99 |
raise RuntimeError(
|
| 100 |
+
"The installed llama.cpp wheel has no usable CUDA backend in this "
|
| 101 |
+
"ZeroGPU worker. CPU fallback is disabled. Check the CUDA wheel, "
|
| 102 |
+
"runtime libraries, and ZeroGPU compatibility."
|
| 103 |
)
|
| 104 |
|
| 105 |
+
logger.info(
|
| 106 |
+
"Initializing llama.cpp with full CUDA layer offload (n_gpu_layers=-1), pid=%s",
|
| 107 |
+
pid,
|
| 108 |
+
)
|
| 109 |
+
started = time.perf_counter()
|
| 110 |
+
model = Llama(
|
| 111 |
+
model_path=MODEL_PATH,
|
| 112 |
+
n_gpu_layers=-1,
|
| 113 |
+
main_gpu=0,
|
| 114 |
+
n_ctx=N_CTX,
|
| 115 |
+
n_batch=N_BATCH,
|
| 116 |
+
n_ubatch=N_UBATCH,
|
| 117 |
+
n_threads=N_THREADS,
|
| 118 |
+
n_threads_batch=N_THREADS,
|
| 119 |
+
offload_kqv=True,
|
| 120 |
+
flash_attn=True,
|
|
|
|
|
|
|
|
|
|
| 121 |
use_mmap=True,
|
| 122 |
use_mlock=False,
|
| 123 |
+
verbose=os.getenv("LLAMA_VERBOSE", "1") == "1",
|
| 124 |
+
)
|
| 125 |
+
_MODEL = model
|
| 126 |
+
_MODEL_PID = pid
|
| 127 |
+
logger.info(
|
| 128 |
+
"CUDA model initialized in %.2fs (worker pid=%s)",
|
| 129 |
+
time.perf_counter() - started,
|
| 130 |
+
pid,
|
| 131 |
+
)
|
| 132 |
+
return model
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def gpu_duration(payload: dict[str, Any], api_key: str) -> int:
|
| 136 |
+
"""Bound each ZeroGPU reservation and keep the default request short."""
|
| 137 |
+
try:
|
| 138 |
+
requested = int(payload.get("max_tokens") or 512)
|
| 139 |
+
except (TypeError, ValueError):
|
| 140 |
+
requested = 512
|
| 141 |
+
requested = max(1, min(requested, MAX_TOKENS))
|
| 142 |
+
# Includes a cold worker/model-load allowance plus output-generation time.
|
| 143 |
+
return min(GPU_DURATION_MAX, max(20, 15 + int(requested * 0.04)))
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def verify_worker_key(api_key: str) -> None:
|
| 147 |
+
if not api_key or not secrets.compare_digest(str(api_key), API_KEY):
|
| 148 |
+
raise PermissionError("Invalid API key")
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def chat_kwargs(payload: dict[str, Any]) -> dict[str, Any]:
|
| 152 |
+
max_tokens = int(payload.get("max_tokens") or 512)
|
| 153 |
+
if max_tokens < 1 or max_tokens > MAX_TOKENS:
|
| 154 |
+
raise ValueError(f"max_tokens must be between 1 and {MAX_TOKENS}")
|
| 155 |
+
kwargs: dict[str, Any] = {
|
| 156 |
+
"temperature": float(payload.get("temperature", 0.2)),
|
| 157 |
+
"top_p": float(payload.get("top_p", 0.95)),
|
| 158 |
+
"max_tokens": max_tokens,
|
| 159 |
+
"repeat_penalty": float(payload.get("repeat_penalty", 1.05)),
|
| 160 |
+
"presence_penalty": float(payload.get("presence_penalty", 0.0)),
|
| 161 |
+
"frequency_penalty": float(payload.get("frequency_penalty", 0.0)),
|
| 162 |
+
"stream": bool(payload.get("stream", False)),
|
| 163 |
+
"model": MODEL_ALIAS,
|
| 164 |
+
}
|
| 165 |
+
if payload.get("stop") is not None:
|
| 166 |
+
kwargs["stop"] = payload["stop"]
|
| 167 |
+
if payload.get("seed") is not None:
|
| 168 |
+
kwargs["seed"] = int(payload["seed"])
|
| 169 |
+
return kwargs
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
# These functions are intentionally the functions bound to Gradio events.
|
| 173 |
+
# ZeroGPU discovers the decorated callbacks through Gradio's event registry.
|
| 174 |
+
@spaces.GPU(duration=gpu_duration)
|
| 175 |
+
def gpu_chat_completion(payload: dict[str, Any], api_key: str) -> dict[str, Any]:
|
| 176 |
+
verify_worker_key(api_key)
|
| 177 |
+
model = get_model()
|
| 178 |
+
started = time.perf_counter()
|
| 179 |
+
result = model.create_chat_completion(
|
| 180 |
+
messages=payload["messages"],
|
| 181 |
+
**chat_kwargs(payload),
|
| 182 |
+
)
|
| 183 |
+
elapsed = max(time.perf_counter() - started, 1e-6)
|
| 184 |
+
usage = result.get("usage", {})
|
| 185 |
+
completion_tokens = int(usage.get("completion_tokens", 0))
|
| 186 |
+
logger.info(
|
| 187 |
+
"Chat request finished: %s output tokens in %.3fs (%.1f tok/s)",
|
| 188 |
+
completion_tokens,
|
| 189 |
+
elapsed,
|
| 190 |
+
completion_tokens / elapsed,
|
| 191 |
+
)
|
| 192 |
+
return result
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
@spaces.GPU(duration=gpu_duration)
|
| 196 |
+
def gpu_chat_stream(payload: dict[str, Any], api_key: str) -> Iterator[str]:
|
| 197 |
+
verify_worker_key(api_key)
|
| 198 |
+
try:
|
| 199 |
+
model = get_model()
|
| 200 |
+
kwargs = chat_kwargs(payload)
|
| 201 |
+
kwargs["stream"] = True
|
| 202 |
+
for chunk in model.create_chat_completion(
|
| 203 |
+
messages=payload["messages"],
|
| 204 |
+
**kwargs,
|
| 205 |
+
):
|
| 206 |
+
yield "data: " + json.dumps(chunk, ensure_ascii=False) + "\n\n"
|
| 207 |
+
yield "data: [DONE]\n\n"
|
| 208 |
+
except Exception as exc:
|
| 209 |
+
logger.exception("Streaming inference failed")
|
| 210 |
+
error = {"error": {"message": str(exc), "type": "server_error"}}
|
| 211 |
+
yield "data: " + json.dumps(error, ensure_ascii=False) + "\n\n"
|
| 212 |
+
yield "data: [DONE]\n\n"
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
@spaces.GPU(duration=gpu_duration)
|
| 216 |
+
def gpu_text_completion(payload: dict[str, Any], api_key: str) -> dict[str, Any]:
|
| 217 |
+
verify_worker_key(api_key)
|
| 218 |
+
model = get_model()
|
| 219 |
+
max_tokens = int(payload.get("max_tokens") or 512)
|
| 220 |
+
if max_tokens < 1 or max_tokens > MAX_TOKENS:
|
| 221 |
+
raise ValueError(f"max_tokens must be between 1 and {MAX_TOKENS}")
|
| 222 |
+
kwargs: dict[str, Any] = {
|
| 223 |
+
"prompt": payload["prompt"],
|
| 224 |
+
"temperature": float(payload.get("temperature", 0.2)),
|
| 225 |
+
"top_p": float(payload.get("top_p", 0.95)),
|
| 226 |
+
"max_tokens": max_tokens,
|
| 227 |
+
"repeat_penalty": float(payload.get("repeat_penalty", 1.05)),
|
| 228 |
+
"stream": False,
|
| 229 |
+
"model": MODEL_ALIAS,
|
| 230 |
+
}
|
| 231 |
+
if payload.get("stop") is not None:
|
| 232 |
+
kwargs["stop"] = payload["stop"]
|
| 233 |
+
if payload.get("seed") is not None:
|
| 234 |
+
kwargs["seed"] = int(payload["seed"])
|
| 235 |
+
return model.create_completion(**kwargs)
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
class ChatRequest(BaseModel):
|
| 239 |
+
model: str = MODEL_ALIAS
|
| 240 |
+
messages: list[dict[str, Any]]
|
| 241 |
+
temperature: float = Field(default=0.2, ge=0, le=2)
|
| 242 |
+
top_p: float = Field(default=0.95, gt=0, le=1)
|
| 243 |
+
max_tokens: int = Field(default=512, ge=1, le=MAX_TOKENS)
|
| 244 |
+
stream: bool = False
|
| 245 |
+
stop: Optional[Union[str, list[str]]] = None
|
| 246 |
+
presence_penalty: float = Field(default=0.0, ge=-2, le=2)
|
| 247 |
+
frequency_penalty: float = Field(default=0.0, ge=-2, le=2)
|
| 248 |
+
repeat_penalty: float = Field(default=1.05, ge=0.5, le=2)
|
| 249 |
+
seed: Optional[int] = None
|
| 250 |
+
model_config = ConfigDict(extra="ignore")
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
class CompletionRequest(BaseModel):
|
| 254 |
+
model: str = MODEL_ALIAS
|
| 255 |
+
prompt: str
|
| 256 |
+
temperature: float = Field(default=0.2, ge=0, le=2)
|
| 257 |
+
top_p: float = Field(default=0.95, gt=0, le=1)
|
| 258 |
+
max_tokens: int = Field(default=512, ge=1, le=MAX_TOKENS)
|
| 259 |
+
stream: bool = False
|
| 260 |
+
stop: Optional[Union[str, list[str]]] = None
|
| 261 |
+
repeat_penalty: float = Field(default=1.05, ge=0.5, le=2)
|
| 262 |
+
seed: Optional[int] = None
|
| 263 |
+
model_config = ConfigDict(extra="ignore")
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
if not API_KEY:
|
| 267 |
+
raise RuntimeError(
|
| 268 |
+
"Missing required Space Secret LLAMA_API_KEY. Add it under "
|
| 269 |
+
"Settings → Variables and secrets → Secrets."
|
| 270 |
+
)
|
| 271 |
+
|
| 272 |
+
api = FastAPI(
|
| 273 |
+
title="Qwen2.5 Coder ZeroGPU API",
|
| 274 |
+
version="2.1.0",
|
| 275 |
+
description=(
|
| 276 |
+
"OpenAI-compatible REST API backed by Qwen2.5-Coder-0.5B-Instruct "
|
| 277 |
+
"Q4_K_M through llama.cpp CUDA on Hugging Face ZeroGPU."
|
| 278 |
+
),
|
| 279 |
+
)
|
| 280 |
+
api.add_middleware(
|
| 281 |
+
CORSMiddleware,
|
| 282 |
+
allow_origins=["*"],
|
| 283 |
+
allow_credentials=False,
|
| 284 |
+
allow_methods=["GET", "POST", "OPTIONS"],
|
| 285 |
+
allow_headers=["Authorization", "Content-Type", "X-IP-Token"],
|
| 286 |
+
)
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
def require_api_key(authorization: Optional[str] = Header(default=None)) -> None:
|
| 290 |
+
if not authorization or not authorization.startswith("Bearer "):
|
| 291 |
+
raise HTTPException(status_code=401, detail="Missing Bearer API key")
|
| 292 |
+
supplied = authorization[7:].strip()
|
| 293 |
+
if not secrets.compare_digest(supplied, API_KEY):
|
| 294 |
+
raise HTTPException(status_code=401, detail="Invalid API key")
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
@lru_cache(maxsize=1)
|
| 298 |
+
def get_internal_gradio_client() -> Client:
|
| 299 |
+
"""Connect to this Space's Gradio queue so ZeroGPU schedules the callback."""
|
| 300 |
+
url = f"http://127.0.0.1:{PORT}/console"
|
| 301 |
+
logger.info("Connecting internal Gradio client to %s", url)
|
| 302 |
+
return Client(
|
| 303 |
+
url,
|
| 304 |
verbose=False,
|
| 305 |
+
analytics_enabled=False,
|
| 306 |
+
max_workers=8,
|
| 307 |
)
|
|
|
|
| 308 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 309 |
|
| 310 |
+
def submit_gpu_job(
|
| 311 |
+
api_name: str,
|
| 312 |
+
payload: dict[str, Any],
|
| 313 |
+
request: Request,
|
| 314 |
+
):
|
| 315 |
+
# Hugging Face adds this caller-specific token at the Space ingress. Forward
|
| 316 |
+
# it through the internal Gradio request so ZeroGPU can attribute quota to
|
| 317 |
+
# the real caller instead of treating every proxied request as anonymous.
|
| 318 |
+
incoming_token = request.headers.get("x-ip-token")
|
| 319 |
+
headers = {"X-IP-Token": incoming_token} if incoming_token else None
|
| 320 |
+
client = get_internal_gradio_client()
|
| 321 |
+
return client.submit(payload, API_KEY, api_name=api_name, headers=headers)
|
| 322 |
|
| 323 |
|
| 324 |
+
@api.get("/health")
|
| 325 |
+
def health() -> dict[str, Any]:
|
| 326 |
+
return {
|
| 327 |
+
"status": "ok",
|
| 328 |
+
"model": MODEL_ALIAS,
|
| 329 |
+
"quantization": "Q4_K_M",
|
| 330 |
+
"runtime": "llama.cpp",
|
| 331 |
+
"gpu": "ZeroGPU on-demand CUDA",
|
| 332 |
+
"cuda_backend_probe": "deferred until a GPU worker is allocated",
|
| 333 |
+
"model_downloaded": Path(MODEL_PATH).is_file(),
|
| 334 |
+
"model_loaded": "inside a ZeroGPU worker on first inference",
|
| 335 |
+
"gpu_offload_required": True,
|
| 336 |
+
}
|
| 337 |
|
|
|
|
|
|
|
|
|
|
| 338 |
|
| 339 |
+
@api.get("/v1/models", dependencies=[Depends(require_api_key)])
|
| 340 |
+
def list_models() -> dict[str, Any]:
|
| 341 |
+
return {
|
| 342 |
+
"object": "list",
|
| 343 |
+
"data": [
|
| 344 |
+
{
|
| 345 |
+
"id": MODEL_ALIAS,
|
| 346 |
+
"object": "model",
|
| 347 |
+
"created": 0,
|
| 348 |
+
"owned_by": "qwen",
|
| 349 |
+
}
|
| 350 |
+
],
|
| 351 |
+
}
|
| 352 |
+
|
| 353 |
+
|
| 354 |
+
@api.post("/v1/chat/completions", dependencies=[Depends(require_api_key)])
|
| 355 |
+
def chat_completions(body: ChatRequest, request: Request):
|
| 356 |
+
if not body.messages:
|
| 357 |
+
raise HTTPException(status_code=422, detail="messages must contain at least one item")
|
| 358 |
+
payload = body.model_dump()
|
| 359 |
+
try:
|
| 360 |
+
if body.stream:
|
| 361 |
+
job = submit_gpu_job("/gpu_chat_stream", payload, request)
|
| 362 |
+
|
| 363 |
+
def stream_job() -> Iterator[str]:
|
| 364 |
+
try:
|
| 365 |
+
for update in job:
|
| 366 |
+
if isinstance(update, str):
|
| 367 |
+
yield update
|
| 368 |
+
except Exception as exc:
|
| 369 |
+
logger.exception("ZeroGPU streaming job failed")
|
| 370 |
+
error = {"error": {"message": str(exc), "type": "server_error"}}
|
| 371 |
+
yield "data: " + json.dumps(error, ensure_ascii=False) + "\n\n"
|
| 372 |
+
yield "data: [DONE]\n\n"
|
| 373 |
|
| 374 |
+
return StreamingResponse(
|
| 375 |
+
stream_job(),
|
| 376 |
+
media_type="text/event-stream",
|
| 377 |
+
headers={
|
| 378 |
+
"Cache-Control": "no-cache",
|
| 379 |
+
"Connection": "keep-alive",
|
| 380 |
+
"X-Accel-Buffering": "no",
|
| 381 |
+
},
|
| 382 |
+
)
|
| 383 |
|
| 384 |
+
job = submit_gpu_job("/gpu_chat_completion", payload, request)
|
| 385 |
+
return job.result()
|
| 386 |
+
except HTTPException:
|
| 387 |
+
raise
|
| 388 |
+
except Exception as exc:
|
| 389 |
+
logger.exception("Chat completion failed")
|
| 390 |
+
raise HTTPException(status_code=503, detail=f"Inference unavailable: {exc}") from exc
|
| 391 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 392 |
|
| 393 |
+
@api.post("/v1/completions", dependencies=[Depends(require_api_key)])
|
| 394 |
+
def completions(body: CompletionRequest, request: Request):
|
| 395 |
+
if body.stream:
|
| 396 |
+
raise HTTPException(
|
| 397 |
+
status_code=400,
|
| 398 |
+
detail="Streaming /v1/completions is not enabled; use /v1/chat/completions",
|
| 399 |
)
|
| 400 |
+
try:
|
| 401 |
+
job = submit_gpu_job("/gpu_text_completion", body.model_dump(), request)
|
| 402 |
+
return job.result()
|
| 403 |
+
except Exception as exc:
|
| 404 |
+
logger.exception("Text completion failed")
|
| 405 |
+
raise HTTPException(status_code=503, detail=f"Inference unavailable: {exc}") from exc
|
| 406 |
+
|
| 407 |
+
|
| 408 |
+
# Register each decorated GPU function as a Gradio event. The named Gradio API
|
| 409 |
+
# is also protected by the same key inside the worker, so it cannot be used to
|
| 410 |
+
# bypass authentication. FastAPI routes above proxy into this queue.
|
| 411 |
+
with gr.Blocks(title="Qwen2.5 Coder API", analytics_enabled=False) as demo:
|
| 412 |
+
gr.Markdown(
|
| 413 |
+
"""
|
| 414 |
+
# ⚡ Qwen2.5-Coder-0.5B-Instruct REST API
|
| 415 |
+
|
| 416 |
+
**Runtime:** llama.cpp · CUDA · Hugging Face ZeroGPU
|
| 417 |
+
**Quantization:** Q4_K_M
|
| 418 |
+
**REST base:** `/v1`
|
| 419 |
+
|
| 420 |
+
All inference routes require the `LLAMA_API_KEY` Space Secret as a
|
| 421 |
+
Bearer token. Use `/docs` for the OpenAPI schema.
|
| 422 |
+
"""
|
| 423 |
+
)
|
| 424 |
+
|
| 425 |
+
with gr.Row(visible=False):
|
| 426 |
+
chat_payload = gr.JSON(value={"messages": [{"role": "user", "content": "ping"}]})
|
| 427 |
+
chat_key = gr.Textbox(value="", type="password")
|
| 428 |
+
chat_result = gr.JSON()
|
| 429 |
+
chat_trigger = gr.Button("Register chat worker")
|
| 430 |
+
chat_trigger.click(
|
| 431 |
+
fn=gpu_chat_completion,
|
| 432 |
+
inputs=[chat_payload, chat_key],
|
| 433 |
+
outputs=chat_result,
|
| 434 |
+
api_name="gpu_chat_completion",
|
| 435 |
+
)
|
| 436 |
+
|
| 437 |
+
with gr.Row(visible=False):
|
| 438 |
+
stream_payload = gr.JSON(value={"messages": [{"role": "user", "content": "ping"}]})
|
| 439 |
+
stream_key = gr.Textbox(value="", type="password")
|
| 440 |
+
stream_result = gr.Textbox()
|
| 441 |
+
stream_trigger = gr.Button("Register streaming worker")
|
| 442 |
+
stream_trigger.click(
|
| 443 |
+
fn=gpu_chat_stream,
|
| 444 |
+
inputs=[stream_payload, stream_key],
|
| 445 |
+
outputs=stream_result,
|
| 446 |
+
api_name="gpu_chat_stream",
|
| 447 |
+
)
|
| 448 |
+
|
| 449 |
+
with gr.Row(visible=False):
|
| 450 |
+
completion_payload = gr.JSON(value={"prompt": "ping"})
|
| 451 |
+
completion_key = gr.Textbox(value="", type="password")
|
| 452 |
+
completion_result = gr.JSON()
|
| 453 |
+
completion_trigger = gr.Button("Register completion worker")
|
| 454 |
+
completion_trigger.click(
|
| 455 |
+
fn=gpu_text_completion,
|
| 456 |
+
inputs=[completion_payload, completion_key],
|
| 457 |
+
outputs=completion_result,
|
| 458 |
+
api_name="gpu_text_completion",
|
| 459 |
+
)
|
| 460 |
+
|
| 461 |
+
# Use the Gradio queue for serialized requests; the ZeroGPU scheduler allocates
|
| 462 |
+
# a GPU for the decorated event callbacks.
|
| 463 |
+
demo.queue(default_concurrency_limit=1, max_size=32)
|
| 464 |
+
|
| 465 |
+
|
| 466 |
+
@api.get("/", include_in_schema=False)
|
| 467 |
+
def root():
|
| 468 |
+
return RedirectResponse(url="/console")
|
| 469 |
|
| 470 |
|
| 471 |
+
app = gr.mount_gradio_app(api, demo, path="/console")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 472 |
|
| 473 |
|
| 474 |
if __name__ == "__main__":
|
| 475 |
+
logger.info("llama-cpp-python version: %s", getattr(llama_cpp, "__version__", "unknown"))
|
| 476 |
+
logger.info("CUDA initialization and model load are deferred to ZeroGPU workers.")
|
| 477 |
+
uvicorn.run(app, host="0.0.0.0", port=PORT, log_level=os.getenv("LOG_LEVEL", "info").lower())
|
requirements.txt
CHANGED
|
@@ -1,3 +1,5 @@
|
|
| 1 |
-
|
| 2 |
-
llama
|
| 3 |
-
|
|
|
|
|
|
|
|
|
| 1 |
+
# Install only the prebuilt CUDA wheel. If no compatible wheel is available,
|
| 2 |
+
# pip must fail rather than spending Space build time compiling llama.cpp.
|
| 3 |
+
--only-binary=llama-cpp-python
|
| 4 |
+
--extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu130
|
| 5 |
+
llama-cpp-python==0.3.36
|