mhd-radin's picture
Upload 5 files
856bb3a verified
Raw History Blame Contribute Delete
18.9 kB
from __future__ import annotations
import os
# Gradio 5/6 may start a Node SSR proxy that reserves the public Space port
# (7860). This app serves a custom FastAPI + mounted Gradio application with
# Uvicorn directly, so disable that proxy before importing Gradio/spaces.
# Otherwise Uvicorn fails with: [Errno 98] address already in use.
os.environ["GRADIO_SSR_MODE"] = "false"
# ZeroGPU must be imported before torch or any module that can touch CUDA.
import spaces
import json
import logging
import secrets
import threading
import time
from functools import lru_cache
from pathlib import Path
from typing import Any, Iterator, Optional, Union
import gradio as gr
import torch # ZeroGPU's Gradio image provides its matching CUDA-enabled PyTorch.
import uvicorn
from fastapi import Depends, FastAPI, Header, HTTPException, Request
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import RedirectResponse, StreamingResponse
from gradio_client import Client
from huggingface_hub import HfApi, hf_hub_download
from pydantic import BaseModel, ConfigDict, Field
# Import llama.cpp after the ZeroGPU bootstrap and the platform CUDA runtime.
import llama_cpp
from llama_cpp import Llama, llama_supports_gpu_offload
logging.basicConfig(
level=os.getenv("LOG_LEVEL", "INFO").upper(),
format="%(asctime)s %(levelname)s %(process)d %(name)s: %(message)s",
)
logger = logging.getLogger("qwen-zerogpu-api")
PORT = int(os.getenv("PORT", "7860"))
MODEL_REPO_ID = os.getenv(
"MODEL_REPO_ID", "Qwen/Qwen2.5-Coder-0.5B-Instruct-GGUF"
)
MODEL_FILENAME = os.getenv(
"MODEL_FILENAME", "qwen2.5-coder-0.5b-instruct-q4_k_m.gguf"
)
MODEL_ALIAS = os.getenv("MODEL_ALIAS", "qwen2.5-coder-0.5b-instruct")
API_KEY = os.getenv("LLAMA_API_KEY", "").strip()
HF_TOKEN = os.getenv("HF_TOKEN", "").strip()
SPACE_HOST = os.getenv("SPACE_HOST", "").strip()
def positive_int(name: str, default: int, maximum: Optional[int] = None) -> int:
raw = os.getenv(name, str(default))
try:
value = int(raw)
except ValueError as exc:
raise RuntimeError(f"{name} must be an integer, got {raw!r}") from exc
if value < 1:
raise RuntimeError(f"{name} must be >= 1")
if maximum is not None and value > maximum:
raise RuntimeError(f"{name} must be <= {maximum}")
return value
N_CTX = positive_int("LLAMA_N_CTX", 4096, 32768)
N_BATCH = positive_int("LLAMA_N_BATCH", 1024, 4096)
N_UBATCH = positive_int("LLAMA_N_UBATCH", 512, N_BATCH)
N_THREADS = positive_int("LLAMA_N_THREADS", 4, 64)
MAX_TOKENS = positive_int("LLAMA_MAX_TOKENS", 2048, 8192)
GPU_DURATION_MAX = positive_int("LLAMA_GPU_DURATION_MAX", 60, 120)
# This cache is created only inside a ZeroGPU worker. It is not initialized in
# the main web process, which has no physical GPU allocation.
_MODEL: Optional[Llama] = None
_MODEL_PID: Optional[int] = None
def download_model() -> str:
logger.info("Downloading/checking model: %s / %s", MODEL_REPO_ID, MODEL_FILENAME)
path = hf_hub_download(
repo_id=MODEL_REPO_ID,
filename=MODEL_FILENAME,
token=HF_TOKEN,
)
size_mb = Path(path).stat().st_size / (1024 * 1024)
logger.info("Model file available: %s (%.1f MiB)", path, size_mb)
return path
# Download the model once into the Hub cache. This step does not request a GPU.
MODEL_PATH = download_model()
def get_model() -> Llama:
"""Load/cache llama.cpp in the active ZeroGPU worker process."""
global _MODEL, _MODEL_PID
pid = os.getpid()
if _MODEL is not None and _MODEL_PID == pid:
return _MODEL
if not llama_supports_gpu_offload():
raise RuntimeError(
"The installed llama.cpp wheel has no usable CUDA backend in this "
"ZeroGPU worker. CPU fallback is disabled. Check the CUDA wheel, "
"runtime libraries, and ZeroGPU compatibility."
)
logger.info(
"Initializing llama.cpp with full CUDA layer offload (n_gpu_layers=-1), pid=%s",
pid,
)
started = time.perf_counter()
model = Llama(
model_path=MODEL_PATH,
n_gpu_layers=-1,
main_gpu=0,
n_ctx=N_CTX,
n_batch=N_BATCH,
n_ubatch=N_UBATCH,
n_threads=N_THREADS,
n_threads_batch=N_THREADS,
offload_kqv=True,
flash_attn=True,
use_mmap=True,
use_mlock=False,
verbose=os.getenv("LLAMA_VERBOSE", "1") == "1",
)
_MODEL = model
_MODEL_PID = pid
logger.info(
"CUDA model initialized in %.2fs (worker pid=%s)",
time.perf_counter() - started,
pid,
)
return model
def gpu_duration(payload: dict[str, Any], api_key: str) -> int:
"""Bound each ZeroGPU reservation and keep the default request short."""
try:
requested = int(payload.get("max_tokens") or 512)
except (TypeError, ValueError):
requested = 512
requested = max(1, min(requested, MAX_TOKENS))
# Keep ZeroGPU's up-front reservation tight for this small 0.5B model.
# The reservation must still leave room for CUDA worker startup/model load.
# 256 tokens -> 15s, 512 -> 20s, 1024 -> 30s, 2048 -> 50s.
return min(GPU_DURATION_MAX, max(15, 10 + int(requested * 0.02)))
def verify_worker_key(api_key: str) -> None:
if not api_key or not secrets.compare_digest(str(api_key), API_KEY):
raise PermissionError("Invalid API key")
def chat_kwargs(payload: dict[str, Any]) -> dict[str, Any]:
max_tokens = int(payload.get("max_tokens") or 512)
if max_tokens < 1 or max_tokens > MAX_TOKENS:
raise ValueError(f"max_tokens must be between 1 and {MAX_TOKENS}")
kwargs: dict[str, Any] = {
"temperature": float(payload.get("temperature", 0.2)),
"top_p": float(payload.get("top_p", 0.95)),
"max_tokens": max_tokens,
"repeat_penalty": float(payload.get("repeat_penalty", 1.05)),
"presence_penalty": float(payload.get("presence_penalty", 0.0)),
"frequency_penalty": float(payload.get("frequency_penalty", 0.0)),
"stream": bool(payload.get("stream", False)),
"model": MODEL_ALIAS,
}
if payload.get("stop") is not None:
kwargs["stop"] = payload["stop"]
if payload.get("seed") is not None:
kwargs["seed"] = int(payload["seed"])
return kwargs
# These functions are intentionally the functions bound to Gradio events.
# ZeroGPU discovers the decorated callbacks through Gradio's event registry.
@spaces.GPU(duration=gpu_duration)
def gpu_chat_completion(payload: dict[str, Any], api_key: str) -> dict[str, Any]:
verify_worker_key(api_key)
model = get_model()
started = time.perf_counter()
result = model.create_chat_completion(
messages=payload["messages"],
**chat_kwargs(payload),
)
elapsed = max(time.perf_counter() - started, 1e-6)
usage = result.get("usage", {})
completion_tokens = int(usage.get("completion_tokens", 0))
logger.info(
"Chat request finished: %s output tokens in %.3fs (%.1f tok/s)",
completion_tokens,
elapsed,
completion_tokens / elapsed,
)
return result
@spaces.GPU(duration=gpu_duration)
def gpu_chat_stream(payload: dict[str, Any], api_key: str) -> Iterator[str]:
verify_worker_key(api_key)
try:
model = get_model()
kwargs = chat_kwargs(payload)
kwargs["stream"] = True
for chunk in model.create_chat_completion(
messages=payload["messages"],
**kwargs,
):
yield "data: " + json.dumps(chunk, ensure_ascii=False) + "\n\n"
yield "data: [DONE]\n\n"
except Exception as exc:
logger.exception("Streaming inference failed")
error = {"error": {"message": str(exc), "type": "server_error"}}
yield "data: " + json.dumps(error, ensure_ascii=False) + "\n\n"
yield "data: [DONE]\n\n"
@spaces.GPU(duration=gpu_duration)
def gpu_text_completion(payload: dict[str, Any], api_key: str) -> dict[str, Any]:
verify_worker_key(api_key)
model = get_model()
max_tokens = int(payload.get("max_tokens") or 512)
if max_tokens < 1 or max_tokens > MAX_TOKENS:
raise ValueError(f"max_tokens must be between 1 and {MAX_TOKENS}")
kwargs: dict[str, Any] = {
"prompt": payload["prompt"],
"temperature": float(payload.get("temperature", 0.2)),
"top_p": float(payload.get("top_p", 0.95)),
"max_tokens": max_tokens,
"repeat_penalty": float(payload.get("repeat_penalty", 1.05)),
"stream": False,
"model": MODEL_ALIAS,
}
if payload.get("stop") is not None:
kwargs["stop"] = payload["stop"]
if payload.get("seed") is not None:
kwargs["seed"] = int(payload["seed"])
return model.create_completion(**kwargs)
class ChatRequest(BaseModel):
model: str = MODEL_ALIAS
messages: list[dict[str, Any]]
temperature: float = Field(default=0.2, ge=0, le=2)
top_p: float = Field(default=0.95, gt=0, le=1)
max_tokens: int = Field(default=256, ge=1, le=MAX_TOKENS)
stream: bool = False
stop: Optional[Union[str, list[str]]] = None
presence_penalty: float = Field(default=0.0, ge=-2, le=2)
frequency_penalty: float = Field(default=0.0, ge=-2, le=2)
repeat_penalty: float = Field(default=1.05, ge=0.5, le=2)
seed: Optional[int] = None
model_config = ConfigDict(extra="ignore")
class CompletionRequest(BaseModel):
model: str = MODEL_ALIAS
prompt: str
temperature: float = Field(default=0.2, ge=0, le=2)
top_p: float = Field(default=0.95, gt=0, le=1)
max_tokens: int = Field(default=256, ge=1, le=MAX_TOKENS)
stream: bool = False
stop: Optional[Union[str, list[str]]] = None
repeat_penalty: float = Field(default=1.05, ge=0.5, le=2)
seed: Optional[int] = None
model_config = ConfigDict(extra="ignore")
if not API_KEY:
raise RuntimeError(
"Missing required Space Secret LLAMA_API_KEY. Add it under "
"Settings → Variables and secrets → Secrets."
)
if not HF_TOKEN:
raise RuntimeError(
"Missing required Space Secret HF_TOKEN. Create a Hugging Face user access "
"token with read access and add it under Settings → Variables and secrets → Secrets."
)
if not SPACE_HOST:
raise RuntimeError(
"Missing Hugging Face SPACE_HOST environment variable. This app must run "
"inside a Hugging Face Space so its authenticated public gateway can be used."
)
# Validate the configured user token before accepting API traffic. Never log the token.
try:
HfApi(token=HF_TOKEN).whoami()
HF_TOKEN_VALID = True
except Exception as exc:
raise RuntimeError(
"HF_TOKEN is not a valid Hugging Face user access token. Create a token "
"with read access at https://huggingface.co/settings/tokens and replace "
"the Space Secret."
) from exc
api = FastAPI(
title="Qwen2.5 Coder ZeroGPU API",
version="2.2.0",
description=(
"OpenAI-compatible REST API backed by Qwen2.5-Coder-0.5B-Instruct "
"Q4_K_M through llama.cpp CUDA on Hugging Face ZeroGPU."
),
)
api.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=False,
allow_methods=["GET", "POST", "OPTIONS"],
allow_headers=["Authorization", "Content-Type", "X-IP-Token"],
)
def require_api_key(authorization: Optional[str] = Header(default=None)) -> None:
if not authorization or not authorization.startswith("Bearer "):
raise HTTPException(status_code=401, detail="Missing Bearer API key")
supplied = authorization[7:].strip()
if not secrets.compare_digest(supplied, API_KEY):
raise HTTPException(status_code=401, detail="Invalid API key")
@lru_cache(maxsize=1)
def get_internal_gradio_client() -> Client:
"""Connect through the public HF Space gateway, not the loopback socket.
Client(token=...) authenticates the request with x-hf-authorization. The
public HF gateway must see that request so it can establish the ZeroGPU
scheduling identity. Calling http://127.0.0.1 bypasses the gateway and can
produce an anonymous/InvalidAccessToken identity at device-api.zero.
"""
url = f"https://{SPACE_HOST}/console"
logger.info("Connecting internal Gradio client through HF gateway: %s", url)
return Client(
url,
token=HF_TOKEN,
verbose=False,
analytics_enabled=False,
max_workers=8,
)
def submit_gpu_job(
api_name: str,
payload: dict[str, Any],
request: Request,
):
# All REST callers authenticate using LLAMA_API_KEY. The Space owner's
# HF_TOKEN authenticates the internal Gradio client. Do not forward an
# arbitrary X-IP-Token supplied to this API: the public HF gateway should
# establish identity from the configured HF_TOKEN instead.
client = get_internal_gradio_client()
return client.submit(payload, API_KEY, api_name=api_name)
@api.get("/health")
def health() -> dict[str, Any]:
return {
"status": "ok",
"model": MODEL_ALIAS,
"quantization": "Q4_K_M",
"runtime": "llama.cpp",
"gpu": "ZeroGPU on-demand CUDA",
"cuda_backend_probe": "deferred until a GPU worker is allocated",
"quota_authentication": "HF_TOKEN via public SPACE_HOST gateway",
"hf_token_configured": bool(HF_TOKEN),
"hf_token_valid": HF_TOKEN_VALID,
"internal_gradio_target": f"https://{SPACE_HOST}/console",
"model_downloaded": Path(MODEL_PATH).is_file(),
"model_loaded": "inside a ZeroGPU worker on first inference",
"gpu_offload_required": True,
}
@api.get("/v1/models", dependencies=[Depends(require_api_key)])
def list_models() -> dict[str, Any]:
return {
"object": "list",
"data": [
{
"id": MODEL_ALIAS,
"object": "model",
"created": 0,
"owned_by": "qwen",
}
],
}
@api.post("/v1/chat/completions", dependencies=[Depends(require_api_key)])
def chat_completions(body: ChatRequest, request: Request):
if not body.messages:
raise HTTPException(status_code=422, detail="messages must contain at least one item")
payload = body.model_dump()
try:
if body.stream:
job = submit_gpu_job("/gpu_chat_stream", payload, request)
def stream_job() -> Iterator[str]:
try:
for update in job:
if isinstance(update, str):
yield update
except Exception as exc:
logger.exception("ZeroGPU streaming job failed")
error = {"error": {"message": str(exc), "type": "server_error"}}
yield "data: " + json.dumps(error, ensure_ascii=False) + "\n\n"
yield "data: [DONE]\n\n"
return StreamingResponse(
stream_job(),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"X-Accel-Buffering": "no",
},
)
job = submit_gpu_job("/gpu_chat_completion", payload, request)
return job.result()
except HTTPException:
raise
except Exception as exc:
logger.exception("Chat completion failed")
raise HTTPException(status_code=503, detail=f"Inference unavailable: {exc}") from exc
@api.post("/v1/completions", dependencies=[Depends(require_api_key)])
def completions(body: CompletionRequest, request: Request):
if body.stream:
raise HTTPException(
status_code=400,
detail="Streaming /v1/completions is not enabled; use /v1/chat/completions",
)
try:
job = submit_gpu_job("/gpu_text_completion", body.model_dump(), request)
return job.result()
except Exception as exc:
logger.exception("Text completion failed")
raise HTTPException(status_code=503, detail=f"Inference unavailable: {exc}") from exc
# Register each decorated GPU function as a Gradio event. The named Gradio API
# is also protected by the same key inside the worker, so it cannot be used to
# bypass authentication. FastAPI routes above proxy into this queue.
with gr.Blocks(title="Qwen2.5 Coder API", analytics_enabled=False) as demo:
gr.Markdown(
"""
# ⚡ Qwen2.5-Coder-0.5B-Instruct REST API
**Runtime:** llama.cpp · CUDA · Hugging Face ZeroGPU
**Quantization:** Q4_K_M
**REST base:** `/v1`
All inference routes require the `LLAMA_API_KEY` Space Secret as a
Bearer token. Use `/docs` for the OpenAPI schema.
"""
)
with gr.Row(visible=False):
chat_payload = gr.JSON(value={"messages": [{"role": "user", "content": "ping"}]})
chat_key = gr.Textbox(value="", type="password")
chat_result = gr.JSON()
chat_trigger = gr.Button("Register chat worker")
chat_trigger.click(
fn=gpu_chat_completion,
inputs=[chat_payload, chat_key],
outputs=chat_result,
api_name="gpu_chat_completion",
)
with gr.Row(visible=False):
stream_payload = gr.JSON(value={"messages": [{"role": "user", "content": "ping"}]})
stream_key = gr.Textbox(value="", type="password")
stream_result = gr.Textbox()
stream_trigger = gr.Button("Register streaming worker")
stream_trigger.click(
fn=gpu_chat_stream,
inputs=[stream_payload, stream_key],
outputs=stream_result,
api_name="gpu_chat_stream",
)
with gr.Row(visible=False):
completion_payload = gr.JSON(value={"prompt": "ping"})
completion_key = gr.Textbox(value="", type="password")
completion_result = gr.JSON()
completion_trigger = gr.Button("Register completion worker")
completion_trigger.click(
fn=gpu_text_completion,
inputs=[completion_payload, completion_key],
outputs=completion_result,
api_name="gpu_text_completion",
)
# Use the Gradio queue for serialized requests; the ZeroGPU scheduler allocates
# a GPU for the decorated event callbacks.
demo.queue(default_concurrency_limit=1, max_size=32)
@api.get("/", include_in_schema=False)
def root():
return RedirectResponse(url="/console")
app = gr.mount_gradio_app(api, demo, path="/console")
if __name__ == "__main__":
logger.info("llama-cpp-python version: %s", getattr(llama_cpp, "__version__", "unknown"))
logger.info("CUDA initialization and model load are deferred to ZeroGPU workers.")
uvicorn.run(app, host="0.0.0.0", port=PORT, log_level=os.getenv("LOG_LEVEL", "info").lower())