from __future__ import annotations import os # Gradio 5/6 may start a Node SSR proxy that reserves the public Space port # (7860). This app serves a custom FastAPI + mounted Gradio application with # Uvicorn directly, so disable that proxy before importing Gradio/spaces. # Otherwise Uvicorn fails with: [Errno 98] address already in use. os.environ["GRADIO_SSR_MODE"] = "false" # ZeroGPU must be imported before torch or any module that can touch CUDA. import spaces import json import logging import secrets import threading import time from functools import lru_cache from pathlib import Path from typing import Any, Iterator, Optional, Union import gradio as gr import torch # ZeroGPU's Gradio image provides its matching CUDA-enabled PyTorch. import uvicorn from fastapi import Depends, FastAPI, Header, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import RedirectResponse, StreamingResponse from gradio_client import Client from huggingface_hub import HfApi, hf_hub_download from pydantic import BaseModel, ConfigDict, Field # Import llama.cpp after the ZeroGPU bootstrap and the platform CUDA runtime. import llama_cpp from llama_cpp import Llama, llama_supports_gpu_offload logging.basicConfig( level=os.getenv("LOG_LEVEL", "INFO").upper(), format="%(asctime)s %(levelname)s %(process)d %(name)s: %(message)s", ) logger = logging.getLogger("qwen-zerogpu-api") PORT = int(os.getenv("PORT", "7860")) MODEL_REPO_ID = os.getenv( "MODEL_REPO_ID", "Qwen/Qwen2.5-Coder-0.5B-Instruct-GGUF" ) MODEL_FILENAME = os.getenv( "MODEL_FILENAME", "qwen2.5-coder-0.5b-instruct-q4_k_m.gguf" ) MODEL_ALIAS = os.getenv("MODEL_ALIAS", "qwen2.5-coder-0.5b-instruct") API_KEY = os.getenv("LLAMA_API_KEY", "").strip() HF_TOKEN = os.getenv("HF_TOKEN", "").strip() SPACE_HOST = os.getenv("SPACE_HOST", "").strip() def positive_int(name: str, default: int, maximum: Optional[int] = None) -> int: raw = os.getenv(name, str(default)) try: value = int(raw) except ValueError as exc: raise RuntimeError(f"{name} must be an integer, got {raw!r}") from exc if value < 1: raise RuntimeError(f"{name} must be >= 1") if maximum is not None and value > maximum: raise RuntimeError(f"{name} must be <= {maximum}") return value N_CTX = positive_int("LLAMA_N_CTX", 4096, 32768) N_BATCH = positive_int("LLAMA_N_BATCH", 1024, 4096) N_UBATCH = positive_int("LLAMA_N_UBATCH", 512, N_BATCH) N_THREADS = positive_int("LLAMA_N_THREADS", 4, 64) MAX_TOKENS = positive_int("LLAMA_MAX_TOKENS", 2048, 8192) GPU_DURATION_MAX = positive_int("LLAMA_GPU_DURATION_MAX", 60, 120) # This cache is created only inside a ZeroGPU worker. It is not initialized in # the main web process, which has no physical GPU allocation. _MODEL: Optional[Llama] = None _MODEL_PID: Optional[int] = None def download_model() -> str: logger.info("Downloading/checking model: %s / %s", MODEL_REPO_ID, MODEL_FILENAME) path = hf_hub_download( repo_id=MODEL_REPO_ID, filename=MODEL_FILENAME, token=HF_TOKEN, ) size_mb = Path(path).stat().st_size / (1024 * 1024) logger.info("Model file available: %s (%.1f MiB)", path, size_mb) return path # Download the model once into the Hub cache. This step does not request a GPU. MODEL_PATH = download_model() def get_model() -> Llama: """Load/cache llama.cpp in the active ZeroGPU worker process.""" global _MODEL, _MODEL_PID pid = os.getpid() if _MODEL is not None and _MODEL_PID == pid: return _MODEL if not llama_supports_gpu_offload(): raise RuntimeError( "The installed llama.cpp wheel has no usable CUDA backend in this " "ZeroGPU worker. CPU fallback is disabled. Check the CUDA wheel, " "runtime libraries, and ZeroGPU compatibility." ) logger.info( "Initializing llama.cpp with full CUDA layer offload (n_gpu_layers=-1), pid=%s", pid, ) started = time.perf_counter() model = Llama( model_path=MODEL_PATH, n_gpu_layers=-1, main_gpu=0, n_ctx=N_CTX, n_batch=N_BATCH, n_ubatch=N_UBATCH, n_threads=N_THREADS, n_threads_batch=N_THREADS, offload_kqv=True, flash_attn=True, use_mmap=True, use_mlock=False, verbose=os.getenv("LLAMA_VERBOSE", "1") == "1", ) _MODEL = model _MODEL_PID = pid logger.info( "CUDA model initialized in %.2fs (worker pid=%s)", time.perf_counter() - started, pid, ) return model def gpu_duration(payload: dict[str, Any], api_key: str) -> int: """Bound each ZeroGPU reservation and keep the default request short.""" try: requested = int(payload.get("max_tokens") or 512) except (TypeError, ValueError): requested = 512 requested = max(1, min(requested, MAX_TOKENS)) # Keep ZeroGPU's up-front reservation tight for this small 0.5B model. # The reservation must still leave room for CUDA worker startup/model load. # 256 tokens -> 15s, 512 -> 20s, 1024 -> 30s, 2048 -> 50s. return min(GPU_DURATION_MAX, max(15, 10 + int(requested * 0.02))) def verify_worker_key(api_key: str) -> None: if not api_key or not secrets.compare_digest(str(api_key), API_KEY): raise PermissionError("Invalid API key") def chat_kwargs(payload: dict[str, Any]) -> dict[str, Any]: max_tokens = int(payload.get("max_tokens") or 512) if max_tokens < 1 or max_tokens > MAX_TOKENS: raise ValueError(f"max_tokens must be between 1 and {MAX_TOKENS}") kwargs: dict[str, Any] = { "temperature": float(payload.get("temperature", 0.2)), "top_p": float(payload.get("top_p", 0.95)), "max_tokens": max_tokens, "repeat_penalty": float(payload.get("repeat_penalty", 1.05)), "presence_penalty": float(payload.get("presence_penalty", 0.0)), "frequency_penalty": float(payload.get("frequency_penalty", 0.0)), "stream": bool(payload.get("stream", False)), "model": MODEL_ALIAS, } if payload.get("stop") is not None: kwargs["stop"] = payload["stop"] if payload.get("seed") is not None: kwargs["seed"] = int(payload["seed"]) return kwargs # These functions are intentionally the functions bound to Gradio events. # ZeroGPU discovers the decorated callbacks through Gradio's event registry. @spaces.GPU(duration=gpu_duration) def gpu_chat_completion(payload: dict[str, Any], api_key: str) -> dict[str, Any]: verify_worker_key(api_key) model = get_model() started = time.perf_counter() result = model.create_chat_completion( messages=payload["messages"], **chat_kwargs(payload), ) elapsed = max(time.perf_counter() - started, 1e-6) usage = result.get("usage", {}) completion_tokens = int(usage.get("completion_tokens", 0)) logger.info( "Chat request finished: %s output tokens in %.3fs (%.1f tok/s)", completion_tokens, elapsed, completion_tokens / elapsed, ) return result @spaces.GPU(duration=gpu_duration) def gpu_chat_stream(payload: dict[str, Any], api_key: str) -> Iterator[str]: verify_worker_key(api_key) try: model = get_model() kwargs = chat_kwargs(payload) kwargs["stream"] = True for chunk in model.create_chat_completion( messages=payload["messages"], **kwargs, ): yield "data: " + json.dumps(chunk, ensure_ascii=False) + "\n\n" yield "data: [DONE]\n\n" except Exception as exc: logger.exception("Streaming inference failed") error = {"error": {"message": str(exc), "type": "server_error"}} yield "data: " + json.dumps(error, ensure_ascii=False) + "\n\n" yield "data: [DONE]\n\n" @spaces.GPU(duration=gpu_duration) def gpu_text_completion(payload: dict[str, Any], api_key: str) -> dict[str, Any]: verify_worker_key(api_key) model = get_model() max_tokens = int(payload.get("max_tokens") or 512) if max_tokens < 1 or max_tokens > MAX_TOKENS: raise ValueError(f"max_tokens must be between 1 and {MAX_TOKENS}") kwargs: dict[str, Any] = { "prompt": payload["prompt"], "temperature": float(payload.get("temperature", 0.2)), "top_p": float(payload.get("top_p", 0.95)), "max_tokens": max_tokens, "repeat_penalty": float(payload.get("repeat_penalty", 1.05)), "stream": False, "model": MODEL_ALIAS, } if payload.get("stop") is not None: kwargs["stop"] = payload["stop"] if payload.get("seed") is not None: kwargs["seed"] = int(payload["seed"]) return model.create_completion(**kwargs) class ChatRequest(BaseModel): model: str = MODEL_ALIAS messages: list[dict[str, Any]] temperature: float = Field(default=0.2, ge=0, le=2) top_p: float = Field(default=0.95, gt=0, le=1) max_tokens: int = Field(default=256, ge=1, le=MAX_TOKENS) stream: bool = False stop: Optional[Union[str, list[str]]] = None presence_penalty: float = Field(default=0.0, ge=-2, le=2) frequency_penalty: float = Field(default=0.0, ge=-2, le=2) repeat_penalty: float = Field(default=1.05, ge=0.5, le=2) seed: Optional[int] = None model_config = ConfigDict(extra="ignore") class CompletionRequest(BaseModel): model: str = MODEL_ALIAS prompt: str temperature: float = Field(default=0.2, ge=0, le=2) top_p: float = Field(default=0.95, gt=0, le=1) max_tokens: int = Field(default=256, ge=1, le=MAX_TOKENS) stream: bool = False stop: Optional[Union[str, list[str]]] = None repeat_penalty: float = Field(default=1.05, ge=0.5, le=2) seed: Optional[int] = None model_config = ConfigDict(extra="ignore") if not API_KEY: raise RuntimeError( "Missing required Space Secret LLAMA_API_KEY. Add it under " "Settings → Variables and secrets → Secrets." ) if not HF_TOKEN: raise RuntimeError( "Missing required Space Secret HF_TOKEN. Create a Hugging Face user access " "token with read access and add it under Settings → Variables and secrets → Secrets." ) if not SPACE_HOST: raise RuntimeError( "Missing Hugging Face SPACE_HOST environment variable. This app must run " "inside a Hugging Face Space so its authenticated public gateway can be used." ) # Validate the configured user token before accepting API traffic. Never log the token. try: HfApi(token=HF_TOKEN).whoami() HF_TOKEN_VALID = True except Exception as exc: raise RuntimeError( "HF_TOKEN is not a valid Hugging Face user access token. Create a token " "with read access at https://huggingface.co/settings/tokens and replace " "the Space Secret." ) from exc api = FastAPI( title="Qwen2.5 Coder ZeroGPU API", version="2.2.0", description=( "OpenAI-compatible REST API backed by Qwen2.5-Coder-0.5B-Instruct " "Q4_K_M through llama.cpp CUDA on Hugging Face ZeroGPU." ), ) api.add_middleware( CORSMiddleware, allow_origins=["*"], allow_credentials=False, allow_methods=["GET", "POST", "OPTIONS"], allow_headers=["Authorization", "Content-Type", "X-IP-Token"], ) def require_api_key(authorization: Optional[str] = Header(default=None)) -> None: if not authorization or not authorization.startswith("Bearer "): raise HTTPException(status_code=401, detail="Missing Bearer API key") supplied = authorization[7:].strip() if not secrets.compare_digest(supplied, API_KEY): raise HTTPException(status_code=401, detail="Invalid API key") @lru_cache(maxsize=1) def get_internal_gradio_client() -> Client: """Connect through the public HF Space gateway, not the loopback socket. Client(token=...) authenticates the request with x-hf-authorization. The public HF gateway must see that request so it can establish the ZeroGPU scheduling identity. Calling http://127.0.0.1 bypasses the gateway and can produce an anonymous/InvalidAccessToken identity at device-api.zero. """ url = f"https://{SPACE_HOST}/console" logger.info("Connecting internal Gradio client through HF gateway: %s", url) return Client( url, token=HF_TOKEN, verbose=False, analytics_enabled=False, max_workers=8, ) def submit_gpu_job( api_name: str, payload: dict[str, Any], request: Request, ): # All REST callers authenticate using LLAMA_API_KEY. The Space owner's # HF_TOKEN authenticates the internal Gradio client. Do not forward an # arbitrary X-IP-Token supplied to this API: the public HF gateway should # establish identity from the configured HF_TOKEN instead. client = get_internal_gradio_client() return client.submit(payload, API_KEY, api_name=api_name) @api.get("/health") def health() -> dict[str, Any]: return { "status": "ok", "model": MODEL_ALIAS, "quantization": "Q4_K_M", "runtime": "llama.cpp", "gpu": "ZeroGPU on-demand CUDA", "cuda_backend_probe": "deferred until a GPU worker is allocated", "quota_authentication": "HF_TOKEN via public SPACE_HOST gateway", "hf_token_configured": bool(HF_TOKEN), "hf_token_valid": HF_TOKEN_VALID, "internal_gradio_target": f"https://{SPACE_HOST}/console", "model_downloaded": Path(MODEL_PATH).is_file(), "model_loaded": "inside a ZeroGPU worker on first inference", "gpu_offload_required": True, } @api.get("/v1/models", dependencies=[Depends(require_api_key)]) def list_models() -> dict[str, Any]: return { "object": "list", "data": [ { "id": MODEL_ALIAS, "object": "model", "created": 0, "owned_by": "qwen", } ], } @api.post("/v1/chat/completions", dependencies=[Depends(require_api_key)]) def chat_completions(body: ChatRequest, request: Request): if not body.messages: raise HTTPException(status_code=422, detail="messages must contain at least one item") payload = body.model_dump() try: if body.stream: job = submit_gpu_job("/gpu_chat_stream", payload, request) def stream_job() -> Iterator[str]: try: for update in job: if isinstance(update, str): yield update except Exception as exc: logger.exception("ZeroGPU streaming job failed") error = {"error": {"message": str(exc), "type": "server_error"}} yield "data: " + json.dumps(error, ensure_ascii=False) + "\n\n" yield "data: [DONE]\n\n" return StreamingResponse( stream_job(), media_type="text/event-stream", headers={ "Cache-Control": "no-cache", "Connection": "keep-alive", "X-Accel-Buffering": "no", }, ) job = submit_gpu_job("/gpu_chat_completion", payload, request) return job.result() except HTTPException: raise except Exception as exc: logger.exception("Chat completion failed") raise HTTPException(status_code=503, detail=f"Inference unavailable: {exc}") from exc @api.post("/v1/completions", dependencies=[Depends(require_api_key)]) def completions(body: CompletionRequest, request: Request): if body.stream: raise HTTPException( status_code=400, detail="Streaming /v1/completions is not enabled; use /v1/chat/completions", ) try: job = submit_gpu_job("/gpu_text_completion", body.model_dump(), request) return job.result() except Exception as exc: logger.exception("Text completion failed") raise HTTPException(status_code=503, detail=f"Inference unavailable: {exc}") from exc # Register each decorated GPU function as a Gradio event. The named Gradio API # is also protected by the same key inside the worker, so it cannot be used to # bypass authentication. FastAPI routes above proxy into this queue. with gr.Blocks(title="Qwen2.5 Coder API", analytics_enabled=False) as demo: gr.Markdown( """ # ⚡ Qwen2.5-Coder-0.5B-Instruct REST API **Runtime:** llama.cpp · CUDA · Hugging Face ZeroGPU **Quantization:** Q4_K_M **REST base:** `/v1` All inference routes require the `LLAMA_API_KEY` Space Secret as a Bearer token. Use `/docs` for the OpenAPI schema. """ ) with gr.Row(visible=False): chat_payload = gr.JSON(value={"messages": [{"role": "user", "content": "ping"}]}) chat_key = gr.Textbox(value="", type="password") chat_result = gr.JSON() chat_trigger = gr.Button("Register chat worker") chat_trigger.click( fn=gpu_chat_completion, inputs=[chat_payload, chat_key], outputs=chat_result, api_name="gpu_chat_completion", ) with gr.Row(visible=False): stream_payload = gr.JSON(value={"messages": [{"role": "user", "content": "ping"}]}) stream_key = gr.Textbox(value="", type="password") stream_result = gr.Textbox() stream_trigger = gr.Button("Register streaming worker") stream_trigger.click( fn=gpu_chat_stream, inputs=[stream_payload, stream_key], outputs=stream_result, api_name="gpu_chat_stream", ) with gr.Row(visible=False): completion_payload = gr.JSON(value={"prompt": "ping"}) completion_key = gr.Textbox(value="", type="password") completion_result = gr.JSON() completion_trigger = gr.Button("Register completion worker") completion_trigger.click( fn=gpu_text_completion, inputs=[completion_payload, completion_key], outputs=completion_result, api_name="gpu_text_completion", ) # Use the Gradio queue for serialized requests; the ZeroGPU scheduler allocates # a GPU for the decorated event callbacks. demo.queue(default_concurrency_limit=1, max_size=32) @api.get("/", include_in_schema=False) def root(): return RedirectResponse(url="/console") app = gr.mount_gradio_app(api, demo, path="/console") if __name__ == "__main__": logger.info("llama-cpp-python version: %s", getattr(llama_cpp, "__version__", "unknown")) logger.info("CUDA initialization and model load are deferred to ZeroGPU workers.") uvicorn.run(app, host="0.0.0.0", port=PORT, log_level=os.getenv("LOG_LEVEL", "info").lower())