Spaces:
Running on Zero
Running on Zero
Download app.py from mhd-radin/qwen25-coder-api: direct link, hf CLI and curl.
- Browser
- Download file 18.9 kB
-
https://huggingface.co/spaces/mhd-radin/qwen25-coder-api/resolve/main/app.py
- Command line
-
hf download hf://spaces/mhd-radin/qwen25-coder-api/app.py
-
curl -L -o app.py https://huggingface.co/spaces/mhd-radin/qwen25-coder-api/resolve/main/app.py
18.9 kB
| from __future__ import annotations | |
| import os | |
| # Gradio 5/6 may start a Node SSR proxy that reserves the public Space port | |
| # (7860). This app serves a custom FastAPI + mounted Gradio application with | |
| # Uvicorn directly, so disable that proxy before importing Gradio/spaces. | |
| # Otherwise Uvicorn fails with: [Errno 98] address already in use. | |
| os.environ["GRADIO_SSR_MODE"] = "false" | |
| # ZeroGPU must be imported before torch or any module that can touch CUDA. | |
| import spaces | |
| import json | |
| import logging | |
| import secrets | |
| import threading | |
| import time | |
| from functools import lru_cache | |
| from pathlib import Path | |
| from typing import Any, Iterator, Optional, Union | |
| import gradio as gr | |
| import torch # ZeroGPU's Gradio image provides its matching CUDA-enabled PyTorch. | |
| import uvicorn | |
| from fastapi import Depends, FastAPI, Header, HTTPException, Request | |
| from fastapi.middleware.cors import CORSMiddleware | |
| from fastapi.responses import RedirectResponse, StreamingResponse | |
| from gradio_client import Client | |
| from huggingface_hub import HfApi, hf_hub_download | |
| from pydantic import BaseModel, ConfigDict, Field | |
| # Import llama.cpp after the ZeroGPU bootstrap and the platform CUDA runtime. | |
| import llama_cpp | |
| from llama_cpp import Llama, llama_supports_gpu_offload | |
| logging.basicConfig( | |
| level=os.getenv("LOG_LEVEL", "INFO").upper(), | |
| format="%(asctime)s %(levelname)s %(process)d %(name)s: %(message)s", | |
| ) | |
| logger = logging.getLogger("qwen-zerogpu-api") | |
| PORT = int(os.getenv("PORT", "7860")) | |
| MODEL_REPO_ID = os.getenv( | |
| "MODEL_REPO_ID", "Qwen/Qwen2.5-Coder-0.5B-Instruct-GGUF" | |
| ) | |
| MODEL_FILENAME = os.getenv( | |
| "MODEL_FILENAME", "qwen2.5-coder-0.5b-instruct-q4_k_m.gguf" | |
| ) | |
| MODEL_ALIAS = os.getenv("MODEL_ALIAS", "qwen2.5-coder-0.5b-instruct") | |
| API_KEY = os.getenv("LLAMA_API_KEY", "").strip() | |
| HF_TOKEN = os.getenv("HF_TOKEN", "").strip() | |
| SPACE_HOST = os.getenv("SPACE_HOST", "").strip() | |
| def positive_int(name: str, default: int, maximum: Optional[int] = None) -> int: | |
| raw = os.getenv(name, str(default)) | |
| try: | |
| value = int(raw) | |
| except ValueError as exc: | |
| raise RuntimeError(f"{name} must be an integer, got {raw!r}") from exc | |
| if value < 1: | |
| raise RuntimeError(f"{name} must be >= 1") | |
| if maximum is not None and value > maximum: | |
| raise RuntimeError(f"{name} must be <= {maximum}") | |
| return value | |
| N_CTX = positive_int("LLAMA_N_CTX", 4096, 32768) | |
| N_BATCH = positive_int("LLAMA_N_BATCH", 1024, 4096) | |
| N_UBATCH = positive_int("LLAMA_N_UBATCH", 512, N_BATCH) | |
| N_THREADS = positive_int("LLAMA_N_THREADS", 4, 64) | |
| MAX_TOKENS = positive_int("LLAMA_MAX_TOKENS", 2048, 8192) | |
| GPU_DURATION_MAX = positive_int("LLAMA_GPU_DURATION_MAX", 60, 120) | |
| # This cache is created only inside a ZeroGPU worker. It is not initialized in | |
| # the main web process, which has no physical GPU allocation. | |
| _MODEL: Optional[Llama] = None | |
| _MODEL_PID: Optional[int] = None | |
| def download_model() -> str: | |
| logger.info("Downloading/checking model: %s / %s", MODEL_REPO_ID, MODEL_FILENAME) | |
| path = hf_hub_download( | |
| repo_id=MODEL_REPO_ID, | |
| filename=MODEL_FILENAME, | |
| token=HF_TOKEN, | |
| ) | |
| size_mb = Path(path).stat().st_size / (1024 * 1024) | |
| logger.info("Model file available: %s (%.1f MiB)", path, size_mb) | |
| return path | |
| # Download the model once into the Hub cache. This step does not request a GPU. | |
| MODEL_PATH = download_model() | |
| def get_model() -> Llama: | |
| """Load/cache llama.cpp in the active ZeroGPU worker process.""" | |
| global _MODEL, _MODEL_PID | |
| pid = os.getpid() | |
| if _MODEL is not None and _MODEL_PID == pid: | |
| return _MODEL | |
| if not llama_supports_gpu_offload(): | |
| raise RuntimeError( | |
| "The installed llama.cpp wheel has no usable CUDA backend in this " | |
| "ZeroGPU worker. CPU fallback is disabled. Check the CUDA wheel, " | |
| "runtime libraries, and ZeroGPU compatibility." | |
| ) | |
| logger.info( | |
| "Initializing llama.cpp with full CUDA layer offload (n_gpu_layers=-1), pid=%s", | |
| pid, | |
| ) | |
| started = time.perf_counter() | |
| model = Llama( | |
| model_path=MODEL_PATH, | |
| n_gpu_layers=-1, | |
| main_gpu=0, | |
| n_ctx=N_CTX, | |
| n_batch=N_BATCH, | |
| n_ubatch=N_UBATCH, | |
| n_threads=N_THREADS, | |
| n_threads_batch=N_THREADS, | |
| offload_kqv=True, | |
| flash_attn=True, | |
| use_mmap=True, | |
| use_mlock=False, | |
| verbose=os.getenv("LLAMA_VERBOSE", "1") == "1", | |
| ) | |
| _MODEL = model | |
| _MODEL_PID = pid | |
| logger.info( | |
| "CUDA model initialized in %.2fs (worker pid=%s)", | |
| time.perf_counter() - started, | |
| pid, | |
| ) | |
| return model | |
| def gpu_duration(payload: dict[str, Any], api_key: str) -> int: | |
| """Bound each ZeroGPU reservation and keep the default request short.""" | |
| try: | |
| requested = int(payload.get("max_tokens") or 512) | |
| except (TypeError, ValueError): | |
| requested = 512 | |
| requested = max(1, min(requested, MAX_TOKENS)) | |
| # Keep ZeroGPU's up-front reservation tight for this small 0.5B model. | |
| # The reservation must still leave room for CUDA worker startup/model load. | |
| # 256 tokens -> 15s, 512 -> 20s, 1024 -> 30s, 2048 -> 50s. | |
| return min(GPU_DURATION_MAX, max(15, 10 + int(requested * 0.02))) | |
| def verify_worker_key(api_key: str) -> None: | |
| if not api_key or not secrets.compare_digest(str(api_key), API_KEY): | |
| raise PermissionError("Invalid API key") | |
| def chat_kwargs(payload: dict[str, Any]) -> dict[str, Any]: | |
| max_tokens = int(payload.get("max_tokens") or 512) | |
| if max_tokens < 1 or max_tokens > MAX_TOKENS: | |
| raise ValueError(f"max_tokens must be between 1 and {MAX_TOKENS}") | |
| kwargs: dict[str, Any] = { | |
| "temperature": float(payload.get("temperature", 0.2)), | |
| "top_p": float(payload.get("top_p", 0.95)), | |
| "max_tokens": max_tokens, | |
| "repeat_penalty": float(payload.get("repeat_penalty", 1.05)), | |
| "presence_penalty": float(payload.get("presence_penalty", 0.0)), | |
| "frequency_penalty": float(payload.get("frequency_penalty", 0.0)), | |
| "stream": bool(payload.get("stream", False)), | |
| "model": MODEL_ALIAS, | |
| } | |
| if payload.get("stop") is not None: | |
| kwargs["stop"] = payload["stop"] | |
| if payload.get("seed") is not None: | |
| kwargs["seed"] = int(payload["seed"]) | |
| return kwargs | |
| # These functions are intentionally the functions bound to Gradio events. | |
| # ZeroGPU discovers the decorated callbacks through Gradio's event registry. | |
| def gpu_chat_completion(payload: dict[str, Any], api_key: str) -> dict[str, Any]: | |
| verify_worker_key(api_key) | |
| model = get_model() | |
| started = time.perf_counter() | |
| result = model.create_chat_completion( | |
| messages=payload["messages"], | |
| **chat_kwargs(payload), | |
| ) | |
| elapsed = max(time.perf_counter() - started, 1e-6) | |
| usage = result.get("usage", {}) | |
| completion_tokens = int(usage.get("completion_tokens", 0)) | |
| logger.info( | |
| "Chat request finished: %s output tokens in %.3fs (%.1f tok/s)", | |
| completion_tokens, | |
| elapsed, | |
| completion_tokens / elapsed, | |
| ) | |
| return result | |
| def gpu_chat_stream(payload: dict[str, Any], api_key: str) -> Iterator[str]: | |
| verify_worker_key(api_key) | |
| try: | |
| model = get_model() | |
| kwargs = chat_kwargs(payload) | |
| kwargs["stream"] = True | |
| for chunk in model.create_chat_completion( | |
| messages=payload["messages"], | |
| **kwargs, | |
| ): | |
| yield "data: " + json.dumps(chunk, ensure_ascii=False) + "\n\n" | |
| yield "data: [DONE]\n\n" | |
| except Exception as exc: | |
| logger.exception("Streaming inference failed") | |
| error = {"error": {"message": str(exc), "type": "server_error"}} | |
| yield "data: " + json.dumps(error, ensure_ascii=False) + "\n\n" | |
| yield "data: [DONE]\n\n" | |
| def gpu_text_completion(payload: dict[str, Any], api_key: str) -> dict[str, Any]: | |
| verify_worker_key(api_key) | |
| model = get_model() | |
| max_tokens = int(payload.get("max_tokens") or 512) | |
| if max_tokens < 1 or max_tokens > MAX_TOKENS: | |
| raise ValueError(f"max_tokens must be between 1 and {MAX_TOKENS}") | |
| kwargs: dict[str, Any] = { | |
| "prompt": payload["prompt"], | |
| "temperature": float(payload.get("temperature", 0.2)), | |
| "top_p": float(payload.get("top_p", 0.95)), | |
| "max_tokens": max_tokens, | |
| "repeat_penalty": float(payload.get("repeat_penalty", 1.05)), | |
| "stream": False, | |
| "model": MODEL_ALIAS, | |
| } | |
| if payload.get("stop") is not None: | |
| kwargs["stop"] = payload["stop"] | |
| if payload.get("seed") is not None: | |
| kwargs["seed"] = int(payload["seed"]) | |
| return model.create_completion(**kwargs) | |
| class ChatRequest(BaseModel): | |
| model: str = MODEL_ALIAS | |
| messages: list[dict[str, Any]] | |
| temperature: float = Field(default=0.2, ge=0, le=2) | |
| top_p: float = Field(default=0.95, gt=0, le=1) | |
| max_tokens: int = Field(default=256, ge=1, le=MAX_TOKENS) | |
| stream: bool = False | |
| stop: Optional[Union[str, list[str]]] = None | |
| presence_penalty: float = Field(default=0.0, ge=-2, le=2) | |
| frequency_penalty: float = Field(default=0.0, ge=-2, le=2) | |
| repeat_penalty: float = Field(default=1.05, ge=0.5, le=2) | |
| seed: Optional[int] = None | |
| model_config = ConfigDict(extra="ignore") | |
| class CompletionRequest(BaseModel): | |
| model: str = MODEL_ALIAS | |
| prompt: str | |
| temperature: float = Field(default=0.2, ge=0, le=2) | |
| top_p: float = Field(default=0.95, gt=0, le=1) | |
| max_tokens: int = Field(default=256, ge=1, le=MAX_TOKENS) | |
| stream: bool = False | |
| stop: Optional[Union[str, list[str]]] = None | |
| repeat_penalty: float = Field(default=1.05, ge=0.5, le=2) | |
| seed: Optional[int] = None | |
| model_config = ConfigDict(extra="ignore") | |
| if not API_KEY: | |
| raise RuntimeError( | |
| "Missing required Space Secret LLAMA_API_KEY. Add it under " | |
| "Settings → Variables and secrets → Secrets." | |
| ) | |
| if not HF_TOKEN: | |
| raise RuntimeError( | |
| "Missing required Space Secret HF_TOKEN. Create a Hugging Face user access " | |
| "token with read access and add it under Settings → Variables and secrets → Secrets." | |
| ) | |
| if not SPACE_HOST: | |
| raise RuntimeError( | |
| "Missing Hugging Face SPACE_HOST environment variable. This app must run " | |
| "inside a Hugging Face Space so its authenticated public gateway can be used." | |
| ) | |
| # Validate the configured user token before accepting API traffic. Never log the token. | |
| try: | |
| HfApi(token=HF_TOKEN).whoami() | |
| HF_TOKEN_VALID = True | |
| except Exception as exc: | |
| raise RuntimeError( | |
| "HF_TOKEN is not a valid Hugging Face user access token. Create a token " | |
| "with read access at https://huggingface.co/settings/tokens and replace " | |
| "the Space Secret." | |
| ) from exc | |
| api = FastAPI( | |
| title="Qwen2.5 Coder ZeroGPU API", | |
| version="2.2.0", | |
| description=( | |
| "OpenAI-compatible REST API backed by Qwen2.5-Coder-0.5B-Instruct " | |
| "Q4_K_M through llama.cpp CUDA on Hugging Face ZeroGPU." | |
| ), | |
| ) | |
| api.add_middleware( | |
| CORSMiddleware, | |
| allow_origins=["*"], | |
| allow_credentials=False, | |
| allow_methods=["GET", "POST", "OPTIONS"], | |
| allow_headers=["Authorization", "Content-Type", "X-IP-Token"], | |
| ) | |
| def require_api_key(authorization: Optional[str] = Header(default=None)) -> None: | |
| if not authorization or not authorization.startswith("Bearer "): | |
| raise HTTPException(status_code=401, detail="Missing Bearer API key") | |
| supplied = authorization[7:].strip() | |
| if not secrets.compare_digest(supplied, API_KEY): | |
| raise HTTPException(status_code=401, detail="Invalid API key") | |
| def get_internal_gradio_client() -> Client: | |
| """Connect through the public HF Space gateway, not the loopback socket. | |
| Client(token=...) authenticates the request with x-hf-authorization. The | |
| public HF gateway must see that request so it can establish the ZeroGPU | |
| scheduling identity. Calling http://127.0.0.1 bypasses the gateway and can | |
| produce an anonymous/InvalidAccessToken identity at device-api.zero. | |
| """ | |
| url = f"https://{SPACE_HOST}/console" | |
| logger.info("Connecting internal Gradio client through HF gateway: %s", url) | |
| return Client( | |
| url, | |
| token=HF_TOKEN, | |
| verbose=False, | |
| analytics_enabled=False, | |
| max_workers=8, | |
| ) | |
| def submit_gpu_job( | |
| api_name: str, | |
| payload: dict[str, Any], | |
| request: Request, | |
| ): | |
| # All REST callers authenticate using LLAMA_API_KEY. The Space owner's | |
| # HF_TOKEN authenticates the internal Gradio client. Do not forward an | |
| # arbitrary X-IP-Token supplied to this API: the public HF gateway should | |
| # establish identity from the configured HF_TOKEN instead. | |
| client = get_internal_gradio_client() | |
| return client.submit(payload, API_KEY, api_name=api_name) | |
| def health() -> dict[str, Any]: | |
| return { | |
| "status": "ok", | |
| "model": MODEL_ALIAS, | |
| "quantization": "Q4_K_M", | |
| "runtime": "llama.cpp", | |
| "gpu": "ZeroGPU on-demand CUDA", | |
| "cuda_backend_probe": "deferred until a GPU worker is allocated", | |
| "quota_authentication": "HF_TOKEN via public SPACE_HOST gateway", | |
| "hf_token_configured": bool(HF_TOKEN), | |
| "hf_token_valid": HF_TOKEN_VALID, | |
| "internal_gradio_target": f"https://{SPACE_HOST}/console", | |
| "model_downloaded": Path(MODEL_PATH).is_file(), | |
| "model_loaded": "inside a ZeroGPU worker on first inference", | |
| "gpu_offload_required": True, | |
| } | |
| def list_models() -> dict[str, Any]: | |
| return { | |
| "object": "list", | |
| "data": [ | |
| { | |
| "id": MODEL_ALIAS, | |
| "object": "model", | |
| "created": 0, | |
| "owned_by": "qwen", | |
| } | |
| ], | |
| } | |
| def chat_completions(body: ChatRequest, request: Request): | |
| if not body.messages: | |
| raise HTTPException(status_code=422, detail="messages must contain at least one item") | |
| payload = body.model_dump() | |
| try: | |
| if body.stream: | |
| job = submit_gpu_job("/gpu_chat_stream", payload, request) | |
| def stream_job() -> Iterator[str]: | |
| try: | |
| for update in job: | |
| if isinstance(update, str): | |
| yield update | |
| except Exception as exc: | |
| logger.exception("ZeroGPU streaming job failed") | |
| error = {"error": {"message": str(exc), "type": "server_error"}} | |
| yield "data: " + json.dumps(error, ensure_ascii=False) + "\n\n" | |
| yield "data: [DONE]\n\n" | |
| return StreamingResponse( | |
| stream_job(), | |
| media_type="text/event-stream", | |
| headers={ | |
| "Cache-Control": "no-cache", | |
| "Connection": "keep-alive", | |
| "X-Accel-Buffering": "no", | |
| }, | |
| ) | |
| job = submit_gpu_job("/gpu_chat_completion", payload, request) | |
| return job.result() | |
| except HTTPException: | |
| raise | |
| except Exception as exc: | |
| logger.exception("Chat completion failed") | |
| raise HTTPException(status_code=503, detail=f"Inference unavailable: {exc}") from exc | |
| def completions(body: CompletionRequest, request: Request): | |
| if body.stream: | |
| raise HTTPException( | |
| status_code=400, | |
| detail="Streaming /v1/completions is not enabled; use /v1/chat/completions", | |
| ) | |
| try: | |
| job = submit_gpu_job("/gpu_text_completion", body.model_dump(), request) | |
| return job.result() | |
| except Exception as exc: | |
| logger.exception("Text completion failed") | |
| raise HTTPException(status_code=503, detail=f"Inference unavailable: {exc}") from exc | |
| # Register each decorated GPU function as a Gradio event. The named Gradio API | |
| # is also protected by the same key inside the worker, so it cannot be used to | |
| # bypass authentication. FastAPI routes above proxy into this queue. | |
| with gr.Blocks(title="Qwen2.5 Coder API", analytics_enabled=False) as demo: | |
| gr.Markdown( | |
| """ | |
| # ⚡ Qwen2.5-Coder-0.5B-Instruct REST API | |
| **Runtime:** llama.cpp · CUDA · Hugging Face ZeroGPU | |
| **Quantization:** Q4_K_M | |
| **REST base:** `/v1` | |
| All inference routes require the `LLAMA_API_KEY` Space Secret as a | |
| Bearer token. Use `/docs` for the OpenAPI schema. | |
| """ | |
| ) | |
| with gr.Row(visible=False): | |
| chat_payload = gr.JSON(value={"messages": [{"role": "user", "content": "ping"}]}) | |
| chat_key = gr.Textbox(value="", type="password") | |
| chat_result = gr.JSON() | |
| chat_trigger = gr.Button("Register chat worker") | |
| chat_trigger.click( | |
| fn=gpu_chat_completion, | |
| inputs=[chat_payload, chat_key], | |
| outputs=chat_result, | |
| api_name="gpu_chat_completion", | |
| ) | |
| with gr.Row(visible=False): | |
| stream_payload = gr.JSON(value={"messages": [{"role": "user", "content": "ping"}]}) | |
| stream_key = gr.Textbox(value="", type="password") | |
| stream_result = gr.Textbox() | |
| stream_trigger = gr.Button("Register streaming worker") | |
| stream_trigger.click( | |
| fn=gpu_chat_stream, | |
| inputs=[stream_payload, stream_key], | |
| outputs=stream_result, | |
| api_name="gpu_chat_stream", | |
| ) | |
| with gr.Row(visible=False): | |
| completion_payload = gr.JSON(value={"prompt": "ping"}) | |
| completion_key = gr.Textbox(value="", type="password") | |
| completion_result = gr.JSON() | |
| completion_trigger = gr.Button("Register completion worker") | |
| completion_trigger.click( | |
| fn=gpu_text_completion, | |
| inputs=[completion_payload, completion_key], | |
| outputs=completion_result, | |
| api_name="gpu_text_completion", | |
| ) | |
| # Use the Gradio queue for serialized requests; the ZeroGPU scheduler allocates | |
| # a GPU for the decorated event callbacks. | |
| demo.queue(default_concurrency_limit=1, max_size=32) | |
| def root(): | |
| return RedirectResponse(url="/console") | |
| app = gr.mount_gradio_app(api, demo, path="/console") | |
| if __name__ == "__main__": | |
| logger.info("llama-cpp-python version: %s", getattr(llama_cpp, "__version__", "unknown")) | |
| logger.info("CUDA initialization and model load are deferred to ZeroGPU workers.") | |
| uvicorn.run(app, host="0.0.0.0", port=PORT, log_level=os.getenv("LOG_LEVEL", "info").lower()) | |