Update app.py

app.py

@@ -1,8 +1,9 @@
 import os
 import time
-import uuid
 import json
-import threading
+import uuid
+import asyncio
+from typing import Any
 
 from fastapi import FastAPI, Request, HTTPException, Depends
 from fastapi.middleware.cors import CORSMiddleware
@@ -18,40 +19,19 @@ from llama_cpp import Llama
 # ============================================================
 
 APP_NAME = "Varsh LLM"
+MODEL_ID = "qwen2.5-coder-7b-instruct-q4_k_m"
 
 MODEL_REPO = "Qwen/Qwen2.5-Coder-7B-Instruct-GGUF"
 MODEL_FILE = "qwen2.5-coder-7b-instruct-q4_k_m.gguf"
 
-# This MUST exactly match the model ID expected by OmniRoute.
-MODEL_ID = "qwen2.5-coder-7b-instruct-q4_k_m"
-
 HOST = "0.0.0.0"
 PORT = int(os.getenv("PORT", "7860"))
 
-# Your current HF Space = 2 vCPU.
-# Keep this at 2 unless you upgrade the Space.
-N_THREADS = int(os.getenv("N_THREADS", "2"))
-
-# llama.cpp prompt processing batch size.
-N_BATCH = int(os.getenv("N_BATCH", "512"))
-
-# Context window.
-N_CTX = int(os.getenv("N_CTX", "4096"))
-
-
-# ============================================================
-# API KEY
-# ============================================================
-
-# Set this in:
+# IMPORTANT:
+# Add CUSTOM_API_KEY in:
+# Hugging Face Space -> Settings -> Secrets
 #
-# Hugging Face Space
-# -> Settings
-# -> Secrets
-# -> CUSTOM_API_KEY
-#
-# DO NOT hard-code the secret in this file.
-
+# Do NOT hard-code the key in this file.
 CUSTOM_API_KEY = os.getenv("CUSTOM_API_KEY")
 
 if not CUSTOM_API_KEY:
@@ -62,33 +42,23 @@ if not CUSTOM_API_KEY:
 
 
 # ============================================================
-# FASTAPI APPLICATION
+# FASTAPI
 # ============================================================
 
 app = FastAPI(
-    title="Varsh LLM API",
-    version="1.0.0",
-    description="OpenAI-compatible Qwen 2.5 Coder API",
+    title=APP_NAME,
+    version="2.0.0",
 )
 
-
-# ============================================================
-# CORS
-# ============================================================
-
 app.add_middleware(
     CORSMiddleware,
     allow_origins=["*"],
-    allow_credentials=False,
+    allow_credentials=True,
     allow_methods=["*"],
     allow_headers=["*"],
 )
 
 
-# ============================================================
-# AUTHENTICATION
-# ============================================================
-
 security = HTTPBearer(auto_error=False)
 
 
@@ -98,16 +68,13 @@ def verify_api_key(
     if credentials is None:
         raise HTTPException(
             status_code=401,
-            detail="Missing API key.",
-            headers={
-                "WWW-Authenticate": "Bearer"
-            },
+            detail="Missing Authorization header.",
         )
 
     if credentials.scheme.lower() != "bearer":
         raise HTTPException(
             status_code=401,
-            detail="Authentication must use Bearer token.",
+            detail="Authorization must use Bearer authentication.",
         )
 
     if credentials.credentials != CUSTOM_API_KEY:
@@ -116,25 +83,19 @@ def verify_api_key(
             detail="Invalid API key.",
         )
 
-    return credentials.credentials
+    return True
 
 
 # ============================================================
-# MODEL LOADING
+# MODEL
 # ============================================================
 
-print("=" * 70)
+print("=" * 60)
 print("Starting Varsh LLM")
-print("=" * 70)
-
-print(f"Model repository : {MODEL_REPO}")
-print(f"Model file       : {MODEL_FILE}")
-print(f"Model ID         : {MODEL_ID}")
-print(f"Threads          : {N_THREADS}")
-print(f"Batch size       : {N_BATCH}")
-print(f"Context size     : {N_CTX}")
-
-print("Checking/downloading model...")
+print("=" * 60)
+print(f"Model repository: {MODEL_REPO}")
+print(f"Model file:       {MODEL_FILE}")
+print("Downloading/checking model...")
 
 model_path = hf_hub_download(
     repo_id=MODEL_REPO,
@@ -142,58 +103,234 @@ model_path = hf_hub_download(
 )
 
 print(f"Model path: {model_path}")
-
 print("Loading Qwen 2.5 Coder 7B...")
 
 llm = Llama(
     model_path=model_path,
 
-    # Context window
-    n_ctx=N_CTX,
-
-    # CPU threads
-    n_threads=N_THREADS,
+    # Your Space has 2 vCPUs / 16 GB RAM.
+    n_ctx=4096,
+    n_threads=2,
+    n_threads_batch=2,
 
-    # Prompt processing batch
-    n_batch=N_BATCH,
+    # Moderate batch size for CPU deployment.
+    n_batch=256,
 
-    # Let llama.cpp use its Qwen chat template
-    chat_format=None,
+    # Keep memory usage reasonable.
+    use_mmap=True,
+    use_mlock=False,
 
     verbose=False,
 )
 
 print("Model loaded successfully.")
+print("=" * 60)
+print("Varsh LLM API READY")
+print("=" * 60)
+print(f"Listening on: {HOST}:{PORT}")
+print("Root endpoint: /")
+print("Models endpoint: /v1/models")
+print("Chat endpoint: /v1/chat/completions")
+print("Authentication: Bearer API key")
+print("=" * 60)
+
+
+# llama.cpp generation should be serialized on a small CPU instance.
+generation_lock = asyncio.Lock()
 
 
 # ============================================================
-# GENERATION LOCK
+# HELPERS
 # ============================================================
 
-# The HF Space has one model instance on CPU.
-#
-# This prevents multiple requests from trying to generate
-# through the same llama.cpp instance simultaneously.
+def normalize_content(content: Any) -> str:
+    """
+    Converts OpenAI-style message content into plain text.
 
-generation_lock = threading.Lock()
+    Supports:
 
+        "Hello"
 
-# ============================================================
-# STARTUP INFORMATION
-# ============================================================
+    and:
 
-print("=" * 70)
-print("Varsh LLM API READY")
-print("=" * 70)
+        [
+            {
+                "type": "text",
+                "text": "Hello"
+            }
+        ]
 
-print(f"Listening on: http://{HOST}:{PORT}")
-print("Root endpoint: /")
-print("Health endpoint: /health")
-print("Models endpoint: /v1/models")
-print("Chat endpoint: /v1/chat/completions")
-print("Authentication: Bearer API key")
-print("Streaming: ENABLED")
-print("=" * 70)
+    Qwen2.5-Coder here is text-only, so image/audio parts
+    cannot be processed.
+    """
+
+    if content is None:
+        return ""
+
+    # Normal OpenAI text content
+    if isinstance(content, str):
+        return content
+
+    # OpenAI multimodal/content-part format
+    if isinstance(content, list):
+        text_parts = []
+
+        for part in content:
+            if isinstance(part, str):
+                text_parts.append(part)
+                continue
+
+            if not isinstance(part, dict):
+                continue
+
+            part_type = part.get("type")
+
+            # Standard text part
+            if part_type == "text":
+                text = part.get("text", "")
+                if isinstance(text, str):
+                    text_parts.append(text)
+
+            # Some clients may use slightly different structures.
+            elif "text" in part and isinstance(part["text"], str):
+                text_parts.append(part["text"])
+
+            # Images/audio are intentionally not processed.
+            elif part_type in {
+                "image",
+                "image_url",
+                "input_image",
+                "audio",
+                "input_audio",
+            }:
+                continue
+
+        return "\n".join(text_parts)
+
+    # Fallback for unusual clients
+    return str(content)
+
+
+def normalize_messages(messages: Any) -> list:
+    """
+    Normalize incoming OpenAI-compatible messages so that
+    llama.cpp receives string content.
+    """
+
+    if not isinstance(messages, list):
+        raise HTTPException(
+            status_code=400,
+            detail="'messages' must be an array.",
+        )
+
+    normalized = []
+
+    for message in messages:
+        if not isinstance(message, dict):
+            continue
+
+        role = message.get("role", "user")
+
+        if not isinstance(role, str):
+            role = "user"
+
+        content = normalize_content(
+            message.get("content", "")
+        )
+
+        new_message = {
+            "role": role,
+            "content": content,
+        }
+
+        # Preserve name if supplied.
+        if "name" in message:
+            new_message["name"] = message["name"]
+
+        # Preserve tool_call_id if supplied.
+        if "tool_call_id" in message:
+            new_message["tool_call_id"] = message["tool_call_id"]
+
+        # Preserve tool calls when clients send them.
+        if "tool_calls" in message:
+            new_message["tool_calls"] = message["tool_calls"]
+
+        normalized.append(new_message)
+
+    if not normalized:
+        raise HTTPException(
+            status_code=400,
+            detail="'messages' cannot be empty.",
+        )
+
+    return normalized
+
+
+def create_chat_id() -> str:
+    return f"chatcmpl-{uuid.uuid4().hex}"
+
+
+def create_timestamp() -> int:
+    return int(time.time())
+
+
+def build_non_stream_response(
+    result: dict,
+    model_id: str,
+) -> dict:
+
+    choices = result.get("choices", [])
+
+    if not choices:
+        text = ""
+        finish_reason = "stop"
+    else:
+        choice = choices[0]
+
+        message = choice.get("message", {})
+
+        text = message.get("content", "")
+
+        if text is None:
+            text = ""
+
+        finish_reason = choice.get(
+            "finish_reason",
+            "stop",
+        )
+
+    usage = result.get("usage", {})
+
+    return {
+        "id": create_chat_id(),
+        "object": "chat.completion",
+        "created": create_timestamp(),
+        "model": model_id,
+        "choices": [
+            {
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": text,
+                },
+                "finish_reason": finish_reason,
+            }
+        ],
+        "usage": {
+            "prompt_tokens": usage.get(
+                "prompt_tokens",
+                0,
+            ),
+            "completion_tokens": usage.get(
+                "completion_tokens",
+                0,
+            ),
+            "total_tokens": usage.get(
+                "total_tokens",
+                0,
+            ),
+        },
+    }
 
 
 # ============================================================
@@ -201,10 +338,10 @@ print("=" * 70)
 # ============================================================
 
 @app.get("/")
-def root():
+async def root():
     return {
         "status": "healthy",
-        "service": APP_NAME,
+        "service": "Varsh LLM",
         "security": "enabled",
         "model": MODEL_ID,
         "endpoint": "/v1/chat/completions",
@@ -214,117 +351,30 @@ def root():
 
 
 # ============================================================
-# HEALTH
-# ============================================================
-
-@app.get("/health")
-def health():
-    return {
-        "status": "healthy",
-        "service": APP_NAME,
-        "model": MODEL_ID,
-    }
-
-
-# ============================================================
-# OPENAI /v1/models
+# MODELS
 # ============================================================
 
 @app.get(
     "/v1/models",
     dependencies=[Depends(verify_api_key)],
 )
-def list_models():
+async def models():
+
+    created = create_timestamp()
+
     return {
         "object": "list",
         "data": [
             {
                 "id": MODEL_ID,
                 "object": "model",
-                "created": int(time.time()),
+                "created": created,
                 "owned_by": "varsshancoder",
             }
         ],
     }
 
 
-# ============================================================
-# OPENAI /v1/models/{model_id}
-# ============================================================
-
-@app.get(
-    "/v1/models/{model_id}",
-    dependencies=[Depends(verify_api_key)],
-)
-def get_model(model_id: str):
-
-    if model_id != MODEL_ID:
-        raise HTTPException(
-            status_code=404,
-            detail=f"Model '{model_id}' not found.",
-        )
-
-    return {
-        "id": MODEL_ID,
-        "object": "model",
-        "created": int(time.time()),
-        "owned_by": "varsshancoder",
-    }
-
-
-# ============================================================
-# MESSAGE VALIDATION
-# ============================================================
-
-def clean_messages(messages):
-
-    if not isinstance(messages, list):
-        raise HTTPException(
-            status_code=400,
-            detail="'messages' must be a list.",
-        )
-
-    cleaned = []
-
-    for message in messages:
-
-        if not isinstance(message, dict):
-            raise HTTPException(
-                status_code=400,
-                detail="Each message must be an object.",
-            )
-
-        role = message.get("role", "user")
-        content = message.get("content", "")
-
-        if content is None:
-            content = ""
-
-        if not isinstance(content, str):
-            raise HTTPException(
-                status_code=400,
-                detail=(
-                    "Message content must be a string. "
-                    "Multimodal content is not supported."
-                ),
-            )
-
-        cleaned.append(
-            {
-                "role": role,
-                "content": content,
-            }
-        )
-
-    if not cleaned:
-        raise HTTPException(
-            status_code=400,
-            detail="No messages were provided.",
-        )
-
-    return cleaned
-
-
 # ============================================================
 # CHAT COMPLETIONS
 # ============================================================
@@ -335,46 +385,36 @@ def clean_messages(messages):
 )
 async def chat_completions(request: Request):
 
-    # --------------------------------------------------------
-    # Parse JSON
-    # --------------------------------------------------------
-
     try:
         body = await request.json()
-
     except Exception:
         raise HTTPException(
             status_code=400,
             detail="Invalid JSON request body.",
         )
 
-    # --------------------------------------------------------
-    # Model
-    # --------------------------------------------------------
-
-    requested_model = body.get(
-        "model",
-        MODEL_ID,
-    )
-
-    if requested_model != MODEL_ID:
-
+    if not isinstance(body, dict):
         raise HTTPException(
-            status_code=404,
-            detail=f"Model '{requested_model}' not found.",
+            status_code=400,
+            detail="Request body must be a JSON object.",
         )
 
-    # --------------------------------------------------------
-    # Messages
-    # --------------------------------------------------------
+    messages = normalize_messages(
+        body.get("messages", [])
+    )
 
-    messages = clean_messages(
-        body.get("messages")
+    requested_model = body.get(
+        "model",
+        MODEL_ID,
     )
 
-    # --------------------------------------------------------
-    # Parameters
-    # --------------------------------------------------------
+    # Accept the model exposed by /v1/models.
+    # Also tolerate a provider-prefixed model ID.
+    if requested_model not in {
+        MODEL_ID,
+        f"varsh/{MODEL_ID}",
+    }:
+        requested_model = MODEL_ID
 
     temperature = body.get(
         "temperature",
@@ -383,87 +423,76 @@ async def chat_completions(request: Request):
 
     max_tokens = body.get(
         "max_tokens",
-        1024,
-    )
-
-    top_p = body.get(
-        "top_p",
-        0.95,
+        512,
     )
 
-    stop = body.get(
-        "stop",
-        None,
-    )
-
-    stream = body.get(
-        "stream",
-        False,
+    stream = bool(
+        body.get(
+            "stream",
+            False,
+        )
     )
 
-    # --------------------------------------------------------
-    # Optional seed
-    # --------------------------------------------------------
-
-    seed = body.get(
-        "seed",
-        None,
-    )
+    # Keep values safe for the 2-vCPU deployment.
+    try:
+        temperature = float(temperature)
+    except Exception:
+        temperature = 0.2
 
-    # --------------------------------------------------------
-    # Request metadata
-    # --------------------------------------------------------
+    try:
+        max_tokens = int(max_tokens)
+    except Exception:
+        max_tokens = 512
 
-    request_id = (
-        "chatcmpl-"
-        + uuid.uuid4().hex
-    )
+    if max_tokens < 1:
+        max_tokens = 1
 
-    created = int(
-        time.time()
-    )
+    if max_tokens > 4096:
+        max_tokens = 4096
 
-    # ========================================================
-    # STREAMING
-    # ========================================================
+    if temperature < 0:
+        temperature = 0
 
-    if stream:
+    if temperature > 2:
+        temperature = 2
 
-        def generate_stream():
+    # Optional parameters supported by many OpenAI clients.
+    top_p = body.get("top_p", 0.95)
 
-            start_time = time.perf_counter()
+    try:
+        top_p = float(top_p)
+    except Exception:
+        top_p = 0.95
 
-            try:
+    if top_p <= 0:
+        top_p = 0.01
 
-                # ------------------------------------------------
-                # Lock model generation
-                # ------------------------------------------------
+    if top_p > 1:
+        top_p = 1.0
 
-                with generation_lock:
+    # --------------------------------------------------------
+    # Streaming
+    # --------------------------------------------------------
 
-                    stream_kwargs = {
-                        "messages": messages,
-                        "temperature": float(temperature),
-                        "max_tokens": int(max_tokens),
-                        "top_p": float(top_p),
-                        "stream": True,
-                    }
+    if stream:
 
-                    if stop is not None:
-                        stream_kwargs["stop"] = stop
+        chat_id = create_chat_id()
+        created = create_timestamp()
 
-                    if seed is not None:
-                        stream_kwargs["seed"] = int(seed)
+        async def event_generator():
 
-                    response_stream = (
-                        llm.create_chat_completion(
-                            **stream_kwargs
-                        )
+            try:
+                async with generation_lock:
+
+                    result_stream = llm.create_chat_completion(
+                        messages=messages,
+                        temperature=temperature,
+                        top_p=top_p,
+                        max_tokens=max_tokens,
+                        stream=True,
                     )
 
-                    first_chunk = True
-
-                    for chunk in response_stream:
+                    for chunk in result_stream:
 
                         choices = chunk.get(
                             "choices",
@@ -480,136 +509,51 @@ async def chat_completions(request: Request):
                             {},
                         )
 
-                        finish_reason = choice.get(
-                            "finish_reason",
-                            None,
-                        )
-
-                        # ----------------------------------------
-                        # First chunk
-                        # ----------------------------------------
-
-                        if first_chunk:
-
-                            first_chunk = False
-
-                            first_data = {
-                                "id": request_id,
-                                "object": "chat.completion.chunk",
-                                "created": created,
-                                "model": MODEL_ID,
-                                "choices": [
-                                    {
-                                        "index": 0,
-                                        "delta": {
-                                            "role": "assistant"
-                                        },
-                                        "finish_reason": None,
-                                    }
-                                ],
-                            }
-
-                            yield (
-                                "data: "
-                                + json.dumps(
-                                    first_data,
-                                    ensure_ascii=False,
-                                )
-                                + "\n\n"
-                            )
-
-                        # ----------------------------------------
-                        # Content chunk
-                        # ----------------------------------------
-
                         content = delta.get(
                             "content",
                             "",
                         )
 
-                        if content:
-
-                            data = {
-                                "id": request_id,
-                                "object": "chat.completion.chunk",
-                                "created": created,
-                                "model": MODEL_ID,
-                                "choices": [
-                                    {
-                                        "index": 0,
-                                        "delta": {
-                                            "content": content
-                                        },
-                                        "finish_reason": None,
-                                    }
-                                ],
-                            }
-
-                            yield (
-                                "data: "
-                                + json.dumps(
-                                    data,
-                                    ensure_ascii=False,
-                                )
-                                + "\n\n"
-                            )
+                        if content is None:
+                            content = ""
 
-                        # ----------------------------------------
-                        # Finish reason
-                        # ----------------------------------------
-
-                        if finish_reason is not None:
-
-                            final_data = {
-                                "id": request_id,
-                                "object": "chat.completion.chunk",
-                                "created": created,
-                                "model": MODEL_ID,
-                                "choices": [
-                                    {
-                                        "index": 0,
-                                        "delta": {},
-                                        "finish_reason": finish_reason,
-                                    }
-                                ],
-                            }
-
-                            yield (
-                                "data: "
-                                + json.dumps(
-                                    final_data,
-                                    ensure_ascii=False,
-                                )
-                                + "\n\n"
-                            )
+                        finish_reason = choice.get(
+                            "finish_reason"
+                        )
 
-                # ------------------------------------------------
-                # DONE
-                # ------------------------------------------------
+                        payload = {
+                            "id": chat_id,
+                            "object": "chat.completion.chunk",
+                            "created": created,
+                            "model": requested_model,
+                            "choices": [
+                                {
+                                    "index": 0,
+                                    "delta": {
+                                        "role": "assistant",
+                                        "content": content,
+                                    },
+                                    "finish_reason": finish_reason,
+                                }
+                            ],
+                        }
+
+                        yield (
+                            "data: "
+                            + json.dumps(
+                                payload,
+                                ensure_ascii=False,
+                            )
+                            + "\n\n"
+                        )
 
                 yield "data: [DONE]\n\n"
 
-                elapsed = (
-                    time.perf_counter()
-                    - start_time
-                )
-
-                print(
-                    f"Stream {request_id} "
-                    f"completed in "
-                    f"{elapsed:.2f}s"
-                )
-
-            except Exception as e:
-
-                print(
-                    f"Streaming error "
-                    f"{request_id}: {e}"
-                )
+            except Exception as exc:
 
-                error_data = {
+                error_payload = {
                     "error": {
-                        "message": str(e),
+                        "message": str(exc),
                         "type": "server_error",
                     }
                 }
@@ -617,16 +561,14 @@ async def chat_completions(request: Request):
                 yield (
                     "data: "
                     + json.dumps(
-                        error_data,
+                        error_payload,
                         ensure_ascii=False,
                     )
                     + "\n\n"
                 )
 
-                yield "data: [DONE]\n\n"
-
         return StreamingResponse(
-            generate_stream(),
+            event_generator(),
             media_type="text/event-stream",
             headers={
                 "Cache-Control": "no-cache",
@@ -635,149 +577,64 @@ async def chat_completions(request: Request):
             },
         )
 
-    # ========================================================
-    # NON-STREAMING
-    # ========================================================
-
-    start_time = time.perf_counter()
-
-    try:
-
-        with generation_lock:
-
-            completion_kwargs = {
-                "messages": messages,
-                "temperature": float(temperature),
-                "max_tokens": int(max_tokens),
-                "top_p": float(top_p),
-                "stream": False,
-            }
-
-            if stop is not None:
-                completion_kwargs["stop"] = stop
-
-            if seed is not None:
-                completion_kwargs["seed"] = int(seed)
-
-            response = (
-                llm.create_chat_completion(
-                    **completion_kwargs
-                )
-            )
-
-    except Exception as e:
-
-        print(
-            f"Inference error "
-            f"{request_id}: {e}"
-        )
-
-        raise HTTPException(
-            status_code=500,
-            detail=f"Model inference failed: {str(e)}",
-        )
-
-    elapsed = (
-        time.perf_counter()
-        - start_time
-    )
-
     # --------------------------------------------------------
-    # Extract response
+    # Non-streaming
     # --------------------------------------------------------
 
     try:
 
-        choice = response["choices"][0]
+        async with generation_lock:
 
-        message = choice.get(
-            "message",
-            {},
-        )
+            result = llm.create_chat_completion(
+                messages=messages,
+                temperature=temperature,
+                top_p=top_p,
+                max_tokens=max_tokens,
+                stream=False,
+            )
 
-        text_output = message.get(
-            "content",
-            "",
+        response = build_non_stream_response(
+            result,
+            requested_model,
         )
 
-        finish_reason = choice.get(
-            "finish_reason",
-            "stop",
+        return JSONResponse(
+            content=response
         )
 
-    except Exception as e:
+    except Exception as exc:
 
         print(
-            f"Response parsing error: {e}"
+            f"Generation error: {type(exc).__name__}: {exc}"
         )
 
         raise HTTPException(
             status_code=500,
-            detail=(
-                "Invalid response returned "
-                "by the model."
-            ),
+            detail=f"Generation failed: {str(exc)}",
         )
 
-    # --------------------------------------------------------
-    # Usage
-    # --------------------------------------------------------
-
-    usage = response.get(
-        "usage",
-        {},
-    )
 
-    prompt_tokens = usage.get(
-        "prompt_tokens",
-        0,
-    )
+# ============================================================
+# HEALTH CHECK
+# ============================================================
 
-    completion_tokens = usage.get(
-        "completion_tokens",
-        0,
-    )
+@app.get("/health")
+async def health():
+    return {
+        "status": "ok",
+        "model": MODEL_ID,
+    }
 
-    total_tokens = usage.get(
-        "total_tokens",
-        prompt_tokens + completion_tokens,
-    )
 
-    print(
-        f"Request {request_id} | "
-        f"prompt={prompt_tokens} | "
-        f"completion={completion_tokens} | "
-        f"time={elapsed:.2f}s"
-    )
+# ============================================================
+# STARTUP
+# ============================================================
 
-    # --------------------------------------------------------
-    # OpenAI-compatible response
-    # --------------------------------------------------------
+if __name__ == "__main__":
+    import uvicorn
 
-    return JSONResponse(
-        content={
-            "id": request_id,
-            "object": "chat.completion",
-            "created": created,
-            "model": MODEL_ID,
-            "choices": [
-                {
-                    "index": 0,
-                    "message": {
-                        "role": "assistant",
-                        "content": text_output,
-                    },
-                    "finish_reason": finish_reason,
-                }
-            ],
-            "usage": {
-                "prompt_tokens": prompt_tokens,
-                "completion_tokens": completion_tokens,
-                "total_tokens": total_tokens,
-            },
-            "x_generation_time_seconds": round(
-                elapsed,
-                3,
-            ),
-        }
+    uvicorn.run(
+        app,
+        host=HOST,
+        port=PORT,
     )