Update app.py

app.py

@@ -1,827 +1,1203 @@
+
 import os
-import sys
+import time
+import json
+import shutil
+import tarfile
+import zipfile
+import stat
 import subprocess
-import base64
-import mimetypes
 import threading
+from pathlib import Path
+
+import requests
+import gradio as gr
+from huggingface_hub import hf_hub_download
+
+
+APP_DIR = Path("/app")
+
+MODEL_DIR = APP_DIR / "models"
+MODEL_DIR.mkdir(parents=True, exist_ok=True)
+
+MODEL_REPO = "Qwen/Qwen2.5-Coder-3B-Instruct-GGUF"
+MODEL_FILE = "qwen2.5-coder-3b-instruct-q4_k_m.gguf"
+MODEL_PATH = MODEL_DIR / MODEL_FILE
+
+LLAMA_DIR = APP_DIR / "llama-server"
+LLAMA_EXTRACT_DIR = APP_DIR / "llama-extracted"
+LLAMA_SERVER = LLAMA_DIR / "llama-server"
 
-# ============================================================
-# Qwen2.5-VL-3B GGUF + Gradio
-#
-# This version DOES NOT use llama-server.
-# It uses llama-cpp-python directly from Python.
-#
-# Model:
-#   Qwen2.5-VL-3B-Instruct-Q4_K_M.gguf
-#
-# Vision projector:
-#   mmproj-Qwen2.5-VL-3B-Instruct-Q8_0.gguf
-#
-# Both files are downloaded automatically from Hugging Face.
-# ============================================================
+SERVER_HOST = "127.0.0.1"
+SERVER_PORT = 8080
 
+GRADIO_HOST = "0.0.0.0"
+GRADIO_PORT = 7860
 
-MODEL_REPO = "ggml-org/Qwen2.5-VL-3B-Instruct-GGUF"
+API_KEY = "local-space-key"
 
-MODEL_FILE = "Qwen2.5-VL-3B-Instruct-Q4_K_M.gguf"
+PARALLEL_USERS = 4
+CONTEXT_PER_USER = 2048
+TOTAL_CONTEXT = PARALLEL_USERS * CONTEXT_PER_USER
 
-MMPROJ_FILE = "mmproj-Qwen2.5-VL-3B-Instruct-Q8_0.gguf"
+CPU_THREADS = int(
+    os.environ.get("CPU_THREADS", "4")
+)
 
-MODEL_DIR = os.environ.get(
-    "MODEL_DIR",
-    "/tmp/qwen25vl"
+CPU_THREADS_BATCH = int(
+    os.environ.get(
+        "CPU_THREADS_BATCH",
+        str(CPU_THREADS)
+    )
 )
 
-MODEL_PATH = os.path.join(
-    MODEL_DIR,
-    MODEL_FILE
+BATCH_SIZE = int(
+    os.environ.get("BATCH_SIZE", "512")
 )
 
-MMPROJ_PATH = os.path.join(
-    MODEL_DIR,
-    MMPROJ_FILE
+MAX_TOKENS = int(
+    os.environ.get("MAX_TOKENS", "512")
 )
 
+TEMPERATURE = float(
+    os.environ.get("TEMPERATURE", "0.2")
+)
 
-# ============================================================
-# SETTINGS
-# ============================================================
+TOP_P = float(
+    os.environ.get("TOP_P", "0.9")
+)
 
-N_CTX = 4096
+server_process = None
+server_lock = threading.Lock()
 
-# -1 = try to offload all supported layers to GPU.
-#
-# If your Space has no CUDA-enabled llama-cpp-python,
-# llama.cpp will run on CPU instead.
-N_GPU_LAYERS = -1
 
-N_THREADS = max(
-    1,
-    os.cpu_count() or 4
-)
+def download_model():
 
-N_BATCH = 512
+    if MODEL_PATH.exists():
 
-DEFAULT_MAX_TOKENS = 1024
+        size_gb = (
+            MODEL_PATH.stat().st_size
+            / 1024
+            / 1024
+            / 1024
+        )
+
+        print(
+            f"[MODEL] Already exists: {MODEL_PATH}"
+        )
 
+        print(
+            f"[MODEL] Size: {size_gb:.2f} GB"
+        )
 
-# Global model
-llm = None
+        return str(MODEL_PATH)
 
-model_lock = threading.Lock()
+    print(
+        "[MODEL] Downloading Qwen2.5-Coder-3B Q4_K_M..."
+    )
 
+    hf_token = os.environ.get("HF_TOKEN")
 
-# ============================================================
-# LOG
-# ============================================================
+    kwargs = {
+        "repo_id": MODEL_REPO,
+        "filename": MODEL_FILE,
+        "local_dir": str(MODEL_DIR),
+    }
 
-def log(message=""):
-    print(message, flush=True)
+    if hf_token:
+        kwargs["token"] = hf_token
 
+    downloaded = hf_hub_download(
+        **kwargs
+    )
 
-# ============================================================
-# INSTALL / CHECK PYTHON DEPENDENCIES
-# ============================================================
+    print(
+        f"[MODEL] Download complete: {downloaded}"
+    )
 
-def ensure_dependencies():
+    return downloaded
 
-    missing = []
 
-    try:
-        import gradio
-    except ImportError:
-        missing.append("gradio")
+def find_llama_server(directory):
 
-    try:
-        import requests
-    except ImportError:
-        missing.append("requests")
+    directory = Path(directory)
 
-    try:
-        import huggingface_hub
-    except ImportError:
-        missing.append("huggingface_hub")
+    for path in directory.rglob(
+        "llama-server"
+    ):
 
-    try:
-        import llama_cpp
-    except ImportError:
-        missing.append("llama-cpp-python")
+        if path.is_file():
 
-    if not missing:
-        return
+            try:
+                path.chmod(
+                    path.stat().st_mode
+                    | stat.S_IXUSR
+                    | stat.S_IXGRP
+                    | stat.S_IXOTH
+                )
+            except Exception:
+                pass
 
-    log("=" * 70)
-    log("INSTALLING MISSING PYTHON PACKAGES")
-    log("=" * 70)
+            return path
 
-    packages = []
+    return None
 
-    for package in missing:
 
-        if package == "llama-cpp-python":
-            packages.append(
-                "llama-cpp-python>=0.3.35"
-            )
-        else:
-            packages.append(package)
+def find_cpu_asset():
 
-    subprocess.check_call(
-        [
-            sys.executable,
-            "-m",
-            "pip",
-            "install",
-            "-U",
-            *packages,
-        ]
+    api_url = (
+        "https://api.github.com/"
+        "repos/ggml-org/llama.cpp/releases"
     )
 
+    headers = {
+        "Accept":
+            "application/vnd.github+json",
+        "User-Agent":
+            "hf-space-qwen-llama-server"
+    }
 
-ensure_dependencies()
+    print(
+        "[LLAMA] Searching llama.cpp releases..."
+    )
 
+    response = requests.get(
+        api_url,
+        params={
+            "per_page": 20
+        },
+        headers=headers,
+        timeout=30
+    )
 
-# Imports after dependency check.
-import gradio as gr
+    response.raise_for_status()
 
-from huggingface_hub import hf_hub_download
+    releases = response.json()
 
-from llama_cpp import Llama
+    if not isinstance(
+        releases,
+        list
+    ):
+        raise RuntimeError(
+            "Invalid GitHub releases response."
+        )
 
-try:
-    from llama_cpp.llama_chat_format import Qwen25VLChatHandler
-except ImportError:
-    Qwen25VLChatHandler = None
+    stable = []
 
+    for release in releases:
 
-# ============================================================
-# DOWNLOAD MODEL
-# ============================================================
+        if not release.get(
+            "draft",
+            False
+        ) and not release.get(
+            "prerelease",
+            False
+        ):
+            stable.append(release)
 
-def download_models():
+    releases = stable + [
+        r for r in releases
+        if r not in stable
+    ]
 
-    os.makedirs(
-        MODEL_DIR,
-        exist_ok=True
-    )
+    for release in releases:
 
-    log("=" * 70)
-    log("DOWNLOADING / CHECKING QWEN2.5-VL MODEL")
-    log("=" * 70)
+        tag = release.get(
+            "tag_name",
+            "unknown"
+        )
 
-    log(
-        "Repository: "
-        + MODEL_REPO
-    )
+        assets = release.get(
+            "assets",
+            []
+        )
 
-    log(
-        "Model: "
-        + MODEL_FILE
-    )
+        print(
+            f"[LLAMA] Checking {tag}"
+        )
+
+        for asset in assets:
+
+            name = asset.get(
+                "name",
+                ""
+            )
+
+            lower = name.lower()
+
+            special = any(
+                x in lower
+                for x in (
+                    "cuda",
+                    "vulkan",
+                    "rocm",
+                    "openvino",
+                    "sycl",
+                    "metal"
+                )
+            )
 
-    log(
-        "MMProj: "
-        + MMPROJ_FILE
+            current_cpu = (
+                "ubuntu-x64" in lower
+                and lower.endswith(
+                    ".tar.gz"
+                )
+            )
+
+            old_cpu = (
+                "ubuntu" in lower
+                and "x64" in lower
+                and (
+                    "cpu" in lower
+                    or "bin-" in lower
+                )
+                and lower.endswith(
+                    (
+                        ".tar.gz",
+                        ".zip"
+                    )
+                )
+            )
+
+            if (
+                not special
+                and (
+                    current_cpu
+                    or old_cpu
+                )
+            ):
+
+                print(
+                    f"[LLAMA] Found CPU build: {name}"
+                )
+
+                return asset
+
+    print(
+        "[LLAMA] No compatible CPU build found."
     )
 
-    # Main Q4_K_M model.
-    if os.path.isfile(MODEL_PATH):
+    for release in releases[:5]:
 
-        log(
-            "[MODEL] Already exists: "
-            + MODEL_PATH
+        print(
+            "Release:",
+            release.get(
+                "tag_name",
+                "unknown"
+            )
         )
 
-    else:
+        for asset in release.get(
+            "assets",
+            []
+        ):
 
-        log(
-            "[MODEL] Downloading Q4_K_M..."
-        )
+            print(
+                " ",
+                asset.get(
+                    "name",
+                    ""
+                )
+            )
 
-        downloaded_model = hf_hub_download(
-            repo_id=MODEL_REPO,
-            filename=MODEL_FILE,
-            local_dir=MODEL_DIR,
-            local_dir_use_symlinks=False,
-        )
+    raise RuntimeError(
+        "Could not find Ubuntu x64 CPU "
+        "llama.cpp release asset."
+    )
 
-        log(
-            "[MODEL] Downloaded: "
-            + downloaded_model
-        )
 
-    # Vision projector.
-    if os.path.isfile(MMPROJ_PATH):
+def download_llama_server():
 
-        log(
-            "[MMPROJ] Already exists: "
-            + MMPROJ_PATH
-        )
+    if LLAMA_SERVER.exists():
 
-    else:
+        try:
+            LLAMA_SERVER.chmod(
+                LLAMA_SERVER.stat().st_mode
+                | stat.S_IXUSR
+                | stat.S_IXGRP
+                | stat.S_IXOTH
+            )
+        except Exception:
+            pass
 
-        log(
-            "[MMPROJ] Downloading Q8_0 projector..."
+        print(
+            f"[LLAMA] Existing server found: "
+            f"{LLAMA_SERVER}"
         )
 
-        downloaded_mmproj = hf_hub_download(
-            repo_id=MODEL_REPO,
-            filename=MMPROJ_FILE,
-            local_dir=MODEL_DIR,
-            local_dir_use_symlinks=False,
+        return str(
+            LLAMA_SERVER
         )
 
-        log(
-            "[MMPROJ] Downloaded: "
-            + downloaded_mmproj
+    print(
+        "[LLAMA] Downloading prebuilt llama.cpp server..."
+    )
+
+    asset = find_cpu_asset()
+
+    asset_name = asset[
+        "name"
+    ]
+
+    download_url = asset[
+        "browser_download_url"
+    ]
+
+    archive_path = (
+        APP_DIR /
+        asset_name
+    )
+
+    print(
+        f"[LLAMA] Selected: {asset_name}"
+    )
+
+    if not archive_path.exists():
+
+        print(
+            "[LLAMA] Downloading archive..."
         )
 
-    if not os.path.isfile(MODEL_PATH):
+        headers = {
+            "User-Agent":
+                "hf-space-qwen-llama-server"
+        }
+
+        with requests.get(
+            download_url,
+            headers=headers,
+            stream=True,
+            timeout=120
+        ) as response:
+
+            response.raise_for_status()
+
+            total = int(
+                response.headers.get(
+                    "content-length",
+                    0
+                )
+            )
+
+            downloaded = 0
+            last_print = 0
+
+            with open(
+                archive_path,
+                "wb"
+            ) as f:
+
+                for chunk in response.iter_content(
+                    chunk_size=1024 * 1024
+                ):
+
+                    if not chunk:
+                        continue
+
+                    f.write(chunk)
+
+                    downloaded += len(
+                        chunk
+                    )
+
+                    if total:
+
+                        now = time.time()
+
+                        if now - last_print >= 0.5:
+
+                            percent = (
+                                downloaded
+                                * 100
+                                / total
+                            )
 
-        raise FileNotFoundError(
-            "Main Qwen model was not found:\n"
-            + MODEL_PATH
+                            print(
+                                f"\r[LLAMA] "
+                                f"{percent:5.1f}% "
+                                f"{downloaded / 1024 / 1024:.1f} MB / "
+                                f"{total / 1024 / 1024:.1f} MB",
+                                end="",
+                                flush=True
+                            )
+
+                            last_print = now
+
+        print()
+
+        print(
+            "[LLAMA] Archive download complete."
         )
 
-    if not os.path.isfile(MMPROJ_PATH):
+    else:
 
-        raise FileNotFoundError(
-            "Qwen vision projector was not found:\n"
-            + MMPROJ_PATH
+        print(
+            "[LLAMA] Archive already exists."
         )
 
-    log(
-        "[MODEL] Size: "
-        + str(
-            os.path.getsize(MODEL_PATH)
-            / (1024 * 1024 * 1024)
+    if LLAMA_EXTRACT_DIR.exists():
+
+        shutil.rmtree(
+            LLAMA_EXTRACT_DIR
         )
-        + " GB"
+
+    LLAMA_EXTRACT_DIR.mkdir(
+        parents=True,
+        exist_ok=True
     )
 
-    log(
-        "[MMPROJ] Size: "
-        + str(
-            os.path.getsize(MMPROJ_PATH)
-            / (1024 * 1024 * 1024)
-        )
-        + " GB"
+    print(
+        "[LLAMA] Extracting..."
     )
 
+    if asset_name.lower().endswith(
+        ".tar.gz"
+    ):
+
+        with tarfile.open(
+            archive_path,
+            "r:gz"
+        ) as tar:
 
-# ============================================================
-# LOAD QWEN2.5-VL
-# ============================================================
+            tar.extractall(
+                LLAMA_EXTRACT_DIR
+            )
 
-def load_model():
+    elif asset_name.lower().endswith(
+        ".zip"
+    ):
 
-    global llm
+        with zipfile.ZipFile(
+            archive_path,
+            "r"
+        ) as z:
 
-    if llm is not None:
-        return
+            z.extractall(
+                LLAMA_EXTRACT_DIR
+            )
 
-    if Qwen25VLChatHandler is None:
+    else:
 
         raise RuntimeError(
-            "Qwen25VLChatHandler is not available in the "
-            "installed llama-cpp-python package.\n\n"
-            "Please use llama-cpp-python 0.3.35 or newer."
+            f"Unsupported archive: {asset_name}"
         )
 
-    download_models()
+    found = find_llama_server(
+        LLAMA_EXTRACT_DIR
+    )
 
-    log("=" * 70)
-    log("LOADING QWEN2.5-VL-3B")
-    log("=" * 70)
+    if found is None:
 
-    log(
-        "Model path: "
-        + MODEL_PATH
-    )
+        for path in LLAMA_EXTRACT_DIR.rglob(
+            "*"
+        ):
 
-    log(
-        "MMProj path: "
-        + MMPROJ_PATH
-    )
+            if path.is_file():
+                print(
+                    path
+                )
 
-    log(
-        "Context: "
-        + str(N_CTX)
-    )
+        raise RuntimeError(
+            "llama-server was not found "
+            "inside the archive."
+        )
 
-    log(
-        "GPU layers: "
-        + str(N_GPU_LAYERS)
+    print(
+        f"[LLAMA] Found: {found}"
     )
 
-    log(
-        "CPU threads: "
-        + str(N_THREADS)
-    )
+    source_dir = found.parent
 
-    log(
-        "Batch: "
-        + str(N_BATCH)
+    LLAMA_DIR.mkdir(
+        parents=True,
+        exist_ok=True
     )
 
-    # Qwen2.5-VL uses a dedicated multimodal chat handler.
-    #
-    # llama-cpp-python 0.3.35 uses clip_model_path for
-    # the multimodal projector.
-    chat_handler = Qwen25VLChatHandler(
-        clip_model_path=MMPROJ_PATH,
-        verbose=False,
-    )
+    for item in source_dir.iterdir():
 
-    llm = Llama(
-        model_path=MODEL_PATH,
+        destination = (
+            LLAMA_DIR /
+            item.name
+        )
 
-        chat_handler=chat_handler,
+        if item.is_dir():
 
-        n_ctx=N_CTX,
+            if destination.exists():
+                shutil.rmtree(
+                    destination
+                )
 
-        n_batch=N_BATCH,
+            shutil.copytree(
+                item,
+                destination
+            )
+
+        else:
+
+            shutil.copy2(
+                item,
+                destination
+            )
 
-        n_threads=N_THREADS,
+    final_server = (
+        LLAMA_DIR /
+        found.name
+    )
 
-        n_gpu_layers=N_GPU_LAYERS,
+    if not final_server.exists():
 
-        use_mmap=True,
+        raise RuntimeError(
+            "Failed to install llama-server."
+        )
 
-        verbose=False,
+    final_server.chmod(
+        final_server.stat().st_mode
+        | stat.S_IXUSR
+        | stat.S_IXGRP
+        | stat.S_IXOTH
     )
 
-    log("=" * 70)
-    log("QWEN2.5-VL READY")
-    log("=" * 70)
+    library_dirs = []
 
+    for root, dirs, files in os.walk(
+        LLAMA_DIR
+    ):
+
+        for filename in files:
 
-# ============================================================
-# IMAGE -> BASE64 DATA URI
-# ============================================================
+            if (
+                filename.endswith(".so")
+                or ".so." in filename
+            ):
+
+                library_dirs.append(
+                    root
+                )
 
-def image_to_data_uri(image_path):
+                break
 
-    if not image_path:
-        raise ValueError(
-            "No image was supplied."
+    library_dirs = list(
+        dict.fromkeys(
+            library_dirs
         )
+    )
 
-    if not os.path.isfile(image_path):
+    if library_dirs:
 
-        raise FileNotFoundError(
-            image_path
+        old_ld = os.environ.get(
+            "LD_LIBRARY_PATH",
+            ""
         )
 
-    mime_type, _ = mimetypes.guess_type(
-        image_path
-    )
+        new_ld = ":".join(
+            library_dirs
+        )
+
+        if old_ld:
 
-    if not mime_type:
-        mime_type = "image/jpeg"
+            os.environ[
+                "LD_LIBRARY_PATH"
+            ] = (
+                new_ld
+                + ":"
+                + old_ld
+            )
+
+        else:
 
-    with open(
-        image_path,
-        "rb"
-    ) as f:
+            os.environ[
+                "LD_LIBRARY_PATH"
+            ] = new_ld
 
-        encoded = base64.b64encode(
-            f.read()
-        ).decode(
-            "utf-8"
+        print(
+            "[LLAMA] Libraries configured."
         )
 
-    return (
-        "data:"
-        + mime_type
-        + ";base64,"
-        + encoded
+    print(
+        f"[LLAMA] Server installed: "
+        f"{final_server}"
     )
 
+    return str(
+        final_server
+    )
 
-# ============================================================
-# DEFAULT OCR INSTRUCTION
-# ============================================================
 
-DEFAULT_INSTRUCTION = """Read all visible text in this image.
+def start_llama_server():
 
-Use natural reading order:
-1. Top to bottom.
-2. Left to right within each line.
-3. Follow columns in their natural order.
-4. Preserve paragraphs and line breaks where possible.
-5. Do not describe the image.
-6. Return only the text you can read.
-7. Do not invent missing or unclear text."""
+    global server_process
 
+    with server_lock:
 
-# ============================================================
-# GENERATE STREAM
-# ============================================================
+        if server_process is not None:
 
-def generate(
-    image,
-    instruction,
-    max_tokens,
-):
+            if (
+                server_process.poll()
+                is None
+            ):
 
-    if not image:
+                return
 
-        raise gr.Error(
-            "Please upload an image."
+        server_path = (
+            download_llama_server()
         )
 
-    if (
-        instruction is None
-        or not instruction.strip()
-    ):
-
-        instruction = DEFAULT_INSTRUCTION
+        command = [
+            server_path,
+            "-m",
+            str(MODEL_PATH),
+            "-np",
+            str(PARALLEL_USERS),
+            "-c",
+            str(TOTAL_CONTEXT),
+            "--cont-batching",
+            "-ngl",
+            "0",
+            "-t",
+            str(CPU_THREADS),
+            "-tb",
+            str(CPU_THREADS_BATCH),
+            "-b",
+            str(BATCH_SIZE),
+            "--host",
+            SERVER_HOST,
+            "--port",
+            str(SERVER_PORT),
+            "--api-key",
+            API_KEY
+        ]
 
-    try:
+        print(
+            "[LLAMA] Starting server..."
+        )
 
-        image_data_uri = image_to_data_uri(
-            image
+        server_process = subprocess.Popen(
+            command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            bufsize=1,
+            env=os.environ.copy()
         )
 
-    except Exception as e:
+        def read_logs():
 
-        raise gr.Error(
-            "Image error:\n"
-            + str(e)
-        )
+            try:
 
-    if llm is None:
+                for line in iter(
+                    server_process.stdout.readline,
+                    ""
+                ):
 
-        raise gr.Error(
-            "The Qwen2.5-VL model is not loaded."
-        )
+                    if line:
+                        print(
+                            "[llama.cpp]",
+                            line.rstrip()
+                        )
 
-    messages = [
-        {
-            "role": "system",
-            "content": (
-                "You are an image reading assistant. "
-                "Follow the user's instruction exactly. "
-                "When extracting text, read it in natural "
-                "reading order from top to bottom and left "
-                "to right. Never invent text."
-            ),
-        },
-        {
-            "role": "user",
-            "content": [
-                {
-                    "type": "image_url",
-                    "image_url": {
-                        "url": image_data_uri
-                    },
-                },
-                {
-                    "type": "text",
-                    "text": instruction,
-                },
-            ],
-        },
-    ]
+            except Exception:
+                pass
 
-    log("=" * 70)
-    log("IMAGE REQUEST")
-    log("=" * 70)
+        threading.Thread(
+            target=read_logs,
+            daemon=True
+        ).start()
 
-    log(
-        "Instruction:\n"
-        + instruction
+    health_url = (
+        f"http://{SERVER_HOST}:"
+        f"{SERVER_PORT}/health"
     )
 
-    log("=" * 70)
+    print(
+        "[LLAMA] Waiting for server..."
+    )
 
-    output = ""
+    started = time.time()
 
-    try:
+    while True:
 
-        # Only one generation at a time.
-        # This avoids multiple users competing for the
-        # same llama.cpp model context.
-        with model_lock:
+        if (
+            server_process is not None
+            and server_process.poll()
+            is not None
+        ):
 
-            stream = llm.create_chat_completion(
-                messages=messages,
+            raise RuntimeError(
+                "llama-server stopped "
+                f"with code "
+                f"{server_process.returncode}"
+            )
 
-                max_tokens=int(
-                    max_tokens
-                ),
+        try:
 
-                temperature=0.0,
+            response = requests.get(
+                health_url,
+                timeout=2
+            )
+
+            if response.status_code in (
+                200,
+                204
+            ):
+
+                print(
+                    "[LLAMA] Server is ready."
+                )
 
-                top_p=1.0,
+                return
 
-                stream=True,
+        except Exception:
+            pass
+
+        if (
+            time.time() - started
+            > 180
+        ):
+
+            raise RuntimeError(
+                "Timed out waiting for llama-server."
             )
 
-            for chunk in stream:
+        time.sleep(1)
 
-                if not chunk:
-                    continue
 
-                choices = chunk.get(
-                    "choices",
-                    []
-                )
+def clean_content(content):
 
-                if not choices:
-                    continue
+    if content is None:
+        return ""
 
-                choice = choices[0]
+    if isinstance(
+        content,
+        str
+    ):
+        return content
 
-                delta = choice.get(
-                    "delta",
-                    {}
-                )
+    if isinstance(
+        content,
+        list
+    ):
 
-                token = delta.get(
-                    "content",
-                    ""
+        result = []
+
+        for item in content:
+
+            if isinstance(
+                item,
+                str
+            ):
+
+                result.append(
+                    item
                 )
 
-                if token:
+            elif isinstance(
+                item,
+                dict
+            ):
 
-                    output += token
+                if "text" in item:
 
-                    yield output
+                    result.append(
+                        str(
+                            item["text"]
+                        )
+                    )
 
-    except Exception as e:
+                elif "content" in item:
 
-        log("=" * 70)
-        log("INFERENCE ERROR")
-        log("=" * 70)
-        log(repr(e))
-        log("=" * 70)
+                    result.append(
+                        str(
+                            item["content"]
+                        )
+                    )
 
-        raise gr.Error(
-            "Qwen2.5-VL inference failed:\n\n"
-            + str(e)
-        )
+            else:
 
-    if not output:
+                result.append(
+                    str(item)
+                )
 
-        yield (
-            "The model returned no text."
+        return "\n".join(
+            x for x in result
+            if x
         )
 
+    if isinstance(
+        content,
+        dict
+    ):
 
-# ============================================================
-# CLEAR
-# ============================================================
+        if "text" in content:
 
-def clear():
+            return str(
+                content["text"]
+            )
 
-    return (
-        None,
-        DEFAULT_INSTRUCTION,
-        "",
-    )
+        if "content" in content:
 
+            return str(
+                content["content"]
+            )
 
-# ============================================================
-# GRADIO UI
-# ============================================================
+        return json.dumps(
+            content,
+            ensure_ascii=False
+        )
 
-def create_ui():
+    return str(content)
+
+
+def build_messages(
+    message,
+    history
+):
+
+    messages = []
 
-    with gr.Blocks(
-        title="Qwen2.5-VL 3B Image Reader"
-    ) as demo:
+    history = history or []
 
-        gr.Markdown(
-            """
-# Qwen2.5-VL 3B Image Reader
+    recent_history = history[-10:]
 
-Upload an image and give Qwen2.5-VL an instruction.
+    for item in recent_history:
 
-The response is streamed as the model generates it.
-"""
+        if not isinstance(
+            item,
+            dict
+        ):
+            continue
+
+        role = item.get(
+            "role"
         )
 
-        with gr.Row():
+        content = clean_content(
+            item.get(
+                "content",
+                ""
+            )
+        )
 
-            # =================================================
-            # INPUT
-            # =================================================
+        if role in (
+            "user",
+            "assistant",
+            "system"
+        ) and content:
+
+            messages.append({
+                "role":
+                    role,
+                "content":
+                    content
+            })
+
+    messages.append({
+        "role":
+            "user",
+        "content":
+            clean_content(
+                message
+            )
+    })
 
-            with gr.Column(
-                scale=1
-            ):
+    return messages
 
-                image = gr.Image(
-                    label="Upload Image",
-                    type="filepath",
-                    sources=[
-                        "upload",
-                        "clipboard",
-                    ],
-                )
 
-                instruction = gr.Textbox(
-                    label="Instruction",
-                    value=DEFAULT_INSTRUCTION,
-                    lines=10,
-                    placeholder=(
-                        "Tell Qwen2.5-VL what "
-                        "to do with the image..."
-                    ),
-                )
+def generate_response(
+    message,
+    history
+):
 
-                max_tokens = gr.Slider(
-                    minimum=64,
-                    maximum=4096,
-                    value=DEFAULT_MAX_TOKENS,
-                    step=64,
-                    label="Maximum output tokens",
-                )
+    if not message:
+        return "Please enter a message."
 
-                with gr.Row():
+    messages = build_messages(
+        message,
+        history
+    )
 
-                    read_button = gr.Button(
-                        "Read Image",
-                        variant="primary",
-                    )
+    url = (
+        f"http://{SERVER_HOST}:"
+        f"{SERVER_PORT}/v1/chat/completions"
+    )
 
-                    clear_button = gr.Button(
-                        "Clear"
-                    )
+    payload = {
+        "model":
+            "qwen2.5-coder-3b",
+        "messages":
+            messages,
+        "temperature":
+            TEMPERATURE,
+        "top_p":
+            TOP_P,
+        "max_tokens":
+            MAX_TOKENS,
+        "stream":
+            False
+    }
+
+    headers = {
+        "Content-Type":
+            "application/json",
+        "Authorization":
+            f"Bearer {API_KEY}"
+    }
 
-            # =================================================
-            # OUTPUT
-            # =================================================
+    try:
 
-            with gr.Column(
-                scale=1
-            ):
+        response = requests.post(
+            url,
+            headers=headers,
+            json=payload,
+            timeout=300
+        )
 
-                output = gr.Textbox(
-                    label="Model Response",
-                    lines=25,
-                    show_copy_button=True,
-                    autoscroll=True,
-                )
+    except requests.RequestException as e:
 
-        gr.Markdown(
-            """
-### Example
+        return (
+            "Server connection error:\n\n"
+            f"{e}"
+        )
 
-**OCR**
+    if response.status_code != 200:
 
-```text
-Read all visible text from top to bottom and left to right.
-Return only the text.
-```
+        try:
 
-**Question**
+            data = response.json()
 
-```text
-What does the sign in this image say?
-```
+            return (
+                "llama.cpp error:\n\n"
+                + json.dumps(
+                    data,
+                    indent=2,
+                    ensure_ascii=False
+                )
+            )
 
-**Specific region**
+        except Exception:
 
-```text
-Read only the text in the upper half of the image.
-```
+            return (
+                f"llama.cpp HTTP "
+                f"{response.status_code}\n\n"
+                f"{response.text}"
+            )
 
-**Extraction**
+    try:
 
-```text
-Extract all names and phone numbers visible in the image.
-```
-"""
+        data = response.json()
+
+        choices = data.get(
+            "choices",
+            []
         )
 
-        read_button.click(
-            fn=generate,
+        if not choices:
+
+            return "No response generated."
 
-            inputs=[
-                image,
-                instruction,
-                max_tokens,
-            ],
+        choice = choices[0]
+
+        message_data = choice.get(
+            "message",
+            {}
+        )
 
-            outputs=output,
+        result = message_data.get(
+            "content",
+            ""
         )
 
-        clear_button.click(
-            fn=clear,
+        if not result:
 
-            inputs=[],
+            result = choice.get(
+                "text",
+                ""
+            )
+
+        return result.strip()
 
-            outputs=[
-                image,
-                instruction,
-                output,
-            ],
+    except Exception as e:
+
+        return (
+            "Response parsing error:\n\n"
+            f"{e}\n\n"
+            f"{response.text}"
         )
 
+
+def create_ui():
+
+    demo = gr.ChatInterface(
+        fn=generate_response,
+        concurrency_limit=PARALLEL_USERS
+    )
+
     return demo
 
 
-# ============================================================
-# MAIN
-# ============================================================
+def initialize():
 
-def main():
+    print(
+        "============================================================"
+    )
 
-    log("=" * 70)
-    log("APP.PY STARTED")
-    log("=" * 70)
+    print(
+        "Qwen2.5-Coder-3B llama.cpp server"
+    )
 
-    log(
-        "Python: "
-        + sys.version
+    print(
+        "============================================================"
     )
 
-    log(
-        "Gradio: "
-        + getattr(
-            gr,
-            "__version__",
-            "unknown",
-        )
+    print(
+        f"[CONFIG] Users: {PARALLEL_USERS}"
+    )
+
+    print(
+        f"[CONFIG] Context/user: {CONTEXT_PER_USER}"
     )
 
-    log(
-        "Model: "
-        + MODEL_REPO
+    print(
+        f"[CONFIG] CPU threads: {CPU_THREADS}"
     )
 
-    log(
-        "Q4 model: "
-        + MODEL_FILE
+    print(
+        f"[CONFIG] Batch threads: {CPU_THREADS_BATCH}"
     )
 
-    log(
-        "MMProj: "
-        + MMPROJ_FILE
+    print(
+        f"[CONFIG] Batch size: {BATCH_SIZE}"
     )
 
-    log(
-        "Context: "
-        + str(N_CTX)
+    print(
+        f"[CONFIG] Max tokens: {MAX_TOKENS}"
     )
 
-    log(
-        "GPU layers: "
-        + str(N_GPU_LAYERS)
+    print(
+        "============================================================"
     )
 
-    log("=" * 70)
+    download_model()
 
-    # Load model before opening Gradio.
-    load_model()
+    download_llama_server()
 
-    demo = create_ui()
+    start_llama_server()
 
-    # Queue enables generator-based streaming.
-    demo.queue(
-        max_size=20,
-        default_concurrency_limit=1,
+    print(
+        "============================================================"
     )
 
-    log("=" * 70)
-    log("STARTING GRADIO")
-    log("=" * 70)
+    print(
+        "INITIALIZATION COMPLETE"
+    )
 
-    demo.launch(
-        server_name="0.0.0.0",
-        server_port=7860,
-        show_error=True,
+    print(
+        f"Model: {MODEL_FILE}"
     )
 
+    print(
+        f"Users: {PARALLEL_USERS}"
+    )
+
+    print(
+        f"Context/user: {CONTEXT_PER_USER}"
+    )
+
+    print(
+        f"CPU threads: {CPU_THREADS}"
+    )
+
+    print(
+        "============================================================"
+    )
+
+
+def shutdown():
+
+    global server_process
+
+    print(
+        "[SYSTEM] Shutting down..."
+    )
+
+    if server_process is None:
+        return
+
+    try:
+
+        if (
+            server_process.poll()
+            is None
+        ):
+
+            server_process.terminate()
+
+            try:
+
+                server_process.wait(
+                    timeout=5
+                )
+
+            except subprocess.TimeoutExpired:
+
+                server_process.kill()
+
+    except Exception as e:
+
+        print(
+            f"[SYSTEM] Shutdown error: {e}"
+        )
 
-# ============================================================
-# ENTRY POINT
-# ============================================================
 
 if __name__ == "__main__":
 
     try:
 
-        main()
+        initialize()
+
+        demo = create_ui()
+
+        demo.queue(
+            default_concurrency_limit=
+                PARALLEL_USERS
+        )
+
+        demo.launch(
+            server_name=
+                GRADIO_HOST,
+            server_port=
+                GRADIO_PORT,
+            show_error=True,
+            max_threads=
+                PARALLEL_USERS + 4,
+            share=False
+        )
 
     except KeyboardInterrupt:
 
-        log(
+        print(
             "[SYSTEM] Interrupted."
         )
 
     except Exception as e:
 
-        log("=" * 70)
-        log("STARTUP FAILED")
-        log("=" * 70)
+        print(
+            "============================================================"
+        )
+
+        print(
+            "STARTUP FAILED"
+        )
 
-        log(
+        print(
             repr(e)
         )
 
-        log("=" * 70)
+        print(
+            "============================================================"
+        )
 
         raise
+
+    finally:
+
+        shutdown()