Add gateway API source for one-click deployment

apps/api/Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libgl1 libglib2.0-0 curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY apps/api/pyproject.toml /app/apps/api/pyproject.toml
+RUN pip install --no-cache-dir -e /app/apps/api
+
+COPY apps/api /app/apps/api
+COPY AGENTS.md README.md /app/
+
+ENV PYTHONPATH=/app/apps/api
+EXPOSE 8000
+
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--app-dir", "/app/apps/api"]

apps/api/app/__init__.py

@@ -0,0 +1 @@
+

apps/api/app/api/chat_completions.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException
+
+from app.orchestration.runtime import Runtime
+from app.schemas.openai import ChatCompletionRequest, ResponsesRequest
+
+router = APIRouter(prefix="/v1/chat")
+
+
+def get_runtime() -> Runtime:
+    from app.main import runtime
+
+    return runtime
+
+
+@router.post("/completions")
+async def create_chat_completion(request: ChatCompletionRequest, runtime: Runtime = Depends(get_runtime)) -> dict:
+    try:
+        response = await runtime.handle_responses(
+            ResponsesRequest(model=request.model, input=request.messages)
+        )
+    except Exception as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+    return {
+        "id": response["id"],
+        "object": "chat.completion",
+        "created": response["created_at"],
+        "model": response["model"],
+        "choices": [
+            {
+                "index": 0,
+                "message": {"role": "assistant", "content": response["output_text"]},
+                "finish_reason": "stop",
+            }
+        ],
+    }

apps/api/app/api/files.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile
+from fastapi.responses import Response
+
+from app.orchestration.runtime import Runtime
+from app.schemas.openai import FileObject
+
+router = APIRouter(prefix="/v1/files")
+
+
+def get_runtime() -> Runtime:
+    from app.main import runtime
+
+    return runtime
+
+
+@router.post("", response_model=FileObject)
+async def create_file(
+    file: UploadFile = File(...),
+    purpose: str = Form(default="vision"),
+    runtime: Runtime = Depends(get_runtime),
+) -> dict:
+    try:
+        data = await file.read()
+        record = await runtime.create_file(file.filename or "upload", data, file.content_type or "application/octet-stream")
+    except Exception as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+    return {
+        "id": record.id,
+        "object": "file",
+        "bytes": record.bytes,
+        "created_at": record.created_at,
+        "filename": record.filename,
+        "purpose": purpose,
+        "mime_type": record.mime_type,
+    }
+
+
+@router.get("/{file_id}", response_model=FileObject)
+async def get_file(file_id: str, runtime: Runtime = Depends(get_runtime)) -> dict:
+    record = runtime.get_file(file_id)
+    if not record:
+        raise HTTPException(status_code=404, detail="file not found")
+    return {
+        "id": record.id,
+        "object": "file",
+        "bytes": record.bytes,
+        "created_at": record.created_at,
+        "filename": record.filename,
+        "purpose": "vision",
+        "mime_type": record.mime_type,
+    }
+
+
+@router.get("/{file_id}/content")
+async def get_file_content(file_id: str, runtime: Runtime = Depends(get_runtime)) -> Response:
+    try:
+        record, data = runtime.get_file_bytes(file_id)
+    except KeyError as exc:
+        raise HTTPException(status_code=404, detail="file not found") from exc
+    return Response(content=data, media_type=record.mime_type)

apps/api/app/api/health.py

@@ -0,0 +1,13 @@
+from fastapi import APIRouter
+
+router = APIRouter()
+
+
+@router.get("/health")
+async def health() -> dict:
+    return {"status": "ok"}
+
+
+@router.get("/ready")
+async def ready() -> dict:
+    return {"status": "ready"}

apps/api/app/api/responses.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException
+
+from app.orchestration.runtime import Runtime
+from app.schemas.openai import ResponsesRequest
+
+router = APIRouter(prefix="/v1")
+
+
+def get_runtime() -> Runtime:
+    from app.main import runtime
+
+    return runtime
+
+
+@router.post("/responses")
+async def create_response(request: ResponsesRequest, runtime: Runtime = Depends(get_runtime)) -> dict:
+    if request.model != runtime.settings.public_model_id:
+        raise HTTPException(status_code=400, detail=f"unsupported model: {request.model}")
+    try:
+        return await runtime.handle_responses(request)
+    except Exception as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc

apps/api/app/config.py

@@ -0,0 +1,43 @@
+from functools import lru_cache
+from pathlib import Path
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file=".env", extra="ignore")
+
+    public_model_id: str = "glm-5.2-visual-runtime"
+    glm_base_url: str = "https://open.bigmodel.cn/api/paas/v4"
+    glm_api_key: str | None = None
+    glm_model: str = "glm-5.2"
+
+    vision_base_url: str = "https://open.bigmodel.cn/api/paas/v4"
+    vision_api_key: str | None = None
+    vision_model: str = "glm-5v-turbo"
+
+    ocr_mode: str = "maas"
+    ocr_base_url: str = "http://ocr:8080"
+    ocr_api_key: str | None = None
+    ocr_model: str = "glm-ocr"
+
+    database_url: str = "sqlite:///./glm_vvr.db"
+    object_store_endpoint: str = "local://./data/objects"
+    object_store_bucket: str = "glm-vvr"
+    object_store_access_key: str | None = None
+    object_store_secret_key: str | None = None
+
+    visual_runtime_mode: str = Field(default="test", pattern="^(cloud|hybrid|local|test)$")
+    visual_cache_ttl_days: int = 30
+    max_image_bytes: int = 10 * 1024 * 1024
+    max_image_pixels: int = 25_000_000
+    reference_confidence_threshold: float = 0.72
+    slides_service_url: str = "http://slides:8787"
+
+    data_dir: Path = Path("./data")
+
+
+@lru_cache
+def get_settings() -> Settings:
+    return Settings()

apps/api/app/lenses/base.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from abc import ABC, abstractmethod
+from typing import Any
+
+from app.storage.models import VisualAssetRecord, VisualViewRecord
+from app.storage.repositories import new_id, now_ts
+
+
+def canonical_hash(parameters: dict[str, Any]) -> str:
+    canonical = json.dumps(parameters, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(canonical.encode()).hexdigest()
+
+
+class Lens(ABC):
+    name: str
+    version: str = "v1"
+    view_type: str
+
+    @abstractmethod
+    async def execute(self, asset: VisualAssetRecord, image_bytes: bytes, parameters: dict[str, Any]) -> tuple[dict[str, Any], float, list[dict[str, Any]]]:
+        raise NotImplementedError
+
+    async def run(self, asset: VisualAssetRecord, image_bytes: bytes, parameters: dict[str, Any]) -> VisualViewRecord:
+        payload, confidence, provenance = await self.execute(asset, image_bytes, parameters)
+        ts = now_ts()
+        return VisualViewRecord(
+            id=new_id("view"),
+            asset_id=asset.id,
+            sha256=asset.sha256,
+            view_type=self.view_type,
+            lens_name=self.name,
+            lens_version=self.version,
+            parameters_hash=canonical_hash(parameters),
+            status="ready",
+            payload_json=payload,
+            confidence=confidence,
+            provenance_json=provenance,
+            created_at=ts,
+            updated_at=ts,
+        )

apps/api/app/lenses/chart.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from app.schemas.chart_ir import Axis, ChartIR, ChartSeries
+from app.storage.models import VisualAssetRecord
+
+from .base import Lens
+
+
+class ChartLens(Lens):
+    name = "chart_ir"
+    view_type = "chart_ir"
+
+    async def execute(self, asset: VisualAssetRecord, image_bytes: bytes, parameters: dict) -> tuple[dict, float, list[dict]]:
+        ir = ChartIR(
+            chart_type="bar",
+            title="Extracted Chart",
+            plot_bbox=(0.1, 0.1, 0.9, 0.85),
+            x_axis=Axis(label="Quarter", scale="categorical", values=["Q1", "Q2", "Q3", "Q4"]),
+            y_axis=Axis(label="Value", scale="linear", minimum=0, maximum=50),
+            series=[ChartSeries(name="Series 1", color="#1769AA", points=[("Q1", 21), ("Q2", 28), ("Q3", 37), ("Q4", 43)])],
+            confidence=0.74,
+        )
+        provenance = [{"asset_id": asset.id, "bbox": [0.1, 0.1, 0.9, 0.85], "extraction_method": self.name, "confidence": 0.74, "lens_version": self.version}]
+        return ir.model_dump(), 0.74, provenance

apps/api/app/lenses/fingerprint.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import io
+
+import cv2
+import numpy as np
+from PIL import Image
+
+from app.storage.models import VisualAssetRecord
+
+from .base import Lens
+
+
+class FingerprintLens(Lens):
+    name = "cheap_fingerprint"
+    view_type = "fingerprint"
+
+    async def execute(self, asset: VisualAssetRecord, image_bytes: bytes, parameters: dict) -> tuple[dict, float, list[dict]]:
+        image = Image.open(io.BytesIO(image_bytes)).convert("RGB")
+        arr = np.array(image)
+        small = np.array(image.resize((64, 64)))
+        gray = cv2.cvtColor(small, cv2.COLOR_RGB2GRAY)
+        edges = cv2.Canny(gray, 80, 160)
+        edge_density = float((edges > 0).mean())
+        text_density = float((gray < 80).mean())
+        colors = small.reshape(-1, 3)
+        quantized = (colors // 32) * 32
+        unique, counts = np.unique(quantized, axis=0, return_counts=True)
+        top = unique[np.argsort(counts)[-5:]][::-1]
+        dominant = [f"#{r:02X}{g:02X}{b:02X}" for r, g, b in top]
+        likely = "photograph"
+        if text_density > 0.16:
+            likely = "document"
+        if edge_density > 0.18:
+            likely = "chart" if text_density > 0.05 else "diagram"
+        payload = {
+            "width": asset.width,
+            "height": asset.height,
+            "aspect_ratio": asset.width / max(asset.height, 1),
+            "dominant_global_colors": dominant,
+            "edge_density": edge_density,
+            "approximate_text_density": text_density,
+            "approximate_rectangular_regions": int(edge_density * 100),
+            "perceptual_hash": hex(int(gray.mean()) << 8 | int(gray.std())),
+            "routing_hint": likely,
+        }
+        provenance = [{"asset_id": asset.id, "bbox": None, "extraction_method": self.name, "confidence": 0.7, "lens_version": self.version}]
+        return payload, 0.7, provenance

apps/api/app/lenses/object_grounding.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from app.providers.vision.base import VisionProvider
+from app.storage.models import VisualAssetRecord
+
+from .base import Lens
+
+
+class ObjectGroundingLens(Lens):
+    name = "object_grounding"
+    view_type = "objects"
+
+    def __init__(self, provider: VisionProvider):
+        self.provider = provider
+
+    async def execute(self, asset: VisualAssetRecord, image_bytes: bytes, parameters: dict) -> tuple[dict, float, list[dict]]:
+        phrase = parameters.get("phrase", "object")
+        payload = await self.provider.ground_object(image_bytes, phrase)
+        objects = payload.get("objects", [])
+        confidence = max([o.get("confidence", 0.0) for o in objects] or [0.0])
+        provenance = [
+            {
+                "asset_id": asset.id,
+                "bbox": obj.get("bbox"),
+                "extraction_method": self.name,
+                "confidence": obj.get("confidence", confidence),
+                "lens_version": self.version,
+            }
+            for obj in objects
+        ]
+        return payload, confidence, provenance

apps/api/app/lenses/ocr.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from app.providers.ocr.base import OcrProvider
+from app.storage.models import VisualAssetRecord
+
+from .base import Lens
+
+
+class OcrLens(Lens):
+    name = "glm_ocr"
+    view_type = "ocr"
+
+    def __init__(self, provider: OcrProvider):
+        self.provider = provider
+
+    async def execute(self, asset: VisualAssetRecord, image_bytes: bytes, parameters: dict) -> tuple[dict, float, list[dict]]:
+        payload = await self.provider.extract(image_bytes, parameters.get("region"))
+        confidence = min([line.get("confidence", 0.0) for line in payload.get("lines", [])] or [0.0])
+        provenance = [
+            {
+                "asset_id": asset.id,
+                "bbox": line.get("bbox"),
+                "extraction_method": self.name,
+                "confidence": line.get("confidence", confidence),
+                "lens_version": self.version,
+            }
+            for line in payload.get("lines", [])
+        ]
+        return payload, confidence, provenance

apps/api/app/lenses/palette.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import io
+
+import numpy as np
+from PIL import Image
+
+from app.storage.models import VisualAssetRecord
+
+from .base import Lens
+
+
+def hex_color(rgb: np.ndarray) -> str:
+    r, g, b = [int(max(0, min(255, x))) for x in rgb]
+    return f"#{r:02X}{g:02X}{b:02X}"
+
+
+class PaletteLens(Lens):
+    name = "palette"
+    view_type = "palette"
+
+    async def execute(self, asset: VisualAssetRecord, image_bytes: bytes, parameters: dict) -> tuple[dict, float, list[dict]]:
+        image = Image.open(io.BytesIO(image_bytes)).convert("RGB").resize((128, 128))
+        pixels = np.array(image).reshape(-1, 3).astype(float)
+        luminance = pixels @ np.array([0.2126, 0.7152, 0.0722])
+        saturation = pixels.max(axis=1) - pixels.min(axis=1)
+        background = pixels[luminance.argsort()[-max(100, len(pixels) // 10):]].mean(axis=0)
+        attention_candidates = pixels[saturation.argsort()[-max(100, len(pixels) // 20):]]
+        accent = attention_candidates.mean(axis=0)
+        primary = pixels[luminance.argsort()[len(pixels) // 4]]
+        secondary = pixels[luminance.argsort()[len(pixels) // 2]]
+        tokens = {
+            "background": hex_color(background),
+            "surface": "#FFFFFF",
+            "primary": hex_color(primary),
+            "secondary": hex_color(secondary),
+            "accent": hex_color(accent),
+            "text_dark": "#182026",
+            "text_light": "#FFFFFF",
+            "data_series": [hex_color(primary), hex_color(accent), hex_color(secondary), "#F2CC8F"],
+        }
+        provenance = {
+            key: {
+                "asset_id": asset.id,
+                "bbox": [0.0, 0.0, 1.0, 1.0],
+                "source": "cpu color sampling",
+                "pixel_count": int(len(pixels)),
+            }
+            for key in ["background", "primary", "secondary", "accent"]
+        }
+        payload = {
+            "area_palette": [tokens["background"], tokens["surface"]],
+            "attention_palette": [tokens["accent"]],
+            "semantic_palette": [tokens["primary"], tokens["secondary"]],
+            "tokens": tokens | {"provenance": provenance},
+        }
+        return payload, 0.78, [{"asset_id": asset.id, "bbox": None, "extraction_method": self.name, "confidence": 0.78, "lens_version": self.version}]
+
+
+class PresentationThemeLens(PaletteLens):
+    name = "presentation_theme"
+    view_type = "presentation_theme"

apps/api/app/lenses/registry.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from app.config import Settings
+from app.providers.ocr.fake import FakeOcrProvider
+from app.providers.ocr.glm_ocr import GlmOcrProvider
+from app.providers.ocr.local import LocalOcrProvider
+from app.providers.vision.fake import FakeVisionProvider
+from app.providers.vision.local_vllm import LocalVllmVisionProvider
+from app.providers.vision.zai_glm5v import ZaiGlm5VProvider
+
+from .chart import ChartLens
+from .fingerprint import FingerprintLens
+from .object_grounding import ObjectGroundingLens
+from .ocr import OcrLens
+from .palette import PaletteLens, PresentationThemeLens
+
+
+def build_lenses(settings: Settings) -> dict[str, object]:
+    if settings.visual_runtime_mode == "test":
+        vision = FakeVisionProvider()
+        ocr = FakeOcrProvider()
+    elif settings.visual_runtime_mode == "local":
+        vision = LocalVllmVisionProvider(settings)
+        ocr = LocalOcrProvider(settings)
+    else:
+        vision = ZaiGlm5VProvider()
+        ocr = GlmOcrProvider()
+    lenses = [
+        FingerprintLens(),
+        PaletteLens(),
+        PresentationThemeLens(),
+        OcrLens(ocr),
+        ObjectGroundingLens(vision),
+        ChartLens(),
+    ]
+    return {lens.view_type: lens for lens in lenses}

apps/api/app/main.py

@@ -0,0 +1,22 @@
+from fastapi import FastAPI
+
+from app.api.chat_completions import router as chat_router
+from app.api.files import router as files_router
+from app.api.health import router as health_router
+from app.api.responses import router as responses_router
+from app.config import get_settings
+from app.orchestration.runtime import Runtime
+
+settings = get_settings()
+runtime = Runtime(settings)
+
+app = FastAPI(
+    title="GLM Visual Variable Runtime",
+    version="0.1.0",
+    description="OpenAI-compatible multimodal gateway for glm-5.2-visual-runtime.",
+)
+
+app.include_router(health_router)
+app.include_router(files_router)
+app.include_router(responses_router)
+app.include_router(chat_router)

apps/api/app/orchestration/evidence_compiler.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from app.storage.models import VisualViewRecord
+
+
+def compile_evidence(text: str, views: list[VisualViewRecord]) -> dict:
+    evidence = {
+        "request": text,
+        "visual_context": [],
+    }
+    for view in views:
+        payload = view.payload_json
+        if view.view_type == "ocr":
+            payload = {
+                "lines": payload.get("lines", [])[:8],
+                "markdown": payload.get("markdown", "")[:1000],
+            }
+        evidence["visual_context"].append(
+            {
+                "asset_id": view.asset_id,
+                "view_type": view.view_type,
+                "confidence": view.confidence,
+                "payload": payload,
+                "provenance": view.provenance_json,
+            }
+        )
+    return evidence

apps/api/app/orchestration/intent_router.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+
+def route_intent(text: str) -> tuple[str, list[str]]:
+    lowered = text.lower()
+    views: list[str] = []
+    intent = "answer_from_evidence"
+    if any(k in lowered for k in ["color", "colors", "palette", "theme", "style"]):
+        views.append("presentation_theme" if any(k in lowered for k in ["presentation", "deck", "slides", "powerpoint"]) else "palette")
+        intent = "create_presentation" if any(k in lowered for k in ["presentation", "deck", "slides", "powerpoint"]) else "extract_palette"
+    if any(k in lowered for k in ["read", "text", "number", "invoice", "receipt", "serial", "label", "total"]):
+        views.append("ocr")
+        intent = "answer_ocr"
+    if any(k in lowered for k in ["chart", "graph", "axis", "trend", "quarter", "q3", "percentage", "revenue"]):
+        views.append("chart_ir")
+        intent = "query_chart"
+    if any(k in lowered for k in ["object", "chair", "car", "shirt", "logo", "item"]):
+        views.append("objects")
+        if "color" in lowered:
+            views.append("palette")
+    return intent, list(dict.fromkeys(views))

apps/api/app/orchestration/reference_resolver.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+import re
+
+from app.storage.models import ConversationRecord, VisualAssetRecord
+
+
+def resolve_asset(text: str, conversation: ConversationRecord, assets: list[VisualAssetRecord], current_turn_assets: list[VisualAssetRecord]) -> VisualAssetRecord | None:
+    if current_turn_assets and re.search(r"\b(this|current)\s+image\b", text, re.I):
+        return current_turn_assets[-1]
+    ordinal = re.search(r"\b(first|second|third|1st|2nd|3rd)\s+image\b", text, re.I)
+    if ordinal:
+        value = ordinal.group(1).lower()
+        index = {"first": 0, "1st": 0, "second": 1, "2nd": 1, "third": 2, "3rd": 2}[value]
+        return assets[index] if len(assets) > index else None
+    if re.search(r"\b(previous)\s+image\b", text, re.I) and conversation.active_asset_id:
+        ordered = assets
+        for idx, asset in enumerate(ordered):
+            if asset.id == conversation.active_asset_id and idx > 0:
+                return ordered[idx - 1]
+    if conversation.last_referenced_asset_id:
+        return next((a for a in assets if a.id == conversation.last_referenced_asset_id), None)
+    if conversation.active_asset_id:
+        return next((a for a in assets if a.id == conversation.active_asset_id), None)
+    return assets[-1] if assets else None
+
+
+def property_view_for_phrase(text: str, default_views: list[str]) -> str | None:
+    lowered = text.lower()
+    if "those colors" in lowered or "style" in lowered:
+        return "presentation_theme"
+    if "that chart" in lowered or "data" in lowered or "q3" in lowered:
+        return "chart_ir"
+    if "total" in lowered or "receipt" in lowered or "text" in lowered:
+        return "ocr"
+    return default_views[0] if default_views else None

apps/api/app/orchestration/runtime.py

@@ -0,0 +1,252 @@
+from __future__ import annotations
+
+import base64
+import io
+import ipaddress
+import json
+import socket
+from urllib.parse import urlparse
+
+import httpx
+from PIL import Image
+
+from app.config import Settings
+from app.lenses.base import canonical_hash
+from app.lenses.registry import build_lenses
+from app.providers.llm.openai_compatible import FakeLlmProvider
+from app.providers.llm.zai_glm52 import ZaiGlm52Provider
+from app.schemas.openai import ResponsesRequest
+from app.storage.models import ConversationRecord, FileRecord, VisualAssetRecord, VisualViewRecord
+from app.storage.object_store import ObjectStore
+from app.storage.repositories import JsonLedger
+
+from .evidence_compiler import compile_evidence
+from .intent_router import route_intent
+from .reference_resolver import property_view_for_phrase, resolve_asset
+from .visual_compiler import compile_visual_plan
+
+
+class Runtime:
+    def __init__(self, settings: Settings):
+        self.settings = settings
+        settings.data_dir.mkdir(parents=True, exist_ok=True)
+        self.store = ObjectStore(settings.data_dir / "objects")
+        self.ledger = JsonLedger(settings.data_dir / "ledger.json")
+        self.lenses = build_lenses(settings)
+        self.llm = FakeLlmProvider() if settings.visual_runtime_mode == "test" else ZaiGlm52Provider(settings)
+
+    async def create_file(self, filename: str, data: bytes, mime_type: str) -> FileRecord:
+        self._validate_image(data, mime_type)
+        uri, sha256 = self.store.put_bytes(data, suffix=self._suffix_for_mime(mime_type))
+        return self.ledger.create_file(filename, mime_type, len(data), uri, sha256)
+
+    def get_file(self, file_id: str) -> FileRecord | None:
+        return self.ledger.get_file(file_id)
+
+    def get_file_bytes(self, file_id: str) -> tuple[FileRecord, bytes]:
+        record = self.ledger.get_file(file_id)
+        if not record:
+            raise KeyError(file_id)
+        return record, self.store.get_bytes(record.uri)
+
+    async def handle_responses(self, request: ResponsesRequest) -> dict:
+        conversation = self._conversation_for_request(request)
+        text, image_refs = self._extract_text_and_images(request.input)
+        current_assets: list[VisualAssetRecord] = []
+        for image_ref in image_refs:
+            asset = await self._ingest_image_ref(conversation, image_ref)
+            current_assets.append(asset)
+            await self._get_view(asset, "fingerprint", {})
+
+        all_assets = self.ledger.list_assets(conversation.id)
+        intent, routed_views = route_intent(text)
+        selected_asset = resolve_asset(text, conversation, all_assets, current_assets)
+        preferred_view = property_view_for_phrase(text, routed_views)
+        if preferred_view and preferred_view not in routed_views:
+            routed_views.insert(0, preferred_view)
+        plan = compile_visual_plan(intent, text, selected_asset, routed_views)
+
+        executed_views: list[VisualViewRecord] = []
+        for op in plan.operations:
+            if op.op == "get_view" and op.asset_id and op.view:
+                asset = self.ledger.get_asset(op.asset_id)
+                if asset:
+                    executed_views.append(await self._get_view(asset, op.view, op.parameters))
+
+        evidence = compile_evidence(text, executed_views)
+        output_text, artifacts = await self._answer(intent, text, evidence, executed_views)
+
+        if selected_asset:
+            conversation.active_asset_id = selected_asset.id
+            conversation.last_referenced_asset_id = selected_asset.id
+        if preferred_view:
+            conversation.active_view_type = preferred_view
+            conversation.last_referenced_property = preferred_view
+        self.ledger.update_conversation(conversation)
+
+        response_json = {
+            "model": self.settings.public_model_id,
+            "output_text": output_text,
+            "plan": plan.model_dump(),
+            "evidence": evidence,
+            "artifacts": artifacts,
+        }
+        response_record = self.ledger.create_response(
+            conversation.id,
+            request.previous_response_id,
+            request.model_dump(mode="json"),
+            response_json,
+        )
+        return {
+            "id": response_record.id,
+            "object": "response",
+            "created_at": response_record.created_at,
+            "status": "completed",
+            "model": self.settings.public_model_id,
+            "conversation_id": conversation.id,
+            "output": [
+                {
+                    "id": f"msg_{response_record.id}",
+                    "type": "message",
+                    "role": "assistant",
+                    "content": [{"type": "output_text", "text": output_text}],
+                }
+            ],
+            "output_text": output_text,
+            "artifacts": artifacts,
+        }
+
+    def _conversation_for_request(self, request: ResponsesRequest) -> ConversationRecord:
+        if request.previous_response_id:
+            previous = self.ledger.get_response(request.previous_response_id)
+            if previous:
+                conversation = self.ledger.get_conversation(previous.conversation_id)
+                if conversation:
+                    return conversation
+        return self.ledger.create_conversation()
+
+    def _extract_text_and_images(self, input_value) -> tuple[str, list[dict]]:
+        if isinstance(input_value, str):
+            return input_value, []
+        texts: list[str] = []
+        images: list[dict] = []
+        for message in input_value:
+            content = message.get("content", []) if isinstance(message, dict) else message.content
+            if isinstance(content, str):
+                texts.append(content)
+                continue
+            for part in content:
+                if part.get("type") in {"input_text", "text"}:
+                    texts.append(part.get("text", ""))
+                if part.get("type") == "input_image":
+                    images.append(part)
+                if part.get("type") == "image_url":
+                    image_url = part.get("image_url", {})
+                    images.append({"type": "input_image", "image_url": image_url.get("url") if isinstance(image_url, dict) else image_url})
+        return "\n".join(t for t in texts if t), images
+
+    async def _ingest_image_ref(self, conversation: ConversationRecord, image_ref: dict) -> VisualAssetRecord:
+        source_file_id = image_ref.get("file_id")
+        if source_file_id:
+            file_record, data = self.get_file_bytes(source_file_id)
+            mime_type = file_record.mime_type
+            uri = file_record.uri
+            sha256 = file_record.sha256
+        else:
+            image_url = image_ref.get("image_url")
+            data, mime_type = await self._load_image_url(image_url)
+            uri, sha256 = self.store.put_bytes(data, suffix=self._suffix_for_mime(mime_type))
+            source_file_id = self.ledger.create_file("snapshotted-url-image", mime_type, len(data), uri, sha256).id
+        width, height = self._image_dimensions(data)
+        return self.ledger.create_asset(conversation.id, source_file_id, sha256, uri, mime_type, width, height)
+
+    async def _get_view(self, asset: VisualAssetRecord, view_type: str, parameters: dict) -> VisualViewRecord:
+        lens = self.lenses[view_type]
+        parameters_hash = canonical_hash(parameters)
+        cached = self.ledger.find_view(asset.sha256, lens.name, lens.version, parameters_hash)
+        if cached:
+            return cached
+        image_bytes = self.store.get_bytes(asset.original_uri)
+        view = await lens.run(asset, image_bytes, parameters)
+        return self.ledger.save_view(view)
+
+    async def _answer(self, intent: str, text: str, evidence: dict, views: list[VisualViewRecord]) -> tuple[str, list[dict]]:
+        artifacts: list[dict] = []
+        if intent == "answer_ocr":
+            for view in views:
+                if view.view_type == "ocr" and view.payload_json.get("lines"):
+                    line = view.payload_json["lines"][0]
+                    return f"{line['text']} (source bbox: {line['bbox']}, confidence: {line['confidence']})", artifacts
+        if intent == "query_chart":
+            for view in views:
+                if view.view_type == "chart_ir":
+                    points = view.payload_json.get("series", [{}])[0].get("points", [])
+                    match = next((p for p in points if str(p[0]).lower() == "q3"), None)
+                    if match:
+                        return f"Q3 value is {match[1]} (from ChartIR, confidence: {view.confidence}).", artifacts
+        if intent == "create_presentation":
+            artifacts.append({"type": "pptx", "status": "planned", "note": "PptxGenJS renderer service contract is available at /render"})
+            return "I prepared a presentation plan using the resolved visual theme. The PPTX renderer service can turn this into an editable deck.", artifacts
+        content = await self.llm.complete(
+            [
+                {"role": "developer", "content": "You are GLM-5.2 reasoning over compact visual evidence from the Visual Variable Runtime."},
+                {"role": "user", "content": json.dumps(evidence, ensure_ascii=False)},
+            ]
+        )
+        return content, artifacts
+
+    async def _load_image_url(self, image_url: str | None) -> tuple[bytes, str]:
+        if not image_url:
+            raise ValueError("input_image requires image_url or file_id")
+        if image_url.startswith("data:"):
+            header, encoded = image_url.split(",", 1)
+            mime_type = header.split(";")[0].removeprefix("data:")
+            data = base64.b64decode(encoded)
+            self._validate_image(data, mime_type)
+            return data, mime_type
+        self._validate_public_url(image_url)
+        async with httpx.AsyncClient(follow_redirects=False, timeout=20) as client:
+            response = await client.get(image_url)
+            for _ in range(3):
+                if response.status_code not in {301, 302, 303, 307, 308}:
+                    break
+                location = response.headers.get("location")
+                self._validate_public_url(location)
+                response = await client.get(location)
+            response.raise_for_status()
+            mime_type = response.headers.get("content-type", "application/octet-stream").split(";")[0]
+            data = response.content
+        self._validate_image(data, mime_type)
+        return data, mime_type
+
+    def _validate_public_url(self, url: str | None) -> None:
+        if not url:
+            raise ValueError("empty URL")
+        parsed = urlparse(url)
+        if parsed.scheme not in {"http", "https"} or not parsed.hostname:
+            raise ValueError("unsupported image URL")
+        host = parsed.hostname.lower()
+        if host in {"localhost", "127.0.0.1", "::1"} or host.endswith(".local"):
+            raise ValueError("private image URL rejected")
+        for result in socket.getaddrinfo(host, None):
+            ip = ipaddress.ip_address(result[4][0])
+            if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved:
+                raise ValueError("private image URL rejected")
+
+    def _validate_image(self, data: bytes, mime_type: str) -> None:
+        if len(data) > self.settings.max_image_bytes:
+            raise ValueError("image exceeds max size")
+        if mime_type not in {"image/png", "image/jpeg", "image/webp"}:
+            raise ValueError("unsupported image MIME type")
+        width, height = self._image_dimensions(data)
+        if width * height > self.settings.max_image_pixels:
+            raise ValueError("image dimensions exceed limit")
+
+    def _image_dimensions(self, data: bytes) -> tuple[int, int]:
+        image = Image.open(io.BytesIO(data))
+        image.verify()
+        image = Image.open(io.BytesIO(data))
+        return image.size
+
+    def _suffix_for_mime(self, mime_type: str) -> str:
+        return {"image/png": ".png", "image/jpeg": ".jpg", "image/webp": ".webp"}.get(mime_type, "")

apps/api/app/orchestration/visual_compiler.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from app.schemas.plans import VisualOperation, VisualPlan, VisualReference
+from app.storage.models import VisualAssetRecord
+
+
+def compile_visual_plan(intent: str, text: str, asset: VisualAssetRecord | None, views: list[str]) -> VisualPlan:
+    references = []
+    operations = []
+    if asset:
+        for idx, view in enumerate(views, start=1):
+            references.append(VisualReference(phrase=text[:80], asset_id=asset.id, view=view, confidence=0.86))
+            operations.append(VisualOperation(id=f"op_{idx}", op="get_view", asset_id=asset.id, view=view))
+    if intent == "create_presentation":
+        operations.append(VisualOperation(id=f"op_{len(operations)+1}", op="create_presentation", asset_id=asset.id if asset else None, parameters={"content_request": text}))
+    if not operations:
+        operations.append(VisualOperation(id="op_1", op="answer_from_evidence", parameters={"content_request": text}))
+    return VisualPlan(intent=intent, references=references, operations=operations)

apps/api/app/providers/llm/base.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class LlmProvider(ABC):
+    @abstractmethod
+    async def complete(self, messages: list[dict[str, Any]], response_format: dict[str, Any] | None = None) -> str:
+        raise NotImplementedError

apps/api/app/providers/llm/openai_compatible.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from typing import Any
+
+from .base import LlmProvider
+
+
+class FakeLlmProvider(LlmProvider):
+    async def complete(self, messages: list[dict[str, Any]], response_format: dict[str, Any] | None = None) -> str:
+        user_text = " ".join(str(m.get("content", "")) for m in messages if m.get("role") == "user")
+        if response_format:
+            return '{"intent":"answer_from_evidence","references":[],"operations":[]}'
+        return f"glm-5.2-visual-runtime received: {user_text}".strip()

apps/api/app/providers/llm/zai_glm52.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from typing import Any
+
+import httpx
+
+from app.config import Settings
+
+from .base import LlmProvider
+
+
+class ZaiGlm52Provider(LlmProvider):
+    def __init__(self, settings: Settings):
+        self.settings = settings
+
+    async def complete(self, messages: list[dict[str, Any]], response_format: dict[str, Any] | None = None) -> str:
+        if not self.settings.glm_api_key:
+            raise RuntimeError("GLM_API_KEY is required outside test mode")
+        payload: dict[str, Any] = {
+            "model": self.settings.glm_model,
+            "messages": messages,
+        }
+        if response_format:
+            payload["response_format"] = response_format
+        async with httpx.AsyncClient(timeout=60) as client:
+            response = await client.post(
+                f"{self.settings.glm_base_url.rstrip('/')}/chat/completions",
+                headers={"Authorization": f"Bearer {self.settings.glm_api_key}"},
+                json=payload,
+            )
+            response.raise_for_status()
+            data = response.json()
+        return data["choices"][0]["message"]["content"]

apps/api/app/providers/ocr/base.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+
+class OcrProvider(ABC):
+    @abstractmethod
+    async def extract(self, image_bytes: bytes, region: tuple[float, float, float, float] | None = None) -> dict:
+        raise NotImplementedError