"""The LightOnOCR-3 variants and how the demo runs them. Each variant has one of two backends: - **local**: the checkpoint is loaded with transformers at start-up and runs on the Space's GPU (ZeroGPU: the GPU is attached for the duration of one ``@spaces.GPU`` call). - **remote**: an OpenAI-compatible server (vLLM) given by ``LIGHTONOCR_ENDPOINT_``. The Space then needs no GPU for that variant, and it is also how the demo is tested without one. Both stream the model output as it is generated. The request is the same everywhere: one user message with the page image, plus the single word ``grounding`` when layout is wanted, and thinking disabled (the 4B and 0.8B checkpoints are thinking models trained with it off). Environment: MODELS variants to offer, e.g. "4B,1B,0.8B" (default: all three) MODEL_4B / MODEL_1B / MODEL_0_8B Hugging Face repo of each variant (default: the lightonai/LightOnOCR-3-* repos) LIGHTONOCR_ENDPOINT_4B / _1B / _0_8B OpenAI-compatible base URL serving the variant LIGHTONOCR_API_KEY API key for the endpoints (default: "EMPTY") LIGHTONOCR_SERVED_ model name on the endpoint (default: the first model it lists) LAZY_MODELS=1 load local checkpoints on first use instead of at start-up HF_TOKEN needed while the checkpoints are private """ from __future__ import annotations import base64 import io import json import os import threading import time from dataclasses import dataclass, field from typing import Iterator import spaces import torch from PIL import Image from grounding import GROUNDING_PROMPT # ---- The variants ------------------------------------------------------------------------------- @dataclass(frozen=True) class Variant: key: str # short name used in the UI and in environment variables repo: str # Hugging Face repo summary: str # one line, from the collection's model table tokens_per_second: float # a conservative estimate for ZeroGPU's duration budget @property def name(self) -> str: return self.repo.split("/")[-1] @property def env_key(self) -> str: # "0.8B" -> "0_8B" return self.key.replace(".", "_") VARIANTS: dict[str, Variant] = { v.key: v for v in [ Variant("4B", os.environ.get("MODEL_4B", "lightonai/LightOnOCR-3-4B"), "Best OCR model, recommended for most tasks", 25), Variant("1B", os.environ.get("MODEL_1B", "lightonai/LightOnOCR-3-1B"), "LightOnOCR-2 architecture, drop-in upgrade for existing deployments", 50), Variant("0.8B", os.environ.get("MODEL_0_8B", "lightonai/LightOnOCR-3-0.8B"), "Smallest variant", 40), ] } ENABLED = [k.strip() for k in os.environ.get("MODELS", "4B,1B,0.8B").split(",") if k.strip() in VARIANTS] DEFAULT_VARIANT = ENABLED[0] if ENABLED else "4B" MODES = ("grounding", "plain") DEFAULT_MAX_TOKENS = 6144 MAX_TOKENS = 12288 # what the model card's evaluations allow; a dense page in grounding mode can need several thousand def chat_content(image_part: dict, mode: str) -> list[dict]: """The one user message of a request: the page image, plus the word "grounding" in grounding mode.""" return [image_part] + ([{"type": "text", "text": GROUNDING_PROMPT}] if mode == "grounding" else []) @dataclass class Update: """One step of a streamed generation: the text so far, and the final statistics when done.""" text: str done: bool = False tokens: int | None = None error: str | None = None # ---- Remote backend: an OpenAI-compatible server (vLLM) ----------------------------------------- @dataclass class Endpoint: base_url: str api_key: str = "EMPTY" model: str | None = None # served model name; discovered from the server when None _lock: threading.Lock = field(default_factory=threading.Lock, repr=False) def client(self): from openai import OpenAI return OpenAI(base_url=self.base_url, api_key=self.api_key, timeout=600.0) def served_model(self) -> str: with self._lock: if not self.model: models = self.client().models.list().data if not models: raise RuntimeError(f"{self.base_url} serves no model") self.model = models[0].id return self.model def stream(self, image: Image.Image, mode: str, temperature: float, max_tokens: int) -> Iterator[Update]: buf = io.BytesIO() image.save(buf, format="PNG") data_url = "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode() response = self.client().chat.completions.create( model=self.served_model(), messages=[{"role": "user", "content": chat_content({"type": "image_url", "image_url": {"url": data_url}}, mode)}], temperature=temperature, max_tokens=max_tokens, stream=True, stream_options={"include_usage": True}, extra_body={"chat_template_kwargs": {"enable_thinking": False}}, ) text, tokens = "", None for chunk in response: if chunk.usage: tokens = chunk.usage.completion_tokens if chunk.choices and chunk.choices[0].delta and chunk.choices[0].delta.content: text += chunk.choices[0].delta.content yield Update(text) yield Update(text, done=True, tokens=tokens) def endpoint_for(variant: Variant) -> Endpoint | None: base_url = os.environ.get(f"LIGHTONOCR_ENDPOINT_{variant.env_key}") if not base_url: return None return Endpoint(base_url.rstrip("/"), os.environ.get("LIGHTONOCR_API_KEY", "EMPTY"), os.environ.get(f"LIGHTONOCR_SERVED_{variant.env_key}") or None) ENDPOINTS: dict[str, Endpoint] = {key: ep for key in ENABLED if (ep := endpoint_for(VARIANTS[key]))} # ---- Local backend: transformers on the Space's GPU --------------------------------------------- # On ZeroGPU, CUDA is emulated outside @spaces.GPU calls. DEVICE=mps runs a small variant on an Apple GPU. DEVICE = os.environ.get("DEVICE") or ("cuda" if torch.cuda.is_available() else "cpu") DTYPE = torch.bfloat16 if DEVICE in ("cuda", "mps") else torch.float32 # The transformers classes for each architecture named in a checkpoint's config.json. The 4B and # 0.8B are Qwen3.5-based; the 1B keeps the LightOnOCR-2 architecture (Pixtral encoder, Qwen3 decoder), # whose config says model_type "mistral3", so Auto classes would pick the wrong model: name them. ARCHITECTURES = { "Qwen3_5ForConditionalGeneration": ("Qwen3_5ForConditionalGeneration", "AutoProcessor"), "LightOnOCRForConditionalGeneration": ("LightOnOcrForConditionalGeneration", "LightOnOcrProcessor"), } @dataclass class LocalModel: variant: Variant model: object processor: object @classmethod def load(cls, variant: Variant) -> "LocalModel": import transformers from huggingface_hub import hf_hub_download config = json.loads(open(hf_hub_download(variant.repo, "config.json"), encoding="utf-8").read()) model_cls, processor_cls = ARCHITECTURES.get(config["architectures"][0], ("AutoModelForImageTextToText", "AutoProcessor")) started = time.monotonic() print(f"loading {variant.repo} ({model_cls}, {DTYPE}, {DEVICE})…", flush=True) model = getattr(transformers, model_cls).from_pretrained(variant.repo, dtype=DTYPE).to(DEVICE).eval() processor = getattr(transformers, processor_cls).from_pretrained(variant.repo) print(f"loaded {variant.repo} in {time.monotonic() - started:.0f}s", flush=True) return cls(variant, model, processor) def stream(self, image: Image.Image, mode: str, temperature: float, max_tokens: int) -> Iterator[Update]: from transformers import TextIteratorStreamer messages = [{"role": "user", "content": chat_content({"type": "image", "image": image}, mode)}] # Thinking off, for the templates that know about it (the 4B and 0.8B); the 1B's template does not. template_kwargs = {"enable_thinking": False} if "enable_thinking" in str(getattr(self.processor, "chat_template", "")) else {} inputs = self.processor.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", **template_kwargs ) inputs = inputs.to(self.model.device) if "pixel_values" in inputs: inputs["pixel_values"] = inputs["pixel_values"].to(self.model.dtype) prompt_length = inputs["input_ids"].shape[1] streamer = TextIteratorStreamer(self.processor.tokenizer, skip_prompt=True, skip_special_tokens=True, timeout=900.0) sampling = {"do_sample": True, "temperature": temperature, "top_p": 1.0, "top_k": 0} if temperature > 0 else {"do_sample": False} result: dict = {} def generate(): try: with torch.inference_mode(): result["ids"] = self.model.generate(**inputs, max_new_tokens=max_tokens, streamer=streamer, **sampling) except Exception as e: # surfaced to the caller below; the streamer must still end result["error"] = f"{type(e).__name__}: {e}" streamer.end() thread = threading.Thread(target=generate, daemon=True) thread.start() text = "" for piece in streamer: text += piece yield Update(text) thread.join() if "error" in result: yield Update(text, done=True, error=result["error"]) else: yield Update(text, done=True, tokens=int(result["ids"].shape[1] - prompt_length)) LOCAL: dict[str, LocalModel] = {} _load_lock = threading.Lock() def local_model(key: str) -> LocalModel: with _load_lock: if key not in LOCAL: LOCAL[key] = LocalModel.load(VARIANTS[key]) return LOCAL[key] if not os.environ.get("LAZY_MODELS"): # Load at import, on "cuda": the pattern ZeroGPU expects (CUDA transfers are optimised for # placements done at start-up). Variants served by an endpoint are not loaded. for _key in ENABLED: if _key not in ENDPOINTS: local_model(_key) def gpu_seconds(key: str, image, mode: str, temperature: float, max_tokens: int) -> int: """ZeroGPU budget for one page: prefill, plus the tokens at the variant's rate. Shorter is better for queue priority.""" return int(min(600, 20 + max_tokens / VARIANTS[key].tokens_per_second)) @spaces.GPU(duration=gpu_seconds) def stream_local(key: str, image: Image.Image, mode: str, temperature: float, max_tokens: int) -> Iterator[Update]: """Generate on the GPU. On ZeroGPU this runs in a worker process with the GPU attached; elsewhere it is a plain call.""" yield from local_model(key).stream(image, mode, temperature, max_tokens) # ---- One entry point ---------------------------------------------------------------------------- def backend_name(key: str) -> str: if key in ENDPOINTS: return "vLLM endpoint" return "ZeroGPU" if os.environ.get("SPACE_ID") else DEVICE def stream(key: str, image: Image.Image, mode: str, temperature: float = 0.2, max_tokens: int = DEFAULT_MAX_TOKENS) -> Iterator[Update]: """Run one variant on one page image and stream the output: the remote endpoint if configured, else the local model.""" if key not in VARIANTS: raise ValueError(f"unknown variant {key!r}") if mode not in MODES: raise ValueError(f"mode must be one of {MODES}, got {mode!r}") max_tokens = max(16, min(int(max_tokens), MAX_TOKENS)) if key in ENDPOINTS: yield from ENDPOINTS[key].stream(image, mode, temperature, max_tokens) else: yield from stream_local(key, image, mode, temperature, max_tokens)