LightOnOCR / ocr.py
Bapt120's picture
LightOnOCR-3 demo
18115ce verified
Raw History Blame Contribute Delete
11.9 kB
"""The LightOnOCR-3 variants and how the demo runs them.
Each variant has one of two backends:
- **local**: the checkpoint is loaded with transformers at start-up and runs on the Space's GPU
(ZeroGPU: the GPU is attached for the duration of one ``@spaces.GPU`` call).
- **remote**: an OpenAI-compatible server (vLLM) given by ``LIGHTONOCR_ENDPOINT_<KEY>``. The
Space then needs no GPU for that variant, and it is also how the demo is tested without one.
Both stream the model output as it is generated. The request is the same everywhere: one user
message with the page image, plus the single word ``grounding`` when layout is wanted, and
thinking disabled (the 4B and 0.8B checkpoints are thinking models trained with it off).
Environment:
MODELS variants to offer, e.g. "4B,1B,0.8B" (default: all three)
MODEL_4B / MODEL_1B / MODEL_0_8B Hugging Face repo of each variant (default: the lightonai/LightOnOCR-3-* repos)
LIGHTONOCR_ENDPOINT_4B / _1B / _0_8B OpenAI-compatible base URL serving the variant
LIGHTONOCR_API_KEY API key for the endpoints (default: "EMPTY")
LIGHTONOCR_SERVED_<KEY> model name on the endpoint (default: the first model it lists)
LAZY_MODELS=1 load local checkpoints on first use instead of at start-up
HF_TOKEN needed while the checkpoints are private
"""
from __future__ import annotations
import base64
import io
import json
import os
import threading
import time
from dataclasses import dataclass, field
from typing import Iterator
import spaces
import torch
from PIL import Image
from grounding import GROUNDING_PROMPT
# ---- The variants -------------------------------------------------------------------------------
@dataclass(frozen=True)
class Variant:
key: str # short name used in the UI and in environment variables
repo: str # Hugging Face repo
summary: str # one line, from the collection's model table
tokens_per_second: float # a conservative estimate for ZeroGPU's duration budget
@property
def name(self) -> str:
return self.repo.split("/")[-1]
@property
def env_key(self) -> str: # "0.8B" -> "0_8B"
return self.key.replace(".", "_")
VARIANTS: dict[str, Variant] = {
v.key: v
for v in [
Variant("4B", os.environ.get("MODEL_4B", "lightonai/LightOnOCR-3-4B"), "Best OCR model, recommended for most tasks", 25),
Variant("1B", os.environ.get("MODEL_1B", "lightonai/LightOnOCR-3-1B"), "LightOnOCR-2 architecture, drop-in upgrade for existing deployments", 50),
Variant("0.8B", os.environ.get("MODEL_0_8B", "lightonai/LightOnOCR-3-0.8B"), "Smallest variant", 40),
]
}
ENABLED = [k.strip() for k in os.environ.get("MODELS", "4B,1B,0.8B").split(",") if k.strip() in VARIANTS]
DEFAULT_VARIANT = ENABLED[0] if ENABLED else "4B"
MODES = ("grounding", "plain")
DEFAULT_MAX_TOKENS = 6144
MAX_TOKENS = 12288 # what the model card's evaluations allow; a dense page in grounding mode can need several thousand
def chat_content(image_part: dict, mode: str) -> list[dict]:
"""The one user message of a request: the page image, plus the word "grounding" in grounding mode."""
return [image_part] + ([{"type": "text", "text": GROUNDING_PROMPT}] if mode == "grounding" else [])
@dataclass
class Update:
"""One step of a streamed generation: the text so far, and the final statistics when done."""
text: str
done: bool = False
tokens: int | None = None
error: str | None = None
# ---- Remote backend: an OpenAI-compatible server (vLLM) -----------------------------------------
@dataclass
class Endpoint:
base_url: str
api_key: str = "EMPTY"
model: str | None = None # served model name; discovered from the server when None
_lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
def client(self):
from openai import OpenAI
return OpenAI(base_url=self.base_url, api_key=self.api_key, timeout=600.0)
def served_model(self) -> str:
with self._lock:
if not self.model:
models = self.client().models.list().data
if not models:
raise RuntimeError(f"{self.base_url} serves no model")
self.model = models[0].id
return self.model
def stream(self, image: Image.Image, mode: str, temperature: float, max_tokens: int) -> Iterator[Update]:
buf = io.BytesIO()
image.save(buf, format="PNG")
data_url = "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode()
response = self.client().chat.completions.create(
model=self.served_model(),
messages=[{"role": "user", "content": chat_content({"type": "image_url", "image_url": {"url": data_url}}, mode)}],
temperature=temperature,
max_tokens=max_tokens,
stream=True,
stream_options={"include_usage": True},
extra_body={"chat_template_kwargs": {"enable_thinking": False}},
)
text, tokens = "", None
for chunk in response:
if chunk.usage:
tokens = chunk.usage.completion_tokens
if chunk.choices and chunk.choices[0].delta and chunk.choices[0].delta.content:
text += chunk.choices[0].delta.content
yield Update(text)
yield Update(text, done=True, tokens=tokens)
def endpoint_for(variant: Variant) -> Endpoint | None:
base_url = os.environ.get(f"LIGHTONOCR_ENDPOINT_{variant.env_key}")
if not base_url:
return None
return Endpoint(base_url.rstrip("/"), os.environ.get("LIGHTONOCR_API_KEY", "EMPTY"), os.environ.get(f"LIGHTONOCR_SERVED_{variant.env_key}") or None)
ENDPOINTS: dict[str, Endpoint] = {key: ep for key in ENABLED if (ep := endpoint_for(VARIANTS[key]))}
# ---- Local backend: transformers on the Space's GPU ---------------------------------------------
# On ZeroGPU, CUDA is emulated outside @spaces.GPU calls. DEVICE=mps runs a small variant on an Apple GPU.
DEVICE = os.environ.get("DEVICE") or ("cuda" if torch.cuda.is_available() else "cpu")
DTYPE = torch.bfloat16 if DEVICE in ("cuda", "mps") else torch.float32
# The transformers classes for each architecture named in a checkpoint's config.json. The 4B and
# 0.8B are Qwen3.5-based; the 1B keeps the LightOnOCR-2 architecture (Pixtral encoder, Qwen3 decoder),
# whose config says model_type "mistral3", so Auto classes would pick the wrong model: name them.
ARCHITECTURES = {
"Qwen3_5ForConditionalGeneration": ("Qwen3_5ForConditionalGeneration", "AutoProcessor"),
"LightOnOCRForConditionalGeneration": ("LightOnOcrForConditionalGeneration", "LightOnOcrProcessor"),
}
@dataclass
class LocalModel:
variant: Variant
model: object
processor: object
@classmethod
def load(cls, variant: Variant) -> "LocalModel":
import transformers
from huggingface_hub import hf_hub_download
config = json.loads(open(hf_hub_download(variant.repo, "config.json"), encoding="utf-8").read())
model_cls, processor_cls = ARCHITECTURES.get(config["architectures"][0], ("AutoModelForImageTextToText", "AutoProcessor"))
started = time.monotonic()
print(f"loading {variant.repo} ({model_cls}, {DTYPE}, {DEVICE})…", flush=True)
model = getattr(transformers, model_cls).from_pretrained(variant.repo, dtype=DTYPE).to(DEVICE).eval()
processor = getattr(transformers, processor_cls).from_pretrained(variant.repo)
print(f"loaded {variant.repo} in {time.monotonic() - started:.0f}s", flush=True)
return cls(variant, model, processor)
def stream(self, image: Image.Image, mode: str, temperature: float, max_tokens: int) -> Iterator[Update]:
from transformers import TextIteratorStreamer
messages = [{"role": "user", "content": chat_content({"type": "image", "image": image}, mode)}]
# Thinking off, for the templates that know about it (the 4B and 0.8B); the 1B's template does not.
template_kwargs = {"enable_thinking": False} if "enable_thinking" in str(getattr(self.processor, "chat_template", "")) else {}
inputs = self.processor.apply_chat_template(
messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", **template_kwargs
)
inputs = inputs.to(self.model.device)
if "pixel_values" in inputs:
inputs["pixel_values"] = inputs["pixel_values"].to(self.model.dtype)
prompt_length = inputs["input_ids"].shape[1]
streamer = TextIteratorStreamer(self.processor.tokenizer, skip_prompt=True, skip_special_tokens=True, timeout=900.0)
sampling = {"do_sample": True, "temperature": temperature, "top_p": 1.0, "top_k": 0} if temperature > 0 else {"do_sample": False}
result: dict = {}
def generate():
try:
with torch.inference_mode():
result["ids"] = self.model.generate(**inputs, max_new_tokens=max_tokens, streamer=streamer, **sampling)
except Exception as e: # surfaced to the caller below; the streamer must still end
result["error"] = f"{type(e).__name__}: {e}"
streamer.end()
thread = threading.Thread(target=generate, daemon=True)
thread.start()
text = ""
for piece in streamer:
text += piece
yield Update(text)
thread.join()
if "error" in result:
yield Update(text, done=True, error=result["error"])
else:
yield Update(text, done=True, tokens=int(result["ids"].shape[1] - prompt_length))
LOCAL: dict[str, LocalModel] = {}
_load_lock = threading.Lock()
def local_model(key: str) -> LocalModel:
with _load_lock:
if key not in LOCAL:
LOCAL[key] = LocalModel.load(VARIANTS[key])
return LOCAL[key]
if not os.environ.get("LAZY_MODELS"):
# Load at import, on "cuda": the pattern ZeroGPU expects (CUDA transfers are optimised for
# placements done at start-up). Variants served by an endpoint are not loaded.
for _key in ENABLED:
if _key not in ENDPOINTS:
local_model(_key)
def gpu_seconds(key: str, image, mode: str, temperature: float, max_tokens: int) -> int:
"""ZeroGPU budget for one page: prefill, plus the tokens at the variant's rate. Shorter is better for queue priority."""
return int(min(600, 20 + max_tokens / VARIANTS[key].tokens_per_second))
@spaces.GPU(duration=gpu_seconds)
def stream_local(key: str, image: Image.Image, mode: str, temperature: float, max_tokens: int) -> Iterator[Update]:
"""Generate on the GPU. On ZeroGPU this runs in a worker process with the GPU attached; elsewhere it is a plain call."""
yield from local_model(key).stream(image, mode, temperature, max_tokens)
# ---- One entry point ----------------------------------------------------------------------------
def backend_name(key: str) -> str:
if key in ENDPOINTS:
return "vLLM endpoint"
return "ZeroGPU" if os.environ.get("SPACE_ID") else DEVICE
def stream(key: str, image: Image.Image, mode: str, temperature: float = 0.2, max_tokens: int = DEFAULT_MAX_TOKENS) -> Iterator[Update]:
"""Run one variant on one page image and stream the output: the remote endpoint if configured, else the local model."""
if key not in VARIANTS:
raise ValueError(f"unknown variant {key!r}")
if mode not in MODES:
raise ValueError(f"mode must be one of {MODES}, got {mode!r}")
max_tokens = max(16, min(int(max_tokens), MAX_TOKENS))
if key in ENDPOINTS:
yield from ENDPOINTS[key].stream(image, mode, temperature, max_tokens)
else:
yield from stream_local(key, image, mode, temperature, max_tokens)