File size: 11,867 Bytes
18115ce
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
"""The LightOnOCR-3 variants and how the demo runs them.

Each variant has one of two backends:

- **local**: the checkpoint is loaded with transformers at start-up and runs on the Space's GPU
  (ZeroGPU: the GPU is attached for the duration of one ``@spaces.GPU`` call).
- **remote**: an OpenAI-compatible server (vLLM) given by ``LIGHTONOCR_ENDPOINT_<KEY>``. The
  Space then needs no GPU for that variant, and it is also how the demo is tested without one.

Both stream the model output as it is generated. The request is the same everywhere: one user
message with the page image, plus the single word ``grounding`` when layout is wanted, and
thinking disabled (the 4B and 0.8B checkpoints are thinking models trained with it off).

Environment:
    MODELS                      variants to offer, e.g. "4B,1B,0.8B" (default: all three)
    MODEL_4B / MODEL_1B / MODEL_0_8B   Hugging Face repo of each variant (default: the lightonai/LightOnOCR-3-* repos)
    LIGHTONOCR_ENDPOINT_4B / _1B / _0_8B   OpenAI-compatible base URL serving the variant
    LIGHTONOCR_API_KEY          API key for the endpoints (default: "EMPTY")
    LIGHTONOCR_SERVED_<KEY>     model name on the endpoint (default: the first model it lists)
    LAZY_MODELS=1               load local checkpoints on first use instead of at start-up
    HF_TOKEN                    needed while the checkpoints are private
"""

from __future__ import annotations

import base64
import io
import json
import os
import threading
import time
from dataclasses import dataclass, field
from typing import Iterator

import spaces
import torch
from PIL import Image

from grounding import GROUNDING_PROMPT

# ---- The variants -------------------------------------------------------------------------------


@dataclass(frozen=True)
class Variant:
    key: str  # short name used in the UI and in environment variables
    repo: str  # Hugging Face repo
    summary: str  # one line, from the collection's model table
    tokens_per_second: float  # a conservative estimate for ZeroGPU's duration budget

    @property
    def name(self) -> str:
        return self.repo.split("/")[-1]

    @property
    def env_key(self) -> str:  # "0.8B" -> "0_8B"
        return self.key.replace(".", "_")


VARIANTS: dict[str, Variant] = {
    v.key: v
    for v in [
        Variant("4B", os.environ.get("MODEL_4B", "lightonai/LightOnOCR-3-4B"), "Best OCR model, recommended for most tasks", 25),
        Variant("1B", os.environ.get("MODEL_1B", "lightonai/LightOnOCR-3-1B"), "LightOnOCR-2 architecture, drop-in upgrade for existing deployments", 50),
        Variant("0.8B", os.environ.get("MODEL_0_8B", "lightonai/LightOnOCR-3-0.8B"), "Smallest variant", 40),
    ]
}
ENABLED = [k.strip() for k in os.environ.get("MODELS", "4B,1B,0.8B").split(",") if k.strip() in VARIANTS]
DEFAULT_VARIANT = ENABLED[0] if ENABLED else "4B"

MODES = ("grounding", "plain")
DEFAULT_MAX_TOKENS = 6144
MAX_TOKENS = 12288  # what the model card's evaluations allow; a dense page in grounding mode can need several thousand


def chat_content(image_part: dict, mode: str) -> list[dict]:
    """The one user message of a request: the page image, plus the word "grounding" in grounding mode."""
    return [image_part] + ([{"type": "text", "text": GROUNDING_PROMPT}] if mode == "grounding" else [])


@dataclass
class Update:
    """One step of a streamed generation: the text so far, and the final statistics when done."""

    text: str
    done: bool = False
    tokens: int | None = None
    error: str | None = None


# ---- Remote backend: an OpenAI-compatible server (vLLM) -----------------------------------------


@dataclass
class Endpoint:
    base_url: str
    api_key: str = "EMPTY"
    model: str | None = None  # served model name; discovered from the server when None
    _lock: threading.Lock = field(default_factory=threading.Lock, repr=False)

    def client(self):
        from openai import OpenAI

        return OpenAI(base_url=self.base_url, api_key=self.api_key, timeout=600.0)

    def served_model(self) -> str:
        with self._lock:
            if not self.model:
                models = self.client().models.list().data
                if not models:
                    raise RuntimeError(f"{self.base_url} serves no model")
                self.model = models[0].id
            return self.model

    def stream(self, image: Image.Image, mode: str, temperature: float, max_tokens: int) -> Iterator[Update]:
        buf = io.BytesIO()
        image.save(buf, format="PNG")
        data_url = "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode()
        response = self.client().chat.completions.create(
            model=self.served_model(),
            messages=[{"role": "user", "content": chat_content({"type": "image_url", "image_url": {"url": data_url}}, mode)}],
            temperature=temperature,
            max_tokens=max_tokens,
            stream=True,
            stream_options={"include_usage": True},
            extra_body={"chat_template_kwargs": {"enable_thinking": False}},
        )
        text, tokens = "", None
        for chunk in response:
            if chunk.usage:
                tokens = chunk.usage.completion_tokens
            if chunk.choices and chunk.choices[0].delta and chunk.choices[0].delta.content:
                text += chunk.choices[0].delta.content
                yield Update(text)
        yield Update(text, done=True, tokens=tokens)


def endpoint_for(variant: Variant) -> Endpoint | None:
    base_url = os.environ.get(f"LIGHTONOCR_ENDPOINT_{variant.env_key}")
    if not base_url:
        return None
    return Endpoint(base_url.rstrip("/"), os.environ.get("LIGHTONOCR_API_KEY", "EMPTY"), os.environ.get(f"LIGHTONOCR_SERVED_{variant.env_key}") or None)


ENDPOINTS: dict[str, Endpoint] = {key: ep for key in ENABLED if (ep := endpoint_for(VARIANTS[key]))}


# ---- Local backend: transformers on the Space's GPU ---------------------------------------------

# On ZeroGPU, CUDA is emulated outside @spaces.GPU calls. DEVICE=mps runs a small variant on an Apple GPU.
DEVICE = os.environ.get("DEVICE") or ("cuda" if torch.cuda.is_available() else "cpu")
DTYPE = torch.bfloat16 if DEVICE in ("cuda", "mps") else torch.float32

# The transformers classes for each architecture named in a checkpoint's config.json. The 4B and
# 0.8B are Qwen3.5-based; the 1B keeps the LightOnOCR-2 architecture (Pixtral encoder, Qwen3 decoder),
# whose config says model_type "mistral3", so Auto classes would pick the wrong model: name them.
ARCHITECTURES = {
    "Qwen3_5ForConditionalGeneration": ("Qwen3_5ForConditionalGeneration", "AutoProcessor"),
    "LightOnOCRForConditionalGeneration": ("LightOnOcrForConditionalGeneration", "LightOnOcrProcessor"),
}


@dataclass
class LocalModel:
    variant: Variant
    model: object
    processor: object

    @classmethod
    def load(cls, variant: Variant) -> "LocalModel":
        import transformers
        from huggingface_hub import hf_hub_download

        config = json.loads(open(hf_hub_download(variant.repo, "config.json"), encoding="utf-8").read())
        model_cls, processor_cls = ARCHITECTURES.get(config["architectures"][0], ("AutoModelForImageTextToText", "AutoProcessor"))
        started = time.monotonic()
        print(f"loading {variant.repo} ({model_cls}, {DTYPE}, {DEVICE})…", flush=True)
        model = getattr(transformers, model_cls).from_pretrained(variant.repo, dtype=DTYPE).to(DEVICE).eval()
        processor = getattr(transformers, processor_cls).from_pretrained(variant.repo)
        print(f"loaded {variant.repo} in {time.monotonic() - started:.0f}s", flush=True)
        return cls(variant, model, processor)

    def stream(self, image: Image.Image, mode: str, temperature: float, max_tokens: int) -> Iterator[Update]:
        from transformers import TextIteratorStreamer

        messages = [{"role": "user", "content": chat_content({"type": "image", "image": image}, mode)}]
        # Thinking off, for the templates that know about it (the 4B and 0.8B); the 1B's template does not.
        template_kwargs = {"enable_thinking": False} if "enable_thinking" in str(getattr(self.processor, "chat_template", "")) else {}
        inputs = self.processor.apply_chat_template(
            messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", **template_kwargs
        )
        inputs = inputs.to(self.model.device)
        if "pixel_values" in inputs:
            inputs["pixel_values"] = inputs["pixel_values"].to(self.model.dtype)
        prompt_length = inputs["input_ids"].shape[1]

        streamer = TextIteratorStreamer(self.processor.tokenizer, skip_prompt=True, skip_special_tokens=True, timeout=900.0)
        sampling = {"do_sample": True, "temperature": temperature, "top_p": 1.0, "top_k": 0} if temperature > 0 else {"do_sample": False}
        result: dict = {}

        def generate():
            try:
                with torch.inference_mode():
                    result["ids"] = self.model.generate(**inputs, max_new_tokens=max_tokens, streamer=streamer, **sampling)
            except Exception as e:  # surfaced to the caller below; the streamer must still end
                result["error"] = f"{type(e).__name__}: {e}"
                streamer.end()

        thread = threading.Thread(target=generate, daemon=True)
        thread.start()
        text = ""
        for piece in streamer:
            text += piece
            yield Update(text)
        thread.join()
        if "error" in result:
            yield Update(text, done=True, error=result["error"])
        else:
            yield Update(text, done=True, tokens=int(result["ids"].shape[1] - prompt_length))


LOCAL: dict[str, LocalModel] = {}
_load_lock = threading.Lock()


def local_model(key: str) -> LocalModel:
    with _load_lock:
        if key not in LOCAL:
            LOCAL[key] = LocalModel.load(VARIANTS[key])
        return LOCAL[key]


if not os.environ.get("LAZY_MODELS"):
    # Load at import, on "cuda": the pattern ZeroGPU expects (CUDA transfers are optimised for
    # placements done at start-up). Variants served by an endpoint are not loaded.
    for _key in ENABLED:
        if _key not in ENDPOINTS:
            local_model(_key)


def gpu_seconds(key: str, image, mode: str, temperature: float, max_tokens: int) -> int:
    """ZeroGPU budget for one page: prefill, plus the tokens at the variant's rate. Shorter is better for queue priority."""
    return int(min(600, 20 + max_tokens / VARIANTS[key].tokens_per_second))


@spaces.GPU(duration=gpu_seconds)
def stream_local(key: str, image: Image.Image, mode: str, temperature: float, max_tokens: int) -> Iterator[Update]:
    """Generate on the GPU. On ZeroGPU this runs in a worker process with the GPU attached; elsewhere it is a plain call."""
    yield from local_model(key).stream(image, mode, temperature, max_tokens)


# ---- One entry point ----------------------------------------------------------------------------


def backend_name(key: str) -> str:
    if key in ENDPOINTS:
        return "vLLM endpoint"
    return "ZeroGPU" if os.environ.get("SPACE_ID") else DEVICE


def stream(key: str, image: Image.Image, mode: str, temperature: float = 0.2, max_tokens: int = DEFAULT_MAX_TOKENS) -> Iterator[Update]:
    """Run one variant on one page image and stream the output: the remote endpoint if configured, else the local model."""
    if key not in VARIANTS:
        raise ValueError(f"unknown variant {key!r}")
    if mode not in MODES:
        raise ValueError(f"mode must be one of {MODES}, got {mode!r}")
    max_tokens = max(16, min(int(max_tokens), MAX_TOKENS))
    if key in ENDPOINTS:
        yield from ENDPOINTS[key].stream(image, mode, temperature, max_tokens)
    else:
        yield from stream_local(key, image, mode, temperature, max_tokens)