Spaces:
Sleeping
Sleeping
Download ocr.py from jetlan/LightOnOCR: direct link, hf CLI and curl.
- Browser
- Download file 11.9 kB
-
https://huggingface.co/spaces/jetlan/LightOnOCR/resolve/main/ocr.py
- Command line
-
hf download hf://spaces/jetlan/LightOnOCR/ocr.py
-
curl -L -o ocr.py https://huggingface.co/spaces/jetlan/LightOnOCR/resolve/main/ocr.py
11.9 kB
| """The LightOnOCR-3 variants and how the demo runs them. | |
| Each variant has one of two backends: | |
| - **local**: the checkpoint is loaded with transformers at start-up and runs on the Space's GPU | |
| (ZeroGPU: the GPU is attached for the duration of one ``@spaces.GPU`` call). | |
| - **remote**: an OpenAI-compatible server (vLLM) given by ``LIGHTONOCR_ENDPOINT_<KEY>``. The | |
| Space then needs no GPU for that variant, and it is also how the demo is tested without one. | |
| Both stream the model output as it is generated. The request is the same everywhere: one user | |
| message with the page image, plus the single word ``grounding`` when layout is wanted, and | |
| thinking disabled (the 4B and 0.8B checkpoints are thinking models trained with it off). | |
| Environment: | |
| MODELS variants to offer, e.g. "4B,1B,0.8B" (default: all three) | |
| MODEL_4B / MODEL_1B / MODEL_0_8B Hugging Face repo of each variant (default: the lightonai/LightOnOCR-3-* repos) | |
| LIGHTONOCR_ENDPOINT_4B / _1B / _0_8B OpenAI-compatible base URL serving the variant | |
| LIGHTONOCR_API_KEY API key for the endpoints (default: "EMPTY") | |
| LIGHTONOCR_SERVED_<KEY> model name on the endpoint (default: the first model it lists) | |
| LAZY_MODELS=1 load local checkpoints on first use instead of at start-up | |
| HF_TOKEN needed while the checkpoints are private | |
| """ | |
| from __future__ import annotations | |
| import base64 | |
| import io | |
| import json | |
| import os | |
| import threading | |
| import time | |
| from dataclasses import dataclass, field | |
| from typing import Iterator | |
| import spaces | |
| import torch | |
| from PIL import Image | |
| from grounding import GROUNDING_PROMPT | |
| # ---- The variants ------------------------------------------------------------------------------- | |
| class Variant: | |
| key: str # short name used in the UI and in environment variables | |
| repo: str # Hugging Face repo | |
| summary: str # one line, from the collection's model table | |
| tokens_per_second: float # a conservative estimate for ZeroGPU's duration budget | |
| def name(self) -> str: | |
| return self.repo.split("/")[-1] | |
| def env_key(self) -> str: # "0.8B" -> "0_8B" | |
| return self.key.replace(".", "_") | |
| VARIANTS: dict[str, Variant] = { | |
| v.key: v | |
| for v in [ | |
| Variant("4B", os.environ.get("MODEL_4B", "lightonai/LightOnOCR-3-4B"), "Best OCR model, recommended for most tasks", 25), | |
| Variant("1B", os.environ.get("MODEL_1B", "lightonai/LightOnOCR-3-1B"), "LightOnOCR-2 architecture, drop-in upgrade for existing deployments", 50), | |
| Variant("0.8B", os.environ.get("MODEL_0_8B", "lightonai/LightOnOCR-3-0.8B"), "Smallest variant", 40), | |
| ] | |
| } | |
| ENABLED = [k.strip() for k in os.environ.get("MODELS", "4B,1B,0.8B").split(",") if k.strip() in VARIANTS] | |
| DEFAULT_VARIANT = ENABLED[0] if ENABLED else "4B" | |
| MODES = ("grounding", "plain") | |
| DEFAULT_MAX_TOKENS = 6144 | |
| MAX_TOKENS = 12288 # what the model card's evaluations allow; a dense page in grounding mode can need several thousand | |
| def chat_content(image_part: dict, mode: str) -> list[dict]: | |
| """The one user message of a request: the page image, plus the word "grounding" in grounding mode.""" | |
| return [image_part] + ([{"type": "text", "text": GROUNDING_PROMPT}] if mode == "grounding" else []) | |
| class Update: | |
| """One step of a streamed generation: the text so far, and the final statistics when done.""" | |
| text: str | |
| done: bool = False | |
| tokens: int | None = None | |
| error: str | None = None | |
| # ---- Remote backend: an OpenAI-compatible server (vLLM) ----------------------------------------- | |
| class Endpoint: | |
| base_url: str | |
| api_key: str = "EMPTY" | |
| model: str | None = None # served model name; discovered from the server when None | |
| _lock: threading.Lock = field(default_factory=threading.Lock, repr=False) | |
| def client(self): | |
| from openai import OpenAI | |
| return OpenAI(base_url=self.base_url, api_key=self.api_key, timeout=600.0) | |
| def served_model(self) -> str: | |
| with self._lock: | |
| if not self.model: | |
| models = self.client().models.list().data | |
| if not models: | |
| raise RuntimeError(f"{self.base_url} serves no model") | |
| self.model = models[0].id | |
| return self.model | |
| def stream(self, image: Image.Image, mode: str, temperature: float, max_tokens: int) -> Iterator[Update]: | |
| buf = io.BytesIO() | |
| image.save(buf, format="PNG") | |
| data_url = "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode() | |
| response = self.client().chat.completions.create( | |
| model=self.served_model(), | |
| messages=[{"role": "user", "content": chat_content({"type": "image_url", "image_url": {"url": data_url}}, mode)}], | |
| temperature=temperature, | |
| max_tokens=max_tokens, | |
| stream=True, | |
| stream_options={"include_usage": True}, | |
| extra_body={"chat_template_kwargs": {"enable_thinking": False}}, | |
| ) | |
| text, tokens = "", None | |
| for chunk in response: | |
| if chunk.usage: | |
| tokens = chunk.usage.completion_tokens | |
| if chunk.choices and chunk.choices[0].delta and chunk.choices[0].delta.content: | |
| text += chunk.choices[0].delta.content | |
| yield Update(text) | |
| yield Update(text, done=True, tokens=tokens) | |
| def endpoint_for(variant: Variant) -> Endpoint | None: | |
| base_url = os.environ.get(f"LIGHTONOCR_ENDPOINT_{variant.env_key}") | |
| if not base_url: | |
| return None | |
| return Endpoint(base_url.rstrip("/"), os.environ.get("LIGHTONOCR_API_KEY", "EMPTY"), os.environ.get(f"LIGHTONOCR_SERVED_{variant.env_key}") or None) | |
| ENDPOINTS: dict[str, Endpoint] = {key: ep for key in ENABLED if (ep := endpoint_for(VARIANTS[key]))} | |
| # ---- Local backend: transformers on the Space's GPU --------------------------------------------- | |
| # On ZeroGPU, CUDA is emulated outside @spaces.GPU calls. DEVICE=mps runs a small variant on an Apple GPU. | |
| DEVICE = os.environ.get("DEVICE") or ("cuda" if torch.cuda.is_available() else "cpu") | |
| DTYPE = torch.bfloat16 if DEVICE in ("cuda", "mps") else torch.float32 | |
| # The transformers classes for each architecture named in a checkpoint's config.json. The 4B and | |
| # 0.8B are Qwen3.5-based; the 1B keeps the LightOnOCR-2 architecture (Pixtral encoder, Qwen3 decoder), | |
| # whose config says model_type "mistral3", so Auto classes would pick the wrong model: name them. | |
| ARCHITECTURES = { | |
| "Qwen3_5ForConditionalGeneration": ("Qwen3_5ForConditionalGeneration", "AutoProcessor"), | |
| "LightOnOCRForConditionalGeneration": ("LightOnOcrForConditionalGeneration", "LightOnOcrProcessor"), | |
| } | |
| class LocalModel: | |
| variant: Variant | |
| model: object | |
| processor: object | |
| def load(cls, variant: Variant) -> "LocalModel": | |
| import transformers | |
| from huggingface_hub import hf_hub_download | |
| config = json.loads(open(hf_hub_download(variant.repo, "config.json"), encoding="utf-8").read()) | |
| model_cls, processor_cls = ARCHITECTURES.get(config["architectures"][0], ("AutoModelForImageTextToText", "AutoProcessor")) | |
| started = time.monotonic() | |
| print(f"loading {variant.repo} ({model_cls}, {DTYPE}, {DEVICE})…", flush=True) | |
| model = getattr(transformers, model_cls).from_pretrained(variant.repo, dtype=DTYPE).to(DEVICE).eval() | |
| processor = getattr(transformers, processor_cls).from_pretrained(variant.repo) | |
| print(f"loaded {variant.repo} in {time.monotonic() - started:.0f}s", flush=True) | |
| return cls(variant, model, processor) | |
| def stream(self, image: Image.Image, mode: str, temperature: float, max_tokens: int) -> Iterator[Update]: | |
| from transformers import TextIteratorStreamer | |
| messages = [{"role": "user", "content": chat_content({"type": "image", "image": image}, mode)}] | |
| # Thinking off, for the templates that know about it (the 4B and 0.8B); the 1B's template does not. | |
| template_kwargs = {"enable_thinking": False} if "enable_thinking" in str(getattr(self.processor, "chat_template", "")) else {} | |
| inputs = self.processor.apply_chat_template( | |
| messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", **template_kwargs | |
| ) | |
| inputs = inputs.to(self.model.device) | |
| if "pixel_values" in inputs: | |
| inputs["pixel_values"] = inputs["pixel_values"].to(self.model.dtype) | |
| prompt_length = inputs["input_ids"].shape[1] | |
| streamer = TextIteratorStreamer(self.processor.tokenizer, skip_prompt=True, skip_special_tokens=True, timeout=900.0) | |
| sampling = {"do_sample": True, "temperature": temperature, "top_p": 1.0, "top_k": 0} if temperature > 0 else {"do_sample": False} | |
| result: dict = {} | |
| def generate(): | |
| try: | |
| with torch.inference_mode(): | |
| result["ids"] = self.model.generate(**inputs, max_new_tokens=max_tokens, streamer=streamer, **sampling) | |
| except Exception as e: # surfaced to the caller below; the streamer must still end | |
| result["error"] = f"{type(e).__name__}: {e}" | |
| streamer.end() | |
| thread = threading.Thread(target=generate, daemon=True) | |
| thread.start() | |
| text = "" | |
| for piece in streamer: | |
| text += piece | |
| yield Update(text) | |
| thread.join() | |
| if "error" in result: | |
| yield Update(text, done=True, error=result["error"]) | |
| else: | |
| yield Update(text, done=True, tokens=int(result["ids"].shape[1] - prompt_length)) | |
| LOCAL: dict[str, LocalModel] = {} | |
| _load_lock = threading.Lock() | |
| def local_model(key: str) -> LocalModel: | |
| with _load_lock: | |
| if key not in LOCAL: | |
| LOCAL[key] = LocalModel.load(VARIANTS[key]) | |
| return LOCAL[key] | |
| if not os.environ.get("LAZY_MODELS"): | |
| # Load at import, on "cuda": the pattern ZeroGPU expects (CUDA transfers are optimised for | |
| # placements done at start-up). Variants served by an endpoint are not loaded. | |
| for _key in ENABLED: | |
| if _key not in ENDPOINTS: | |
| local_model(_key) | |
| def gpu_seconds(key: str, image, mode: str, temperature: float, max_tokens: int) -> int: | |
| """ZeroGPU budget for one page: prefill, plus the tokens at the variant's rate. Shorter is better for queue priority.""" | |
| return int(min(600, 20 + max_tokens / VARIANTS[key].tokens_per_second)) | |
| def stream_local(key: str, image: Image.Image, mode: str, temperature: float, max_tokens: int) -> Iterator[Update]: | |
| """Generate on the GPU. On ZeroGPU this runs in a worker process with the GPU attached; elsewhere it is a plain call.""" | |
| yield from local_model(key).stream(image, mode, temperature, max_tokens) | |
| # ---- One entry point ---------------------------------------------------------------------------- | |
| def backend_name(key: str) -> str: | |
| if key in ENDPOINTS: | |
| return "vLLM endpoint" | |
| return "ZeroGPU" if os.environ.get("SPACE_ID") else DEVICE | |
| def stream(key: str, image: Image.Image, mode: str, temperature: float = 0.2, max_tokens: int = DEFAULT_MAX_TOKENS) -> Iterator[Update]: | |
| """Run one variant on one page image and stream the output: the remote endpoint if configured, else the local model.""" | |
| if key not in VARIANTS: | |
| raise ValueError(f"unknown variant {key!r}") | |
| if mode not in MODES: | |
| raise ValueError(f"mode must be one of {MODES}, got {mode!r}") | |
| max_tokens = max(16, min(int(max_tokens), MAX_TOKENS)) | |
| if key in ENDPOINTS: | |
| yield from ENDPOINTS[key].stream(image, mode, temperature, max_tokens) | |
| else: | |
| yield from stream_local(key, image, mode, temperature, max_tokens) | |