File size: 10,255 Bytes
3750f59
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
#!/usr/bin/env python3
"""
Layout service (CPU only, 0 GB VRAM).

Pipeline for /OCR_parse:
  1. PP-DocLayoutV3 (ONNX Runtime, CPU) detects layout regions (polygons,
     classes, scores) and reading order on the input image / PDF page.
  2. Each region is cropped and sent to the local GLM-OCR vLLM instance
     (127.0.0.1:8001, served model name from SERVED_OCR) with a prompt adapted
     to the region class (text / table / formula / figure ...).
  3. Results are aggregated in reading order into a single Markdown document.

Endpoints (internal, behind Nginx at /OCR_parse/*):
  GET  /health   -> 200 once the ONNX model is loaded, 503 otherwise
  POST /parse    -> {"image": "data:<mime>;base64,...", "max_tokens": 4096,
                     "score_threshold": 0.5}
                  -> {"pages": n, "regions": [...], "markdown": "..."}

PaddlePaddle is intentionally NOT used: the official ONNX export of
PaddlePaddle/PP-DocLayoutV3 runs on CPU via onnxruntime (no conflict with torch/vLLM).
"""
import base64
import glob
import io
import os
import time
from contextlib import asynccontextmanager

import numpy as np
import httpx
import onnxruntime as ort
from fastapi import FastAPI, HTTPException
from PIL import Image
from pydantic import BaseModel
from huggingface_hub import snapshot_download

LAYOUT_MODEL = os.getenv("LAYOUT_MODEL", "PaddlePaddle/PP-DocLayoutV3_onnx")
LAYOUT_PORT = int(os.getenv("LAYOUT_PORT", "8003"))
PORT_OCR = int(os.getenv("PORT_OCR", "8001"))
SERVED_OCR = os.getenv("SERVED_OCR", "glm-ocr")

# Partial label map for PP-DocLayout (25 classes); unknown ids fall back to class_<id>
LABELS = {
    0: "text", 1: "title", 2: "figure", 3: "table", 4: "formula",
    5: "header", 6: "footer", 7: "page_number", 8: "caption", 9: "reference",
    10: "list", 11: "code", 12: "abstract", 13: "author", 14: "keywords",
    15: "seal", 16: "image", 17: "paragraph",
}

PROMPTS = {
    "table": "Convertis le tableau de cette image en HTML fidele (balises <table>), sans aucun commentaire.",
    "formula": "Transcris la ou les formules mathematiques de cette image en LaTeX pur, sans explication.",
    "figure": "Decris brievement cette figure en une phrase.",
    "image": "Decris brievement cette image en une phrase.",
    "seal": "Decris ce sceau/tampon : texte lisible et forme.",
    "title": "Extrais le titre de cette image, tel quel, sans commentaire.",
    "code": "Extrais le code de cette image tel quel, dans un bloc de code, sans commentaire.",
}
DEFAULT_PROMPT = "Extrais tout le texte de cette image tel quel, sans commentaire."

state = {"ready": False, "session": None, "input_name": None,
         "in_w": 640, "in_h": 640, "error": None}
HTTPX = httpx.Client(timeout=900.0)


def load_model():
    """Download (cached in HF_HOME) and load the PP-DocLayoutV3 ONNX model on CPU."""
    print(f"[layout] Downloading {LAYOUT_MODEL} (cached in {os.getenv('HF_HOME', '/data/hf')})...")
    t0 = time.time()
    snapshot_dir = snapshot_download(
        repo_id=LAYOUT_MODEL,
        allow_patterns=["*.onnx", "*.json", "*.txt", "*.yml", "*.yaml"],
    )
    onnx_files = glob.glob(os.path.join(snapshot_dir, "**", "*.onnx"), recursive=True)
    if not onnx_files:
        raise RuntimeError(f"No .onnx file found in {snapshot_dir}")
    # prefer model.onnx, else the largest (fp32) one
    onnx_files.sort(key=lambda p: (os.path.basename(p) != "model.onnx", -os.path.getsize(p)))
    model_path = onnx_files[0]
    print(f"[layout] Loading ONNX model: {model_path} ({os.path.getsize(model_path)/1e6:.1f} MB)")

    sess = ort.InferenceSession(model_path, providers=["CPUExecutionProvider"])
    inputs = sess.get_inputs()
    # Paddle-style export: 3 inputs -> image (NCHW), im_shape (N,2), scale_factor (N,2)
    image_input = next(i for i in inputs if len(i.shape) == 4)
    h = image_input.shape[2] if isinstance(image_input.shape[2], int) else 800
    w = image_input.shape[3] if isinstance(image_input.shape[3], int) else 800
    state["in_h"], state["in_w"] = int(h), int(w)
    state["input_names"] = {i.name: len(i.shape) for i in inputs}
    state["session"] = sess
    state["ready"] = True
    print(f"[layout] Model ready in {time.time()-t0:.1f}s (input {state['in_w']}x{state['in_h']}, CPU, inputs={list(state['input_names'])})")


@asynccontextmanager
async def lifespan(app: FastAPI):
    try:
        load_model()
    except Exception as e:
        state["error"] = str(e)
        print(f"[layout] FATAL: {e}")
    yield


app = FastAPI(title="PP-DocLayoutV3 layout service (CPU)", lifespan=lifespan)


class ParseRequest(BaseModel):
    image: str                      # data URL (image/* or application/pdf) or raw base64
    max_tokens: int = 4096
    score_threshold: float = 0.5


@app.get("/health")
def health():
    if state["ready"]:
        return {"status": "ok", "model": LAYOUT_MODEL, "device": "cpu"}
    raise HTTPException(status_code=503, detail=f"layout model not ready: {state['error']}")


def split_data_url(s: str):
    if s.startswith("data:"):
        head, _, b64 = s.partition(",")
        mime = head[5:].split(";")[0] or "application/octet-stream"
        return mime, b64
    return "application/octet-stream", s


def pdf_pages_to_images(raw: bytes):
    import pymupdf  # pymupdf (fitz)
    doc = pymupdf.open(stream=raw, filetype="pdf")
    pages = []
    for page in doc:
        pix = page.get_pixmap(dpi=200)
        pages.append(Image.frombytes("RGB", (pix.width, pix.height), pix.samples))
    doc.close()
    return pages


def detect_regions(img: Image.Image, score_threshold: float):
    """Run PP-DocLayoutV3 ONNX on one page image, return regions in reading order.

    Official PaddlePaddle/PP-DocLayoutV3_onnx export:
      inputs : image (N,3,H,W) float32 rescaled 1/255 (mean 0, std 1),
               im_shape (N,2) = [orig_h, orig_w], scale_factor (N,2) = [orig_h/H, orig_w/W]
      outputs: fetch_name_0 (N,7) rows = [class_id, score, x1, y1, x2, y2, order]
               (boxes in original image coordinates), plus a label count and a mask (ignored)
    """
    sess = state["session"]
    orig_w, orig_h = img.size
    in_w, in_h = state["in_w"], state["in_h"]

    resized = img.resize((in_w, in_h))
    arr = np.asarray(resized, dtype=np.float32) / 255.0
    image_tensor = arr.transpose(2, 0, 1)[None, ...]                       # NCHW
    im_shape = np.array([[orig_h, orig_w]], dtype=np.float32)
    scale_factor = np.array([[orig_h / in_h, orig_w / in_w]], dtype=np.float32)

    feed = {}
    for name, ndim in state["input_names"].items():
        lname = name.lower()
        if ndim == 4 or lname == "image":
            feed[name] = image_tensor
        elif "im_shape" in lname or ndim == 2 and "scale" not in lname and "shape" in lname:
            feed[name] = im_shape
        elif "scale" in lname or ndim == 2:
            feed[name] = scale_factor

    outputs = sess.run(None, feed)
    rows = None
    for out in outputs:
        a = np.asarray(out)
        if a.ndim == 2 and a.shape[-1] == 7:
            rows = a
            break
    if rows is None:
        raise RuntimeError(f"Unexpected ONNX outputs: {[np.asarray(o).shape for o in outputs]}")

    regions = []
    for row in rows:
        cls_id, score = int(row[0]), float(row[1])
        x1, y1, x2, y2 = [float(v) for v in row[2:6]]
        order = int(row[6])
        if score < score_threshold:
            continue
        x1, y1 = max(0.0, x1), max(0.0, y1)
        x2, y2 = min(orig_w, x2), min(orig_h, y2)
        if x2 - x1 < 8 or y2 - y1 < 8:
            continue
        regions.append({
            "label": LABELS.get(cls_id, f"class_{cls_id}"),
            "score": round(score, 4),
            "bbox": [round(x1, 1), round(y1, 1), round(x2, 1), round(y2, 1)],
            "polygon": None,
            "order": order,
        })

    regions.sort(key=lambda r: r["order"])
    for idx, r in enumerate(regions):
        r["order"] = idx
    return regions


def ocr_region(crop: Image.Image, label: str, max_tokens: int) -> str:
    buf = io.BytesIO()
    crop.save(buf, format="JPEG", quality=90)
    b64 = base64.b64encode(buf.getvalue()).decode()
    payload = {
        "model": SERVED_OCR,
        "messages": [{
            "role": "user",
            "content": [
                {"type": "text", "text": PROMPTS.get(label, DEFAULT_PROMPT)},
                {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}},
            ],
        }],
        "max_tokens": max_tokens,
        "temperature": 0.0,
    }
    r = HTTPX.post(f"http://127.0.0.1:{PORT_OCR}/v1/chat/completions", json=payload)
    r.raise_for_status()
    return r.json()["choices"][0]["message"]["content"]


@app.post("/parse")
def parse(req: ParseRequest):
    if not state["ready"]:
        raise HTTPException(status_code=503, detail=f"layout model not ready: {state['error']}")

    mime, b64 = split_data_url(req.image)
    try:
        raw = base64.b64decode(b64, validate=True)
    except Exception:
        raise HTTPException(status_code=400, detail="Invalid base64 image data")

    if mime == "application/pdf" or raw[:5] == b"%PDF-":
        pages = pdf_pages_to_images(raw)
    else:
        try:
            pages = [Image.open(io.BytesIO(raw)).convert("RGB")]
        except Exception:
            raise HTTPException(status_code=400, detail=f"Unsupported or unreadable image (mime={mime})")

    all_regions = []
    md_parts = []
    for pidx, page in enumerate(pages):
        regions = detect_regions(page, req.score_threshold)
        for r in regions:
            x1, y1, x2, y2 = [int(round(v)) for v in r["bbox"]]
            crop = page.crop((x1, y1, x2, y2))
            try:
                r["ocr"] = ocr_region(crop, r["label"], req.max_tokens)
            except Exception as e:
                r["ocr"] = f"[OCR error: {e}]"
            r["page"] = pidx + 1
            all_regions.append(r)
            md_parts.append(r["ocr"])
        print(f"[layout] page {pidx+1}/{len(pages)}: {len(regions)} regions")

    return {"pages": len(pages), "regions": all_regions, "markdown": "\n\n".join(md_parts)}


if __name__ == "__main__":
    import uvicorn
    uvicorn.run(app, host="127.0.0.1", port=LAYOUT_PORT, log_level="info")