Spaces:
Paused
Paused
File size: 10,255 Bytes
3750f59 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 | #!/usr/bin/env python3
"""
Layout service (CPU only, 0 GB VRAM).
Pipeline for /OCR_parse:
1. PP-DocLayoutV3 (ONNX Runtime, CPU) detects layout regions (polygons,
classes, scores) and reading order on the input image / PDF page.
2. Each region is cropped and sent to the local GLM-OCR vLLM instance
(127.0.0.1:8001, served model name from SERVED_OCR) with a prompt adapted
to the region class (text / table / formula / figure ...).
3. Results are aggregated in reading order into a single Markdown document.
Endpoints (internal, behind Nginx at /OCR_parse/*):
GET /health -> 200 once the ONNX model is loaded, 503 otherwise
POST /parse -> {"image": "data:<mime>;base64,...", "max_tokens": 4096,
"score_threshold": 0.5}
-> {"pages": n, "regions": [...], "markdown": "..."}
PaddlePaddle is intentionally NOT used: the official ONNX export of
PaddlePaddle/PP-DocLayoutV3 runs on CPU via onnxruntime (no conflict with torch/vLLM).
"""
import base64
import glob
import io
import os
import time
from contextlib import asynccontextmanager
import numpy as np
import httpx
import onnxruntime as ort
from fastapi import FastAPI, HTTPException
from PIL import Image
from pydantic import BaseModel
from huggingface_hub import snapshot_download
LAYOUT_MODEL = os.getenv("LAYOUT_MODEL", "PaddlePaddle/PP-DocLayoutV3_onnx")
LAYOUT_PORT = int(os.getenv("LAYOUT_PORT", "8003"))
PORT_OCR = int(os.getenv("PORT_OCR", "8001"))
SERVED_OCR = os.getenv("SERVED_OCR", "glm-ocr")
# Partial label map for PP-DocLayout (25 classes); unknown ids fall back to class_<id>
LABELS = {
0: "text", 1: "title", 2: "figure", 3: "table", 4: "formula",
5: "header", 6: "footer", 7: "page_number", 8: "caption", 9: "reference",
10: "list", 11: "code", 12: "abstract", 13: "author", 14: "keywords",
15: "seal", 16: "image", 17: "paragraph",
}
PROMPTS = {
"table": "Convertis le tableau de cette image en HTML fidele (balises <table>), sans aucun commentaire.",
"formula": "Transcris la ou les formules mathematiques de cette image en LaTeX pur, sans explication.",
"figure": "Decris brievement cette figure en une phrase.",
"image": "Decris brievement cette image en une phrase.",
"seal": "Decris ce sceau/tampon : texte lisible et forme.",
"title": "Extrais le titre de cette image, tel quel, sans commentaire.",
"code": "Extrais le code de cette image tel quel, dans un bloc de code, sans commentaire.",
}
DEFAULT_PROMPT = "Extrais tout le texte de cette image tel quel, sans commentaire."
state = {"ready": False, "session": None, "input_name": None,
"in_w": 640, "in_h": 640, "error": None}
HTTPX = httpx.Client(timeout=900.0)
def load_model():
"""Download (cached in HF_HOME) and load the PP-DocLayoutV3 ONNX model on CPU."""
print(f"[layout] Downloading {LAYOUT_MODEL} (cached in {os.getenv('HF_HOME', '/data/hf')})...")
t0 = time.time()
snapshot_dir = snapshot_download(
repo_id=LAYOUT_MODEL,
allow_patterns=["*.onnx", "*.json", "*.txt", "*.yml", "*.yaml"],
)
onnx_files = glob.glob(os.path.join(snapshot_dir, "**", "*.onnx"), recursive=True)
if not onnx_files:
raise RuntimeError(f"No .onnx file found in {snapshot_dir}")
# prefer model.onnx, else the largest (fp32) one
onnx_files.sort(key=lambda p: (os.path.basename(p) != "model.onnx", -os.path.getsize(p)))
model_path = onnx_files[0]
print(f"[layout] Loading ONNX model: {model_path} ({os.path.getsize(model_path)/1e6:.1f} MB)")
sess = ort.InferenceSession(model_path, providers=["CPUExecutionProvider"])
inputs = sess.get_inputs()
# Paddle-style export: 3 inputs -> image (NCHW), im_shape (N,2), scale_factor (N,2)
image_input = next(i for i in inputs if len(i.shape) == 4)
h = image_input.shape[2] if isinstance(image_input.shape[2], int) else 800
w = image_input.shape[3] if isinstance(image_input.shape[3], int) else 800
state["in_h"], state["in_w"] = int(h), int(w)
state["input_names"] = {i.name: len(i.shape) for i in inputs}
state["session"] = sess
state["ready"] = True
print(f"[layout] Model ready in {time.time()-t0:.1f}s (input {state['in_w']}x{state['in_h']}, CPU, inputs={list(state['input_names'])})")
@asynccontextmanager
async def lifespan(app: FastAPI):
try:
load_model()
except Exception as e:
state["error"] = str(e)
print(f"[layout] FATAL: {e}")
yield
app = FastAPI(title="PP-DocLayoutV3 layout service (CPU)", lifespan=lifespan)
class ParseRequest(BaseModel):
image: str # data URL (image/* or application/pdf) or raw base64
max_tokens: int = 4096
score_threshold: float = 0.5
@app.get("/health")
def health():
if state["ready"]:
return {"status": "ok", "model": LAYOUT_MODEL, "device": "cpu"}
raise HTTPException(status_code=503, detail=f"layout model not ready: {state['error']}")
def split_data_url(s: str):
if s.startswith("data:"):
head, _, b64 = s.partition(",")
mime = head[5:].split(";")[0] or "application/octet-stream"
return mime, b64
return "application/octet-stream", s
def pdf_pages_to_images(raw: bytes):
import pymupdf # pymupdf (fitz)
doc = pymupdf.open(stream=raw, filetype="pdf")
pages = []
for page in doc:
pix = page.get_pixmap(dpi=200)
pages.append(Image.frombytes("RGB", (pix.width, pix.height), pix.samples))
doc.close()
return pages
def detect_regions(img: Image.Image, score_threshold: float):
"""Run PP-DocLayoutV3 ONNX on one page image, return regions in reading order.
Official PaddlePaddle/PP-DocLayoutV3_onnx export:
inputs : image (N,3,H,W) float32 rescaled 1/255 (mean 0, std 1),
im_shape (N,2) = [orig_h, orig_w], scale_factor (N,2) = [orig_h/H, orig_w/W]
outputs: fetch_name_0 (N,7) rows = [class_id, score, x1, y1, x2, y2, order]
(boxes in original image coordinates), plus a label count and a mask (ignored)
"""
sess = state["session"]
orig_w, orig_h = img.size
in_w, in_h = state["in_w"], state["in_h"]
resized = img.resize((in_w, in_h))
arr = np.asarray(resized, dtype=np.float32) / 255.0
image_tensor = arr.transpose(2, 0, 1)[None, ...] # NCHW
im_shape = np.array([[orig_h, orig_w]], dtype=np.float32)
scale_factor = np.array([[orig_h / in_h, orig_w / in_w]], dtype=np.float32)
feed = {}
for name, ndim in state["input_names"].items():
lname = name.lower()
if ndim == 4 or lname == "image":
feed[name] = image_tensor
elif "im_shape" in lname or ndim == 2 and "scale" not in lname and "shape" in lname:
feed[name] = im_shape
elif "scale" in lname or ndim == 2:
feed[name] = scale_factor
outputs = sess.run(None, feed)
rows = None
for out in outputs:
a = np.asarray(out)
if a.ndim == 2 and a.shape[-1] == 7:
rows = a
break
if rows is None:
raise RuntimeError(f"Unexpected ONNX outputs: {[np.asarray(o).shape for o in outputs]}")
regions = []
for row in rows:
cls_id, score = int(row[0]), float(row[1])
x1, y1, x2, y2 = [float(v) for v in row[2:6]]
order = int(row[6])
if score < score_threshold:
continue
x1, y1 = max(0.0, x1), max(0.0, y1)
x2, y2 = min(orig_w, x2), min(orig_h, y2)
if x2 - x1 < 8 or y2 - y1 < 8:
continue
regions.append({
"label": LABELS.get(cls_id, f"class_{cls_id}"),
"score": round(score, 4),
"bbox": [round(x1, 1), round(y1, 1), round(x2, 1), round(y2, 1)],
"polygon": None,
"order": order,
})
regions.sort(key=lambda r: r["order"])
for idx, r in enumerate(regions):
r["order"] = idx
return regions
def ocr_region(crop: Image.Image, label: str, max_tokens: int) -> str:
buf = io.BytesIO()
crop.save(buf, format="JPEG", quality=90)
b64 = base64.b64encode(buf.getvalue()).decode()
payload = {
"model": SERVED_OCR,
"messages": [{
"role": "user",
"content": [
{"type": "text", "text": PROMPTS.get(label, DEFAULT_PROMPT)},
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}},
],
}],
"max_tokens": max_tokens,
"temperature": 0.0,
}
r = HTTPX.post(f"http://127.0.0.1:{PORT_OCR}/v1/chat/completions", json=payload)
r.raise_for_status()
return r.json()["choices"][0]["message"]["content"]
@app.post("/parse")
def parse(req: ParseRequest):
if not state["ready"]:
raise HTTPException(status_code=503, detail=f"layout model not ready: {state['error']}")
mime, b64 = split_data_url(req.image)
try:
raw = base64.b64decode(b64, validate=True)
except Exception:
raise HTTPException(status_code=400, detail="Invalid base64 image data")
if mime == "application/pdf" or raw[:5] == b"%PDF-":
pages = pdf_pages_to_images(raw)
else:
try:
pages = [Image.open(io.BytesIO(raw)).convert("RGB")]
except Exception:
raise HTTPException(status_code=400, detail=f"Unsupported or unreadable image (mime={mime})")
all_regions = []
md_parts = []
for pidx, page in enumerate(pages):
regions = detect_regions(page, req.score_threshold)
for r in regions:
x1, y1, x2, y2 = [int(round(v)) for v in r["bbox"]]
crop = page.crop((x1, y1, x2, y2))
try:
r["ocr"] = ocr_region(crop, r["label"], req.max_tokens)
except Exception as e:
r["ocr"] = f"[OCR error: {e}]"
r["page"] = pidx + 1
all_regions.append(r)
md_parts.append(r["ocr"])
print(f"[layout] page {pidx+1}/{len(pages)}: {len(regions)} regions")
return {"pages": len(pages), "regions": all_regions, "markdown": "\n\n".join(md_parts)}
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="127.0.0.1", port=LAYOUT_PORT, log_level="info")
|