""" MSME Document Verifier — Hugging Face Spaces Demo ================================================== Self-hosted · multilingual · zero per-doc cost Pipeline: PIL pre-processing → EasyOCR → Qwen2.5-7B-Instruct (HF Inference API) → IndicTrans2 (Helsinki-NLP fallback) → Gradio UI """ import base64 import json import logging import os import warnings from pathlib import Path from typing import Any import cv2 import numpy as np import requests from PIL import Image, ImageOps import gradio as gr import easyocr from huggingface_hub import InferenceClient import concurrent.futures warnings.filterwarnings("ignore") # ── Logging ─────────────────────────────────────────────────────────────────── logging.basicConfig( level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s", datefmt="%H:%M:%S", ) logger = logging.getLogger("msme_verifier") for _noisy in ("httpx", "httpcore", "urllib3", "gradio", "PIL", "easyocr"): logging.getLogger(_noisy).setLevel(logging.WARNING) import io as _io import threading as _threading class _UILogHandler(logging.Handler): def __init__(self): super().__init__() self._local = _threading.local() def start_capture(self): self._local.buf = _io.StringIO() def stop_capture(self): buf = getattr(self._local, "buf", None) if buf is None: return "" text = buf.getvalue() self._local.buf = None return text def emit(self, record): buf = getattr(self._local, "buf", None) if buf is not None: buf.write(self.format(record) + "\n") _ui_handler = _UILogHandler() _ui_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)-8s %(message)s", datefmt="%H:%M:%S")) logger.addHandler(_ui_handler) # ── Constants ───────────────────────────────────────────────────────────────── HF_TOKEN = os.environ.get("HF_TOKEN", "") LLM_MODEL = "Qwen/Qwen2.5-7B-Instruct" MAX_IMAGE_DIM = 1024 # English-only EasyOCR — Hindi/Indic adds ~300MB models + 5x slower on CPU # Scene heuristics + LLM handle classification without needing script-specific OCR EASYOCR_LANGS = ["en", "hi"] # Hindi + English; models download on first run (~150MB) OCR_TIMEOUT = 120 # seconds — fall back to heuristics-only if exceeded INDIC_LANG_CODES: dict[str, str] = { "hi": "hin_Deva", "ta": "tam_Taml", "te": "tel_Telu", "ml": "mal_Mlym", "bn": "ben_Beng", "kn": "kan_Knda", "gu": "guj_Gujr", "mr": "mar_Deva", "pa": "pan_Guru", "or": "ory_Orya", "as": "asm_Beng", "ur": "urd_Arab", } # ── EasyOCR — loaded once ───────────────────────────────────────────────────── try: logger.info("Loading EasyOCR (first run downloads ~500 MB model) …") _ocr_reader = easyocr.Reader(EASYOCR_LANGS, gpu=False, verbose=False) OCR_BACKEND = "easyocr" logger.info("✓ EasyOCR ready") except Exception as e: logger.error("✗ EasyOCR failed: %s", e) _ocr_reader = None OCR_BACKEND = "none" # ── HF Inference client ─────────────────────────────────────────────────────── _llm_client = InferenceClient(model=LLM_MODEL, token=HF_TOKEN) if HF_TOKEN else None SYSTEM_PROMPT = """ You are a document-analysis assistant specialising in Indian MSME paperwork. Analyse the provided OCR text and image signals, then return ONLY a valid JSON object matching the schema below — no markdown fences, no prose outside the object. SCHEMA: { "imageType": "photo|certificate|letter|document|invoice|identity|screenshot", "category": "string, snake_case, e.g. gst_certificate / storefront / admission_letter", "categoryConfidence": , "subCategory": "string or null", "languages": [ {"code": "en|hi|ml|ta|...", "name": "string", "confidence": , "isPrimary": true|false} ], "text": { "english": "full English text or translation of all visible text", "original": "original mixed-language text exactly as seen" }, "content": [ {"type": "field|table|signature|stamp|logo|barcode|qr", "label": "snake_case", "value": "string", "confidence": } ], "scene": { "description": "1-2 sentence visual description", "objects": ["string"], "isStorefront": true|false, "isPhotoOfDocument": true|false }, "verificationSignals": { "tamperingIndicators": ["string"], "anomalies": ["string"] }, "qualityIndicators": { "meanOcrConfidence": , "issues": ["string"] } } FEW-SHOT EXAMPLE 1 — Shop storefront OCR: ["SOMANY Tiles", "JANTA HARDWARE AND PAINT", "9450882490", "orientbell tiles"] OUTPUT: {"imageType":"photo","category":"storefront","categoryConfidence":0.97,...} FEW-SHOT EXAMPLE 2 — GST Certificate OCR: ["GSTIN: 27AABCS1234F1Z5", "Certificate of Registration", "Sharma Textiles Pvt Ltd"] OUTPUT: {"imageType":"certificate","category":"gst_certificate","categoryConfidence":0.99,...} Now analyse and return JSON only. """.strip() # ── Image pre-processing ────────────────────────────────────────────────────── def preprocess(image_path: str) -> str: try: pil = Image.open(image_path) pil = ImageOps.exif_transpose(pil).convert("RGB") bgr = cv2.cvtColor(np.array(pil), cv2.COLOR_RGB2BGR) # CLAHE contrast enhancement lab = cv2.cvtColor(bgr, cv2.COLOR_BGR2LAB) l, a, b = cv2.split(lab) l = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8)).apply(l) bgr = cv2.cvtColor(cv2.merge([l, a, b]), cv2.COLOR_LAB2BGR) # Downscale h, w = bgr.shape[:2] if max(h, w) > MAX_IMAGE_DIM: scale = MAX_IMAGE_DIM / max(h, w) bgr = cv2.resize(bgr, (int(w * scale), int(h * scale)), interpolation=cv2.INTER_AREA) out = str(Path(image_path).with_stem(Path(image_path).stem + "_prep")) cv2.imwrite(out, bgr) return out except Exception as exc: logger.warning("preprocess failed: %s", exc) return image_path # ── OCR ─────────────────────────────────────────────────────────────────────── def run_ocr(image_path: str) -> list[dict]: if _ocr_reader is None: return [] def _do_ocr(): return _ocr_reader.readtext(image_path) try: with concurrent.futures.ThreadPoolExecutor(max_workers=1) as ex: future = ex.submit(_do_ocr) try: results = future.result(timeout=OCR_TIMEOUT) except concurrent.futures.TimeoutError: logger.warning("EasyOCR timed out after %ds — falling back to heuristics only", OCR_TIMEOUT) return [] lines = [] for (bbox, text, conf) in results: if not text.strip() or conf < 0.3: continue xs = [p[0] for p in bbox] ys = [p[1] for p in bbox] lines.append({ "text": text, "confidence": float(conf), "bbox": [[int(min(xs)), int(min(ys))], [int(max(xs)), int(min(ys))], [int(max(xs)), int(max(ys))], [int(min(xs)), int(max(ys))]], }) logger.info("EasyOCR: %d lines detected", len(lines)) return lines except Exception as exc: logger.warning("run_ocr failed: %s", exc) return [] # ── Scene heuristics ────────────────────────────────────────────────────────── def _scene_heuristics(image_path: str, ocr_lines: list[dict]) -> dict: try: pil = Image.open(image_path).convert("RGB") w, h = pil.size arr = np.array(pil.resize((256, 256), Image.LANCZOS), dtype=float) rg = arr[:, :, 0] - arr[:, :, 1] yb = 0.5 * (arr[:, :, 0] + arr[:, :, 1]) - arr[:, :, 2] colorfulness = float( np.sqrt(rg.std() ** 2 + yb.std() ** 2) + 0.3 * np.sqrt(rg.mean() ** 2 + yb.mean() ** 2) ) aspect = w / max(h, 1) is_landscape = aspect > 1.1 mpx = (w * h) / 1_000_000 density = len(ocr_lines) / max(mpx, 0.1) score = 0.0 if colorfulness > 40: score += 0.5 elif colorfulness > 20: score += 0.25 if is_landscape: score += 0.2 if density < 3: score += 0.3 if score >= 0.7: hint = "LIKELY SCENE/PHOTO (real-world shop, person, place, or object)" elif score <= 0.3: hint = "LIKELY DOCUMENT (scanned/photographed form, certificate, letter)" else: hint = "UNCERTAIN — classify from text content" return dict(w=w, h=h, colorfulness=round(colorfulness, 1), is_landscape=is_landscape, ocr_density=round(density, 1), scene_score=round(score, 2), scene_hint=hint) except Exception as exc: logger.debug("_scene_heuristics: %s", exc) return dict(w=0, h=0, colorfulness=0.0, is_landscape=False, ocr_density=0.0, scene_score=0.5, scene_hint="UNCERTAIN") # ── Translation ─────────────────────────────────────────────────────────────── def translate_to_english(text: str, src_lang_code: str) -> str | None: """Use Helsinki-NLP/opus-mt models via HF Inference API for translation.""" if src_lang_code == "en" or src_lang_code not in INDIC_LANG_CODES: return None if not _llm_client: return None try: # Helsinki-NLP has specific indic→en models model = f"Helsinki-NLP/opus-mt-{src_lang_code}-en" client = InferenceClient(model=model, token=HF_TOKEN) result = client.translation(text, model=model) translated = result.translation_text if hasattr(result, "translation_text") else str(result) logger.info("Translation %s→en: %d chars", src_lang_code, len(translated)) return translated except Exception as exc: logger.warning("Translation failed (%s): %s", src_lang_code, exc) return None # ── JSON repair ─────────────────────────────────────────────────────────────── def _repair_json(raw: str) -> str: s = raw.strip() depth_brace = s.count("{") - s.count("}") depth_bracket = s.count("[") - s.count("]") if s.endswith(","): s = s[:-1] s += "]" * max(0, depth_bracket) + "}" * max(0, depth_brace) return s # ── LLM call ───────────────────────────────────────────────────────────────── def call_llm(image_path: str, ocr_lines: list[dict]) -> dict: if not _llm_client: return {"error": "no_hf_token", "detail": "HF_TOKEN secret not set in Space settings"} hints = _scene_heuristics(image_path, ocr_lines) logger.info("Scene: colorfulness=%.1f score=%.2f → %s", hints["colorfulness"], hints["scene_score"], hints["scene_hint"]) if ocr_lines: ocr_text = "\n".join(f'- "{ln["text"]}" (conf {ln["confidence"]:.2f})' for ln in ocr_lines) ocr_block = f"OCR extracted {len(ocr_lines)} text lines:\n{ocr_text}" else: ocr_block = "OCR returned no text lines." user_text = ( f"IMAGE SIGNALS\n" f"Dimensions : {hints['w']}×{hints['h']} px " f"({'landscape' if hints['is_landscape'] else 'portrait'})\n" f"Colorfulness: {hints['colorfulness']:.0f}/100 (documents≈0–15, photos≈30–90)\n" f"OCR density : {hints['ocr_density']:.1f} lines/MP\n" f"Pre-class : {hints['scene_hint']}\n\n" f"{ocr_block}\n\n" "Trust the pre-classification unless OCR clearly shows certificate/GSTIN/form fields.\n" "Return ONLY the JSON schema — no markdown, no prose." ) messages = [ {"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": user_text}, ] logger.info("Calling %s via HF Inference API …", LLM_MODEL) for attempt in range(1, 3): try: response = _llm_client.chat_completion( messages=messages, max_tokens=2048, temperature=0.1 if attempt == 1 else 0.0, ) raw = response.choices[0].message.content.strip() except Exception as exc: return {"error": "llm_error", "detail": str(exc)} if raw.startswith("```"): raw = raw.split("```", 2)[1] if raw.startswith("json"): raw = raw[4:] raw = raw.rsplit("```", 1)[0].strip() try: return json.loads(raw) except json.JSONDecodeError: try: return json.loads(_repair_json(raw)) except json.JSONDecodeError: pass if attempt == 1: messages.append({"role": "assistant", "content": raw}) messages.append({"role": "user", "content": "Not valid JSON. Return ONLY the JSON object."}) else: return {"error": "parse_failed", "raw": raw[:500]} return {"error": "parse_failed", "raw": ""} # ── UI ──────────────────────────────────────────────────────────────────────── def verify(image_path: str, progress=gr.Progress(track_tqdm=True)) -> tuple: _ui_handler.start_capture() logger.info("┌─ Pipeline: %s", Path(image_path).name) progress(0.05, desc="[1/3] Preprocessing — CLAHE · resize …") logger.info("│ [1/3] Preprocessing") prep = preprocess(image_path) progress(0.25, desc="[2/3] EasyOCR — detecting text … (first run downloads models, ~60s. Subsequent runs ~15s)") logger.info("│ [2/3] EasyOCR (~10–20s on CPU)") ocr_lines = run_ocr(prep) logger.info("│ → %d lines", len(ocr_lines)) progress(0.55, desc="[3/3] Qwen2.5-7B — classifying via HF API … (5–15 s)") logger.info("│ [3/3] Qwen2.5-7B via HF Inference API") result = call_llm(prep, ocr_lines) progress(0.90, desc="Finishing …") # Annotations annotations = [] for ln in ocr_lines: try: xs = [p[0] for p in ln["bbox"]] ys = [p[1] for p in ln["bbox"]] annotations.append(((int(min(xs)), int(min(ys)), int(max(xs)), int(max(ys))), ln["text"][:30])) except Exception: pass annotated = (prep, annotations) if "error" in result: err = result["error"] if err == "no_hf_token": msg = "**Setup required:** Add `HF_TOKEN` in Space Settings → Repository secrets." else: msg = f"**Error:** `{err}` — {result.get('detail', '')}" pipeline_log = _ui_handler.stop_capture() return annotated, msg, [], msg, "_N/A_", result, pipeline_log # Summary card conf = float(result.get("categoryConfidence", 0.0)) indicator = "🟢" if conf > 0.85 else ("🟡" if conf > 0.60 else "🔴") category = result.get("category", "unknown").replace("_", " ").title() image_type = result.get("imageType", "—") langs = result.get("languages", []) primary_langs = [lg["name"] for lg in langs if lg.get("isPrimary")] or ([langs[0]["name"]] if langs else ["—"]) scene_desc = result.get("scene", {}).get("description", "") summary = ( f"## {indicator} {category}\n" f"**Type:** {image_type}  |  " f"**Confidence:** {conf*100:.0f}%  |  " f"**Language(s):** {', '.join(primary_langs)}\n\n" f"_{scene_desc}_" ) # Fields table fields = [ [item.get("label", "").replace("_", " ").title(), item.get("value", ""), f"{item.get('confidence', 0)*100:.0f}%"] for item in result.get("content", []) ] # Flags signals = result.get("verificationSignals", {}) tampering = signals.get("tamperingIndicators", []) anomalies = signals.get("anomalies", []) flags = "" if tampering: flags += "**⚠ Tampering indicators:**\n" + "\n".join(f"- {t}" for t in tampering) + "\n\n" if anomalies: flags += "**⚠ Anomalies:**\n" + "\n".join(f"- {a}" for a in anomalies) if not flags: flags = "✅ No flags raised" # Translation non_english = [lg for lg in langs if lg.get("code") != "en" and lg.get("code") in INDIC_LANG_CODES] translation_md = "" if non_english: orig_text = result.get("text", {}).get("original", "") if orig_text: for lg in non_english[:2]: translated = translate_to_english(orig_text, lg["code"]) if translated: translation_md += f"**{lg['name']} → English:**\n> {translated}\n\n" if not translation_md: translation_md = "_No regional language text detected or translation not available._" progress(1.0, desc="Done ✓") pipeline_log = _ui_handler.stop_capture() return annotated, summary, fields, flags, translation_md, result, pipeline_log def build_ui() -> gr.Blocks: with gr.Blocks(title="Image Classifier & Text Reader", theme=gr.themes.Soft()) as demo: gr.Markdown(""" # 🔍 Image Classifier & Text Reader **Upload any image — storefronts, documents, invoices, certificates, photos** Automatically classifies the image, reads all text (English + Hindi), extracts key fields, and translates regional language content to English. """) with gr.Row(): with gr.Column(scale=1): img_input = gr.Image(type="filepath", label="Upload Image") submit_btn = gr.Button("🔍 Verify Document", variant="primary", size="lg") gr.Examples( examples=[], inputs=img_input, ) with gr.Column(scale=2): with gr.Tabs(): with gr.Tab("📋 Summary"): annotated_out = gr.AnnotatedImage(label="OCR Overlay") summary_out = gr.Markdown() with gr.Tab("📊 Fields"): fields_out = gr.Dataframe( headers=["Field", "Value", "Confidence"], datatype=["str", "str", "str"], label="Extracted Fields", ) with gr.Tab("🚩 Flags"): flags_out = gr.Markdown() with gr.Tab("🌐 Translation"): translation_out = gr.Markdown() with gr.Tab("🔧 Raw JSON"): json_out = gr.JSON(label="Full VLM Output") with gr.Tab("📜 Pipeline Log"): log_out = gr.Textbox( label="Processing Log", lines=15, max_lines=30, ) submit_btn.click( fn=verify, inputs=[img_input], outputs=[annotated_out, summary_out, fields_out, flags_out, translation_out, json_out, log_out], ) return demo demo = build_ui() demo.queue(default_concurrency_limit=2) demo.launch()