amarnathshenoy's picture
Rename UI to Image Classifier & Text Reader
de64e2c verified
Raw History Blame Contribute Delete
20.4 kB
"""
MSME Document Verifier β€” Hugging Face Spaces Demo
==================================================
Self-hosted Β· multilingual Β· zero per-doc cost
Pipeline: PIL pre-processing β†’ EasyOCR β†’ Qwen2.5-7B-Instruct (HF Inference API)
β†’ IndicTrans2 (Helsinki-NLP fallback) β†’ Gradio UI
"""
import base64
import json
import logging
import os
import warnings
from pathlib import Path
from typing import Any
import cv2
import numpy as np
import requests
from PIL import Image, ImageOps
import gradio as gr
import easyocr
from huggingface_hub import InferenceClient
import concurrent.futures
warnings.filterwarnings("ignore")
# ── Logging ───────────────────────────────────────────────────────────────────
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)-8s %(message)s",
datefmt="%H:%M:%S",
)
logger = logging.getLogger("msme_verifier")
for _noisy in ("httpx", "httpcore", "urllib3", "gradio", "PIL", "easyocr"):
logging.getLogger(_noisy).setLevel(logging.WARNING)
import io as _io
import threading as _threading
class _UILogHandler(logging.Handler):
def __init__(self):
super().__init__()
self._local = _threading.local()
def start_capture(self):
self._local.buf = _io.StringIO()
def stop_capture(self):
buf = getattr(self._local, "buf", None)
if buf is None:
return ""
text = buf.getvalue()
self._local.buf = None
return text
def emit(self, record):
buf = getattr(self._local, "buf", None)
if buf is not None:
buf.write(self.format(record) + "\n")
_ui_handler = _UILogHandler()
_ui_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)-8s %(message)s", datefmt="%H:%M:%S"))
logger.addHandler(_ui_handler)
# ── Constants ─────────────────────────────────────────────────────────────────
HF_TOKEN = os.environ.get("HF_TOKEN", "")
LLM_MODEL = "Qwen/Qwen2.5-7B-Instruct"
MAX_IMAGE_DIM = 1024
# English-only EasyOCR β€” Hindi/Indic adds ~300MB models + 5x slower on CPU
# Scene heuristics + LLM handle classification without needing script-specific OCR
EASYOCR_LANGS = ["en", "hi"] # Hindi + English; models download on first run (~150MB)
OCR_TIMEOUT = 120 # seconds β€” fall back to heuristics-only if exceeded
INDIC_LANG_CODES: dict[str, str] = {
"hi": "hin_Deva", "ta": "tam_Taml", "te": "tel_Telu", "ml": "mal_Mlym",
"bn": "ben_Beng", "kn": "kan_Knda", "gu": "guj_Gujr", "mr": "mar_Deva",
"pa": "pan_Guru", "or": "ory_Orya", "as": "asm_Beng", "ur": "urd_Arab",
}
# ── EasyOCR β€” loaded once ─────────────────────────────────────────────────────
try:
logger.info("Loading EasyOCR (first run downloads ~500 MB model) …")
_ocr_reader = easyocr.Reader(EASYOCR_LANGS, gpu=False, verbose=False)
OCR_BACKEND = "easyocr"
logger.info("βœ“ EasyOCR ready")
except Exception as e:
logger.error("βœ— EasyOCR failed: %s", e)
_ocr_reader = None
OCR_BACKEND = "none"
# ── HF Inference client ───────────────────────────────────────────────────────
_llm_client = InferenceClient(model=LLM_MODEL, token=HF_TOKEN) if HF_TOKEN else None
SYSTEM_PROMPT = """
You are a document-analysis assistant specialising in Indian MSME paperwork.
Analyse the provided OCR text and image signals, then return ONLY a valid JSON object
matching the schema below β€” no markdown fences, no prose outside the object.
SCHEMA:
{
"imageType": "photo|certificate|letter|document|invoice|identity|screenshot",
"category": "string, snake_case, e.g. gst_certificate / storefront / admission_letter",
"categoryConfidence": <float 0.0–1.0>,
"subCategory": "string or null",
"languages": [
{"code": "en|hi|ml|ta|...", "name": "string", "confidence": <float>, "isPrimary": true|false}
],
"text": {
"english": "full English text or translation of all visible text",
"original": "original mixed-language text exactly as seen"
},
"content": [
{"type": "field|table|signature|stamp|logo|barcode|qr", "label": "snake_case", "value": "string", "confidence": <float>}
],
"scene": {
"description": "1-2 sentence visual description",
"objects": ["string"],
"isStorefront": true|false,
"isPhotoOfDocument": true|false
},
"verificationSignals": {
"tamperingIndicators": ["string"],
"anomalies": ["string"]
},
"qualityIndicators": {
"meanOcrConfidence": <float 0.0–1.0>,
"issues": ["string"]
}
}
FEW-SHOT EXAMPLE 1 β€” Shop storefront
OCR: ["SOMANY Tiles", "JANTA HARDWARE AND PAINT", "9450882490", "orientbell tiles"]
OUTPUT: {"imageType":"photo","category":"storefront","categoryConfidence":0.97,...}
FEW-SHOT EXAMPLE 2 β€” GST Certificate
OCR: ["GSTIN: 27AABCS1234F1Z5", "Certificate of Registration", "Sharma Textiles Pvt Ltd"]
OUTPUT: {"imageType":"certificate","category":"gst_certificate","categoryConfidence":0.99,...}
Now analyse and return JSON only.
""".strip()
# ── Image pre-processing ──────────────────────────────────────────────────────
def preprocess(image_path: str) -> str:
try:
pil = Image.open(image_path)
pil = ImageOps.exif_transpose(pil).convert("RGB")
bgr = cv2.cvtColor(np.array(pil), cv2.COLOR_RGB2BGR)
# CLAHE contrast enhancement
lab = cv2.cvtColor(bgr, cv2.COLOR_BGR2LAB)
l, a, b = cv2.split(lab)
l = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8)).apply(l)
bgr = cv2.cvtColor(cv2.merge([l, a, b]), cv2.COLOR_LAB2BGR)
# Downscale
h, w = bgr.shape[:2]
if max(h, w) > MAX_IMAGE_DIM:
scale = MAX_IMAGE_DIM / max(h, w)
bgr = cv2.resize(bgr, (int(w * scale), int(h * scale)), interpolation=cv2.INTER_AREA)
out = str(Path(image_path).with_stem(Path(image_path).stem + "_prep"))
cv2.imwrite(out, bgr)
return out
except Exception as exc:
logger.warning("preprocess failed: %s", exc)
return image_path
# ── OCR ───────────────────────────────────────────────────────────────────────
def run_ocr(image_path: str) -> list[dict]:
if _ocr_reader is None:
return []
def _do_ocr():
return _ocr_reader.readtext(image_path)
try:
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as ex:
future = ex.submit(_do_ocr)
try:
results = future.result(timeout=OCR_TIMEOUT)
except concurrent.futures.TimeoutError:
logger.warning("EasyOCR timed out after %ds β€” falling back to heuristics only", OCR_TIMEOUT)
return []
lines = []
for (bbox, text, conf) in results:
if not text.strip() or conf < 0.3:
continue
xs = [p[0] for p in bbox]
ys = [p[1] for p in bbox]
lines.append({
"text": text,
"confidence": float(conf),
"bbox": [[int(min(xs)), int(min(ys))], [int(max(xs)), int(min(ys))],
[int(max(xs)), int(max(ys))], [int(min(xs)), int(max(ys))]],
})
logger.info("EasyOCR: %d lines detected", len(lines))
return lines
except Exception as exc:
logger.warning("run_ocr failed: %s", exc)
return []
# ── Scene heuristics ──────────────────────────────────────────────────────────
def _scene_heuristics(image_path: str, ocr_lines: list[dict]) -> dict:
try:
pil = Image.open(image_path).convert("RGB")
w, h = pil.size
arr = np.array(pil.resize((256, 256), Image.LANCZOS), dtype=float)
rg = arr[:, :, 0] - arr[:, :, 1]
yb = 0.5 * (arr[:, :, 0] + arr[:, :, 1]) - arr[:, :, 2]
colorfulness = float(
np.sqrt(rg.std() ** 2 + yb.std() ** 2) +
0.3 * np.sqrt(rg.mean() ** 2 + yb.mean() ** 2)
)
aspect = w / max(h, 1)
is_landscape = aspect > 1.1
mpx = (w * h) / 1_000_000
density = len(ocr_lines) / max(mpx, 0.1)
score = 0.0
if colorfulness > 40:
score += 0.5
elif colorfulness > 20:
score += 0.25
if is_landscape:
score += 0.2
if density < 3:
score += 0.3
if score >= 0.7:
hint = "LIKELY SCENE/PHOTO (real-world shop, person, place, or object)"
elif score <= 0.3:
hint = "LIKELY DOCUMENT (scanned/photographed form, certificate, letter)"
else:
hint = "UNCERTAIN β€” classify from text content"
return dict(w=w, h=h, colorfulness=round(colorfulness, 1),
is_landscape=is_landscape, ocr_density=round(density, 1),
scene_score=round(score, 2), scene_hint=hint)
except Exception as exc:
logger.debug("_scene_heuristics: %s", exc)
return dict(w=0, h=0, colorfulness=0.0, is_landscape=False,
ocr_density=0.0, scene_score=0.5, scene_hint="UNCERTAIN")
# ── Translation ───────────────────────────────────────────────────────────────
def translate_to_english(text: str, src_lang_code: str) -> str | None:
"""Use Helsinki-NLP/opus-mt models via HF Inference API for translation."""
if src_lang_code == "en" or src_lang_code not in INDIC_LANG_CODES:
return None
if not _llm_client:
return None
try:
# Helsinki-NLP has specific indic→en models
model = f"Helsinki-NLP/opus-mt-{src_lang_code}-en"
client = InferenceClient(model=model, token=HF_TOKEN)
result = client.translation(text, model=model)
translated = result.translation_text if hasattr(result, "translation_text") else str(result)
logger.info("Translation %s→en: %d chars", src_lang_code, len(translated))
return translated
except Exception as exc:
logger.warning("Translation failed (%s): %s", src_lang_code, exc)
return None
# ── JSON repair ───────────────────────────────────────────────────────────────
def _repair_json(raw: str) -> str:
s = raw.strip()
depth_brace = s.count("{") - s.count("}")
depth_bracket = s.count("[") - s.count("]")
if s.endswith(","):
s = s[:-1]
s += "]" * max(0, depth_bracket) + "}" * max(0, depth_brace)
return s
# ── LLM call ─────────────────────────────────────────────────────────────────
def call_llm(image_path: str, ocr_lines: list[dict]) -> dict:
if not _llm_client:
return {"error": "no_hf_token", "detail": "HF_TOKEN secret not set in Space settings"}
hints = _scene_heuristics(image_path, ocr_lines)
logger.info("Scene: colorfulness=%.1f score=%.2f β†’ %s",
hints["colorfulness"], hints["scene_score"], hints["scene_hint"])
if ocr_lines:
ocr_text = "\n".join(f'- "{ln["text"]}" (conf {ln["confidence"]:.2f})' for ln in ocr_lines)
ocr_block = f"OCR extracted {len(ocr_lines)} text lines:\n{ocr_text}"
else:
ocr_block = "OCR returned no text lines."
user_text = (
f"IMAGE SIGNALS\n"
f"Dimensions : {hints['w']}Γ—{hints['h']} px "
f"({'landscape' if hints['is_landscape'] else 'portrait'})\n"
f"Colorfulness: {hints['colorfulness']:.0f}/100 (documentsβ‰ˆ0–15, photosβ‰ˆ30–90)\n"
f"OCR density : {hints['ocr_density']:.1f} lines/MP\n"
f"Pre-class : {hints['scene_hint']}\n\n"
f"{ocr_block}\n\n"
"Trust the pre-classification unless OCR clearly shows certificate/GSTIN/form fields.\n"
"Return ONLY the JSON schema β€” no markdown, no prose."
)
messages = [
{"role": "system", "content": SYSTEM_PROMPT},
{"role": "user", "content": user_text},
]
logger.info("Calling %s via HF Inference API …", LLM_MODEL)
for attempt in range(1, 3):
try:
response = _llm_client.chat_completion(
messages=messages,
max_tokens=2048,
temperature=0.1 if attempt == 1 else 0.0,
)
raw = response.choices[0].message.content.strip()
except Exception as exc:
return {"error": "llm_error", "detail": str(exc)}
if raw.startswith("```"):
raw = raw.split("```", 2)[1]
if raw.startswith("json"):
raw = raw[4:]
raw = raw.rsplit("```", 1)[0].strip()
try:
return json.loads(raw)
except json.JSONDecodeError:
try:
return json.loads(_repair_json(raw))
except json.JSONDecodeError:
pass
if attempt == 1:
messages.append({"role": "assistant", "content": raw})
messages.append({"role": "user", "content": "Not valid JSON. Return ONLY the JSON object."})
else:
return {"error": "parse_failed", "raw": raw[:500]}
return {"error": "parse_failed", "raw": ""}
# ── UI ────────────────────────────────────────────────────────────────────────
def verify(image_path: str, progress=gr.Progress(track_tqdm=True)) -> tuple:
_ui_handler.start_capture()
logger.info("β”Œβ”€ Pipeline: %s", Path(image_path).name)
progress(0.05, desc="[1/3] Preprocessing β€” CLAHE Β· resize …")
logger.info("β”‚ [1/3] Preprocessing")
prep = preprocess(image_path)
progress(0.25, desc="[2/3] EasyOCR β€” detecting text … (first run downloads models, ~60s. Subsequent runs ~15s)")
logger.info("β”‚ [2/3] EasyOCR (~10–20s on CPU)")
ocr_lines = run_ocr(prep)
logger.info("β”‚ β†’ %d lines", len(ocr_lines))
progress(0.55, desc="[3/3] Qwen2.5-7B β€” classifying via HF API … (5–15 s)")
logger.info("β”‚ [3/3] Qwen2.5-7B via HF Inference API")
result = call_llm(prep, ocr_lines)
progress(0.90, desc="Finishing …")
# Annotations
annotations = []
for ln in ocr_lines:
try:
xs = [p[0] for p in ln["bbox"]]
ys = [p[1] for p in ln["bbox"]]
annotations.append(((int(min(xs)), int(min(ys)), int(max(xs)), int(max(ys))), ln["text"][:30]))
except Exception:
pass
annotated = (prep, annotations)
if "error" in result:
err = result["error"]
if err == "no_hf_token":
msg = "**Setup required:** Add `HF_TOKEN` in Space Settings β†’ Repository secrets."
else:
msg = f"**Error:** `{err}` β€” {result.get('detail', '')}"
pipeline_log = _ui_handler.stop_capture()
return annotated, msg, [], msg, "_N/A_", result, pipeline_log
# Summary card
conf = float(result.get("categoryConfidence", 0.0))
indicator = "🟒" if conf > 0.85 else ("🟑" if conf > 0.60 else "πŸ”΄")
category = result.get("category", "unknown").replace("_", " ").title()
image_type = result.get("imageType", "β€”")
langs = result.get("languages", [])
primary_langs = [lg["name"] for lg in langs if lg.get("isPrimary")] or ([langs[0]["name"]] if langs else ["β€”"])
scene_desc = result.get("scene", {}).get("description", "")
summary = (
f"## {indicator} {category}\n"
f"**Type:** {image_type} &nbsp;|&nbsp; "
f"**Confidence:** {conf*100:.0f}% &nbsp;|&nbsp; "
f"**Language(s):** {', '.join(primary_langs)}\n\n"
f"_{scene_desc}_"
)
# Fields table
fields = [
[item.get("label", "").replace("_", " ").title(),
item.get("value", ""),
f"{item.get('confidence', 0)*100:.0f}%"]
for item in result.get("content", [])
]
# Flags
signals = result.get("verificationSignals", {})
tampering = signals.get("tamperingIndicators", [])
anomalies = signals.get("anomalies", [])
flags = ""
if tampering:
flags += "**⚠ Tampering indicators:**\n" + "\n".join(f"- {t}" for t in tampering) + "\n\n"
if anomalies:
flags += "**⚠ Anomalies:**\n" + "\n".join(f"- {a}" for a in anomalies)
if not flags:
flags = "βœ… No flags raised"
# Translation
non_english = [lg for lg in langs if lg.get("code") != "en" and lg.get("code") in INDIC_LANG_CODES]
translation_md = ""
if non_english:
orig_text = result.get("text", {}).get("original", "")
if orig_text:
for lg in non_english[:2]:
translated = translate_to_english(orig_text, lg["code"])
if translated:
translation_md += f"**{lg['name']} β†’ English:**\n> {translated}\n\n"
if not translation_md:
translation_md = "_No regional language text detected or translation not available._"
progress(1.0, desc="Done βœ“")
pipeline_log = _ui_handler.stop_capture()
return annotated, summary, fields, flags, translation_md, result, pipeline_log
def build_ui() -> gr.Blocks:
with gr.Blocks(title="Image Classifier & Text Reader", theme=gr.themes.Soft()) as demo:
gr.Markdown("""
# πŸ” Image Classifier & Text Reader
**Upload any image β€” storefronts, documents, invoices, certificates, photos**
Automatically classifies the image, reads all text (English + Hindi), extracts key fields, and translates regional language content to English.
""")
with gr.Row():
with gr.Column(scale=1):
img_input = gr.Image(type="filepath", label="Upload Image")
submit_btn = gr.Button("πŸ” Verify Document", variant="primary", size="lg")
gr.Examples(
examples=[],
inputs=img_input,
)
with gr.Column(scale=2):
with gr.Tabs():
with gr.Tab("πŸ“‹ Summary"):
annotated_out = gr.AnnotatedImage(label="OCR Overlay")
summary_out = gr.Markdown()
with gr.Tab("πŸ“Š Fields"):
fields_out = gr.Dataframe(
headers=["Field", "Value", "Confidence"],
datatype=["str", "str", "str"],
label="Extracted Fields",
)
with gr.Tab("🚩 Flags"):
flags_out = gr.Markdown()
with gr.Tab("🌐 Translation"):
translation_out = gr.Markdown()
with gr.Tab("πŸ”§ Raw JSON"):
json_out = gr.JSON(label="Full VLM Output")
with gr.Tab("πŸ“œ Pipeline Log"):
log_out = gr.Textbox(
label="Processing Log",
lines=15,
max_lines=30,
)
submit_btn.click(
fn=verify,
inputs=[img_input],
outputs=[annotated_out, summary_out, fields_out,
flags_out, translation_out, json_out, log_out],
)
return demo
demo = build_ui()
demo.queue(default_concurrency_limit=2)
demo.launch()