Spaces:
Sleeping
Sleeping
Download app.py from amarnathshenoy/image-classifier: direct link, hf CLI and curl.
- Browser
- Download file 20.4 kB
-
https://huggingface.co/spaces/amarnathshenoy/image-classifier/resolve/main/app.py
- Command line
-
hf download hf://spaces/amarnathshenoy/image-classifier/app.py
-
curl -L -o app.py https://huggingface.co/spaces/amarnathshenoy/image-classifier/resolve/main/app.py
20.4 kB
| """ | |
| MSME Document Verifier β Hugging Face Spaces Demo | |
| ================================================== | |
| Self-hosted Β· multilingual Β· zero per-doc cost | |
| Pipeline: PIL pre-processing β EasyOCR β Qwen2.5-7B-Instruct (HF Inference API) | |
| β IndicTrans2 (Helsinki-NLP fallback) β Gradio UI | |
| """ | |
| import base64 | |
| import json | |
| import logging | |
| import os | |
| import warnings | |
| from pathlib import Path | |
| from typing import Any | |
| import cv2 | |
| import numpy as np | |
| import requests | |
| from PIL import Image, ImageOps | |
| import gradio as gr | |
| import easyocr | |
| from huggingface_hub import InferenceClient | |
| import concurrent.futures | |
| warnings.filterwarnings("ignore") | |
| # ββ Logging βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| logging.basicConfig( | |
| level=logging.INFO, | |
| format="%(asctime)s %(levelname)-8s %(message)s", | |
| datefmt="%H:%M:%S", | |
| ) | |
| logger = logging.getLogger("msme_verifier") | |
| for _noisy in ("httpx", "httpcore", "urllib3", "gradio", "PIL", "easyocr"): | |
| logging.getLogger(_noisy).setLevel(logging.WARNING) | |
| import io as _io | |
| import threading as _threading | |
| class _UILogHandler(logging.Handler): | |
| def __init__(self): | |
| super().__init__() | |
| self._local = _threading.local() | |
| def start_capture(self): | |
| self._local.buf = _io.StringIO() | |
| def stop_capture(self): | |
| buf = getattr(self._local, "buf", None) | |
| if buf is None: | |
| return "" | |
| text = buf.getvalue() | |
| self._local.buf = None | |
| return text | |
| def emit(self, record): | |
| buf = getattr(self._local, "buf", None) | |
| if buf is not None: | |
| buf.write(self.format(record) + "\n") | |
| _ui_handler = _UILogHandler() | |
| _ui_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)-8s %(message)s", datefmt="%H:%M:%S")) | |
| logger.addHandler(_ui_handler) | |
| # ββ Constants βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| HF_TOKEN = os.environ.get("HF_TOKEN", "") | |
| LLM_MODEL = "Qwen/Qwen2.5-7B-Instruct" | |
| MAX_IMAGE_DIM = 1024 | |
| # English-only EasyOCR β Hindi/Indic adds ~300MB models + 5x slower on CPU | |
| # Scene heuristics + LLM handle classification without needing script-specific OCR | |
| EASYOCR_LANGS = ["en", "hi"] # Hindi + English; models download on first run (~150MB) | |
| OCR_TIMEOUT = 120 # seconds β fall back to heuristics-only if exceeded | |
| INDIC_LANG_CODES: dict[str, str] = { | |
| "hi": "hin_Deva", "ta": "tam_Taml", "te": "tel_Telu", "ml": "mal_Mlym", | |
| "bn": "ben_Beng", "kn": "kan_Knda", "gu": "guj_Gujr", "mr": "mar_Deva", | |
| "pa": "pan_Guru", "or": "ory_Orya", "as": "asm_Beng", "ur": "urd_Arab", | |
| } | |
| # ββ EasyOCR β loaded once βββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| try: | |
| logger.info("Loading EasyOCR (first run downloads ~500 MB model) β¦") | |
| _ocr_reader = easyocr.Reader(EASYOCR_LANGS, gpu=False, verbose=False) | |
| OCR_BACKEND = "easyocr" | |
| logger.info("β EasyOCR ready") | |
| except Exception as e: | |
| logger.error("β EasyOCR failed: %s", e) | |
| _ocr_reader = None | |
| OCR_BACKEND = "none" | |
| # ββ HF Inference client βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| _llm_client = InferenceClient(model=LLM_MODEL, token=HF_TOKEN) if HF_TOKEN else None | |
| SYSTEM_PROMPT = """ | |
| You are a document-analysis assistant specialising in Indian MSME paperwork. | |
| Analyse the provided OCR text and image signals, then return ONLY a valid JSON object | |
| matching the schema below β no markdown fences, no prose outside the object. | |
| SCHEMA: | |
| { | |
| "imageType": "photo|certificate|letter|document|invoice|identity|screenshot", | |
| "category": "string, snake_case, e.g. gst_certificate / storefront / admission_letter", | |
| "categoryConfidence": <float 0.0β1.0>, | |
| "subCategory": "string or null", | |
| "languages": [ | |
| {"code": "en|hi|ml|ta|...", "name": "string", "confidence": <float>, "isPrimary": true|false} | |
| ], | |
| "text": { | |
| "english": "full English text or translation of all visible text", | |
| "original": "original mixed-language text exactly as seen" | |
| }, | |
| "content": [ | |
| {"type": "field|table|signature|stamp|logo|barcode|qr", "label": "snake_case", "value": "string", "confidence": <float>} | |
| ], | |
| "scene": { | |
| "description": "1-2 sentence visual description", | |
| "objects": ["string"], | |
| "isStorefront": true|false, | |
| "isPhotoOfDocument": true|false | |
| }, | |
| "verificationSignals": { | |
| "tamperingIndicators": ["string"], | |
| "anomalies": ["string"] | |
| }, | |
| "qualityIndicators": { | |
| "meanOcrConfidence": <float 0.0β1.0>, | |
| "issues": ["string"] | |
| } | |
| } | |
| FEW-SHOT EXAMPLE 1 β Shop storefront | |
| OCR: ["SOMANY Tiles", "JANTA HARDWARE AND PAINT", "9450882490", "orientbell tiles"] | |
| OUTPUT: {"imageType":"photo","category":"storefront","categoryConfidence":0.97,...} | |
| FEW-SHOT EXAMPLE 2 β GST Certificate | |
| OCR: ["GSTIN: 27AABCS1234F1Z5", "Certificate of Registration", "Sharma Textiles Pvt Ltd"] | |
| OUTPUT: {"imageType":"certificate","category":"gst_certificate","categoryConfidence":0.99,...} | |
| Now analyse and return JSON only. | |
| """.strip() | |
| # ββ Image pre-processing ββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def preprocess(image_path: str) -> str: | |
| try: | |
| pil = Image.open(image_path) | |
| pil = ImageOps.exif_transpose(pil).convert("RGB") | |
| bgr = cv2.cvtColor(np.array(pil), cv2.COLOR_RGB2BGR) | |
| # CLAHE contrast enhancement | |
| lab = cv2.cvtColor(bgr, cv2.COLOR_BGR2LAB) | |
| l, a, b = cv2.split(lab) | |
| l = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8)).apply(l) | |
| bgr = cv2.cvtColor(cv2.merge([l, a, b]), cv2.COLOR_LAB2BGR) | |
| # Downscale | |
| h, w = bgr.shape[:2] | |
| if max(h, w) > MAX_IMAGE_DIM: | |
| scale = MAX_IMAGE_DIM / max(h, w) | |
| bgr = cv2.resize(bgr, (int(w * scale), int(h * scale)), interpolation=cv2.INTER_AREA) | |
| out = str(Path(image_path).with_stem(Path(image_path).stem + "_prep")) | |
| cv2.imwrite(out, bgr) | |
| return out | |
| except Exception as exc: | |
| logger.warning("preprocess failed: %s", exc) | |
| return image_path | |
| # ββ OCR βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def run_ocr(image_path: str) -> list[dict]: | |
| if _ocr_reader is None: | |
| return [] | |
| def _do_ocr(): | |
| return _ocr_reader.readtext(image_path) | |
| try: | |
| with concurrent.futures.ThreadPoolExecutor(max_workers=1) as ex: | |
| future = ex.submit(_do_ocr) | |
| try: | |
| results = future.result(timeout=OCR_TIMEOUT) | |
| except concurrent.futures.TimeoutError: | |
| logger.warning("EasyOCR timed out after %ds β falling back to heuristics only", OCR_TIMEOUT) | |
| return [] | |
| lines = [] | |
| for (bbox, text, conf) in results: | |
| if not text.strip() or conf < 0.3: | |
| continue | |
| xs = [p[0] for p in bbox] | |
| ys = [p[1] for p in bbox] | |
| lines.append({ | |
| "text": text, | |
| "confidence": float(conf), | |
| "bbox": [[int(min(xs)), int(min(ys))], [int(max(xs)), int(min(ys))], | |
| [int(max(xs)), int(max(ys))], [int(min(xs)), int(max(ys))]], | |
| }) | |
| logger.info("EasyOCR: %d lines detected", len(lines)) | |
| return lines | |
| except Exception as exc: | |
| logger.warning("run_ocr failed: %s", exc) | |
| return [] | |
| # ββ Scene heuristics ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def _scene_heuristics(image_path: str, ocr_lines: list[dict]) -> dict: | |
| try: | |
| pil = Image.open(image_path).convert("RGB") | |
| w, h = pil.size | |
| arr = np.array(pil.resize((256, 256), Image.LANCZOS), dtype=float) | |
| rg = arr[:, :, 0] - arr[:, :, 1] | |
| yb = 0.5 * (arr[:, :, 0] + arr[:, :, 1]) - arr[:, :, 2] | |
| colorfulness = float( | |
| np.sqrt(rg.std() ** 2 + yb.std() ** 2) + | |
| 0.3 * np.sqrt(rg.mean() ** 2 + yb.mean() ** 2) | |
| ) | |
| aspect = w / max(h, 1) | |
| is_landscape = aspect > 1.1 | |
| mpx = (w * h) / 1_000_000 | |
| density = len(ocr_lines) / max(mpx, 0.1) | |
| score = 0.0 | |
| if colorfulness > 40: | |
| score += 0.5 | |
| elif colorfulness > 20: | |
| score += 0.25 | |
| if is_landscape: | |
| score += 0.2 | |
| if density < 3: | |
| score += 0.3 | |
| if score >= 0.7: | |
| hint = "LIKELY SCENE/PHOTO (real-world shop, person, place, or object)" | |
| elif score <= 0.3: | |
| hint = "LIKELY DOCUMENT (scanned/photographed form, certificate, letter)" | |
| else: | |
| hint = "UNCERTAIN β classify from text content" | |
| return dict(w=w, h=h, colorfulness=round(colorfulness, 1), | |
| is_landscape=is_landscape, ocr_density=round(density, 1), | |
| scene_score=round(score, 2), scene_hint=hint) | |
| except Exception as exc: | |
| logger.debug("_scene_heuristics: %s", exc) | |
| return dict(w=0, h=0, colorfulness=0.0, is_landscape=False, | |
| ocr_density=0.0, scene_score=0.5, scene_hint="UNCERTAIN") | |
| # ββ Translation βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def translate_to_english(text: str, src_lang_code: str) -> str | None: | |
| """Use Helsinki-NLP/opus-mt models via HF Inference API for translation.""" | |
| if src_lang_code == "en" or src_lang_code not in INDIC_LANG_CODES: | |
| return None | |
| if not _llm_client: | |
| return None | |
| try: | |
| # Helsinki-NLP has specific indicβen models | |
| model = f"Helsinki-NLP/opus-mt-{src_lang_code}-en" | |
| client = InferenceClient(model=model, token=HF_TOKEN) | |
| result = client.translation(text, model=model) | |
| translated = result.translation_text if hasattr(result, "translation_text") else str(result) | |
| logger.info("Translation %sβen: %d chars", src_lang_code, len(translated)) | |
| return translated | |
| except Exception as exc: | |
| logger.warning("Translation failed (%s): %s", src_lang_code, exc) | |
| return None | |
| # ββ JSON repair βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def _repair_json(raw: str) -> str: | |
| s = raw.strip() | |
| depth_brace = s.count("{") - s.count("}") | |
| depth_bracket = s.count("[") - s.count("]") | |
| if s.endswith(","): | |
| s = s[:-1] | |
| s += "]" * max(0, depth_bracket) + "}" * max(0, depth_brace) | |
| return s | |
| # ββ LLM call βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def call_llm(image_path: str, ocr_lines: list[dict]) -> dict: | |
| if not _llm_client: | |
| return {"error": "no_hf_token", "detail": "HF_TOKEN secret not set in Space settings"} | |
| hints = _scene_heuristics(image_path, ocr_lines) | |
| logger.info("Scene: colorfulness=%.1f score=%.2f β %s", | |
| hints["colorfulness"], hints["scene_score"], hints["scene_hint"]) | |
| if ocr_lines: | |
| ocr_text = "\n".join(f'- "{ln["text"]}" (conf {ln["confidence"]:.2f})' for ln in ocr_lines) | |
| ocr_block = f"OCR extracted {len(ocr_lines)} text lines:\n{ocr_text}" | |
| else: | |
| ocr_block = "OCR returned no text lines." | |
| user_text = ( | |
| f"IMAGE SIGNALS\n" | |
| f"Dimensions : {hints['w']}Γ{hints['h']} px " | |
| f"({'landscape' if hints['is_landscape'] else 'portrait'})\n" | |
| f"Colorfulness: {hints['colorfulness']:.0f}/100 (documentsβ0β15, photosβ30β90)\n" | |
| f"OCR density : {hints['ocr_density']:.1f} lines/MP\n" | |
| f"Pre-class : {hints['scene_hint']}\n\n" | |
| f"{ocr_block}\n\n" | |
| "Trust the pre-classification unless OCR clearly shows certificate/GSTIN/form fields.\n" | |
| "Return ONLY the JSON schema β no markdown, no prose." | |
| ) | |
| messages = [ | |
| {"role": "system", "content": SYSTEM_PROMPT}, | |
| {"role": "user", "content": user_text}, | |
| ] | |
| logger.info("Calling %s via HF Inference API β¦", LLM_MODEL) | |
| for attempt in range(1, 3): | |
| try: | |
| response = _llm_client.chat_completion( | |
| messages=messages, | |
| max_tokens=2048, | |
| temperature=0.1 if attempt == 1 else 0.0, | |
| ) | |
| raw = response.choices[0].message.content.strip() | |
| except Exception as exc: | |
| return {"error": "llm_error", "detail": str(exc)} | |
| if raw.startswith("```"): | |
| raw = raw.split("```", 2)[1] | |
| if raw.startswith("json"): | |
| raw = raw[4:] | |
| raw = raw.rsplit("```", 1)[0].strip() | |
| try: | |
| return json.loads(raw) | |
| except json.JSONDecodeError: | |
| try: | |
| return json.loads(_repair_json(raw)) | |
| except json.JSONDecodeError: | |
| pass | |
| if attempt == 1: | |
| messages.append({"role": "assistant", "content": raw}) | |
| messages.append({"role": "user", "content": "Not valid JSON. Return ONLY the JSON object."}) | |
| else: | |
| return {"error": "parse_failed", "raw": raw[:500]} | |
| return {"error": "parse_failed", "raw": ""} | |
| # ββ UI ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def verify(image_path: str, progress=gr.Progress(track_tqdm=True)) -> tuple: | |
| _ui_handler.start_capture() | |
| logger.info("ββ Pipeline: %s", Path(image_path).name) | |
| progress(0.05, desc="[1/3] Preprocessing β CLAHE Β· resize β¦") | |
| logger.info("β [1/3] Preprocessing") | |
| prep = preprocess(image_path) | |
| progress(0.25, desc="[2/3] EasyOCR β detecting text β¦ (first run downloads models, ~60s. Subsequent runs ~15s)") | |
| logger.info("β [2/3] EasyOCR (~10β20s on CPU)") | |
| ocr_lines = run_ocr(prep) | |
| logger.info("β β %d lines", len(ocr_lines)) | |
| progress(0.55, desc="[3/3] Qwen2.5-7B β classifying via HF API β¦ (5β15 s)") | |
| logger.info("β [3/3] Qwen2.5-7B via HF Inference API") | |
| result = call_llm(prep, ocr_lines) | |
| progress(0.90, desc="Finishing β¦") | |
| # Annotations | |
| annotations = [] | |
| for ln in ocr_lines: | |
| try: | |
| xs = [p[0] for p in ln["bbox"]] | |
| ys = [p[1] for p in ln["bbox"]] | |
| annotations.append(((int(min(xs)), int(min(ys)), int(max(xs)), int(max(ys))), ln["text"][:30])) | |
| except Exception: | |
| pass | |
| annotated = (prep, annotations) | |
| if "error" in result: | |
| err = result["error"] | |
| if err == "no_hf_token": | |
| msg = "**Setup required:** Add `HF_TOKEN` in Space Settings β Repository secrets." | |
| else: | |
| msg = f"**Error:** `{err}` β {result.get('detail', '')}" | |
| pipeline_log = _ui_handler.stop_capture() | |
| return annotated, msg, [], msg, "_N/A_", result, pipeline_log | |
| # Summary card | |
| conf = float(result.get("categoryConfidence", 0.0)) | |
| indicator = "π’" if conf > 0.85 else ("π‘" if conf > 0.60 else "π΄") | |
| category = result.get("category", "unknown").replace("_", " ").title() | |
| image_type = result.get("imageType", "β") | |
| langs = result.get("languages", []) | |
| primary_langs = [lg["name"] for lg in langs if lg.get("isPrimary")] or ([langs[0]["name"]] if langs else ["β"]) | |
| scene_desc = result.get("scene", {}).get("description", "") | |
| summary = ( | |
| f"## {indicator} {category}\n" | |
| f"**Type:** {image_type} | " | |
| f"**Confidence:** {conf*100:.0f}% | " | |
| f"**Language(s):** {', '.join(primary_langs)}\n\n" | |
| f"_{scene_desc}_" | |
| ) | |
| # Fields table | |
| fields = [ | |
| [item.get("label", "").replace("_", " ").title(), | |
| item.get("value", ""), | |
| f"{item.get('confidence', 0)*100:.0f}%"] | |
| for item in result.get("content", []) | |
| ] | |
| # Flags | |
| signals = result.get("verificationSignals", {}) | |
| tampering = signals.get("tamperingIndicators", []) | |
| anomalies = signals.get("anomalies", []) | |
| flags = "" | |
| if tampering: | |
| flags += "**β Tampering indicators:**\n" + "\n".join(f"- {t}" for t in tampering) + "\n\n" | |
| if anomalies: | |
| flags += "**β Anomalies:**\n" + "\n".join(f"- {a}" for a in anomalies) | |
| if not flags: | |
| flags = "β No flags raised" | |
| # Translation | |
| non_english = [lg for lg in langs if lg.get("code") != "en" and lg.get("code") in INDIC_LANG_CODES] | |
| translation_md = "" | |
| if non_english: | |
| orig_text = result.get("text", {}).get("original", "") | |
| if orig_text: | |
| for lg in non_english[:2]: | |
| translated = translate_to_english(orig_text, lg["code"]) | |
| if translated: | |
| translation_md += f"**{lg['name']} β English:**\n> {translated}\n\n" | |
| if not translation_md: | |
| translation_md = "_No regional language text detected or translation not available._" | |
| progress(1.0, desc="Done β") | |
| pipeline_log = _ui_handler.stop_capture() | |
| return annotated, summary, fields, flags, translation_md, result, pipeline_log | |
| def build_ui() -> gr.Blocks: | |
| with gr.Blocks(title="Image Classifier & Text Reader", theme=gr.themes.Soft()) as demo: | |
| gr.Markdown(""" | |
| # π Image Classifier & Text Reader | |
| **Upload any image β storefronts, documents, invoices, certificates, photos** | |
| Automatically classifies the image, reads all text (English + Hindi), extracts key fields, and translates regional language content to English. | |
| """) | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| img_input = gr.Image(type="filepath", label="Upload Image") | |
| submit_btn = gr.Button("π Verify Document", variant="primary", size="lg") | |
| gr.Examples( | |
| examples=[], | |
| inputs=img_input, | |
| ) | |
| with gr.Column(scale=2): | |
| with gr.Tabs(): | |
| with gr.Tab("π Summary"): | |
| annotated_out = gr.AnnotatedImage(label="OCR Overlay") | |
| summary_out = gr.Markdown() | |
| with gr.Tab("π Fields"): | |
| fields_out = gr.Dataframe( | |
| headers=["Field", "Value", "Confidence"], | |
| datatype=["str", "str", "str"], | |
| label="Extracted Fields", | |
| ) | |
| with gr.Tab("π© Flags"): | |
| flags_out = gr.Markdown() | |
| with gr.Tab("π Translation"): | |
| translation_out = gr.Markdown() | |
| with gr.Tab("π§ Raw JSON"): | |
| json_out = gr.JSON(label="Full VLM Output") | |
| with gr.Tab("π Pipeline Log"): | |
| log_out = gr.Textbox( | |
| label="Processing Log", | |
| lines=15, | |
| max_lines=30, | |
| ) | |
| submit_btn.click( | |
| fn=verify, | |
| inputs=[img_input], | |
| outputs=[annotated_out, summary_out, fields_out, | |
| flags_out, translation_out, json_out, log_out], | |
| ) | |
| return demo | |
| demo = build_ui() | |
| demo.queue(default_concurrency_limit=2) | |
| demo.launch() | |