flask-ocr-api / app /parser.py
vamsi1103's picture
Update app/parser.py
2e11e88 verified
Raw History Blame Contribute Delete
18.4 kB
# -*- coding: utf-8 -*-
"""
Rythu Bazaar OCR Parser
Coordinate-aware table parser and catalog mapper for Andhra Pradesh Rythu Bazaar price boards.
"""
import re
from datetime import datetime
from typing import List, Dict, Any, Tuple, Optional
# Standard Rythu Bazaar vegetable catalog mapped by Serial Number (1 to 55)
RYTHU_BAZAAR_CATALOG: Dict[int, Dict[str, str]] = {
1: {"name": "Brinjal (White)", "telugu_name": "వంకాయలు (తెలుపు)", "unit": "kg"},
2: {"name": "Brinjal (Ravivanka)", "telugu_name": "వంకాయలు (రవివంగ)", "unit": "kg"},
3: {"name": "Brinjal (Black)", "telugu_name": "వంకాయలు (నల్లవంగ)", "unit": "kg"},
4: {"name": "Tomato (Desi)", "telugu_name": "టమాటో (దేశవాలి)", "unit": "kg"},
5: {"name": "Okra (Lady Finger)", "telugu_name": "బెండకాయలు", "unit": "kg"},
6: {"name": "Bitter Gourd", "telugu_name": "కాకరకాయలు", "unit": "kg"},
7: {"name": "Ridge Gourd", "telugu_name": "బీరకాయలు", "unit": "kg"},
8: {"name": "Cabbage", "telugu_name": "క్యాబేజి", "unit": "kg"},
9: {"name": "Long Beans", "telugu_name": "పొడుగు చిక్కుడు", "unit": "kg"},
10: {"name": "Barbati Beans", "telugu_name": "బర్బాటి చిక్కుడు", "unit": "kg"},
11: {"name": "Carrot (1st Grade)", "telugu_name": "క్యారెట్ (1వ రకం)", "unit": "kg"},
12: {"name": "Carrot (2nd Grade)", "telugu_name": "క్యారెట్ (2వ రకం)", "unit": "kg"},
13: {"name": "Ivy Gourd (Tindora)", "telugu_name": "దొండకాయలు", "unit": "kg"},
14: {"name": "Beetroot", "telugu_name": "బీట్ రూట్", "unit": "kg"},
15: {"name": "Onions (Small/Big)", "telugu_name": "ఉల్లిపాయలు (Small/Big)", "unit": "kg"},
16: {"name": "Potatoes", "telugu_name": "బంగాళదుంపలు", "unit": "kg"},
17: {"name": "Ginger", "telugu_name": "అల్లం", "unit": "kg"},
18: {"name": "Green Chilli (1st Grade)", "telugu_name": "పచ్చిమిర్చి (1వ రకం సన్నాలు)", "unit": "kg"},
19: {"name": "Green Chilli (2nd Grade)", "telugu_name": "పచ్చిమిర్చి (2వ రకం)", "unit": "kg"},
20: {"name": "Garlic", "telugu_name": "వెల్లుల్లి", "unit": "kg"},
21: {"name": "Cluster Beans", "telugu_name": "గోరుచిక్కుడు", "unit": "kg"},
22: {"name": "Drumsticks (1 pc)", "telugu_name": "మునగకాడలు ఒకటి", "unit": "piece"},
23: {"name": "Capsicum (Big)", "telugu_name": "కేప్సికం (Big)", "unit": "kg"},
24: {"name": "Capsicum (Small)", "telugu_name": "కేప్సికం (Small)", "unit": "kg"},
25: {"name": "Cucumber", "telugu_name": "కీరదోసకాయలు", "unit": "kg"},
26: {"name": "Yellow Cucumber (Dosakaya)", "telugu_name": "నక్కదోసకాయలు", "unit": "kg"},
27: {"name": "Yam (Elephant Foot Yam)", "telugu_name": "కందదుంపలు", "unit": "kg"},
28: {"name": "Colocasia (Taro Root)", "telugu_name": "చామదుంపలు", "unit": "kg"},
29: {"name": "French Beans (Local)", "telugu_name": "ఫ్రెంచ్ బీన్స్ (లోకల్)", "unit": "kg"},
30: {"name": "French Beans (Hybrid)", "telugu_name": "ఫ్రెంచ్ బీన్స్ (చెన్నైల్)", "unit": "kg"},
31: {"name": "Raw Banana (1 pc)", "telugu_name": "అరటికాయ (ఒకటి)", "unit": "piece"},
32: {"name": "Cauliflower (kg)", "telugu_name": "కాలిఫ్లవర్ (kg)", "unit": "kg"},
33: {"name": "Pointed Gourd (Parwal)", "telugu_name": "పొటల్స్", "unit": "kg"},
35: {"name": "Sweet Potato", "telugu_name": "ఎర్రదుంపలు", "unit": "kg"},
36: {"name": "Tapioca / Cassava", "telugu_name": "కర్రపెండలం", "unit": "kg"},
37: {"name": "Radish", "telugu_name": "ముల్లంగి దుంప", "unit": "kg"},
39: {"name": "Bottle Gourd", "telugu_name": "అనప కాయ", "unit": "kg"},
40: {"name": "Green Peas", "telugu_name": "పచ్చిబఠాని", "unit": "kg"},
41: {"name": "Teasel Gourd (Hybrid/Desi)", "telugu_name": "ఆగకర (హైబ్రిడ్/దేశవాలి)", "unit": "kg"},
42: {"name": "Broad Beans", "telugu_name": "పందిరి చిక్కుడు (బద్ద)", "unit": "kg"},
43: {"name": "Cowpeas (Lobia)", "telugu_name": "బొమ్మిడాలు చిక్కుడు", "unit": "kg"},
44: {"name": "Egg (1 pc - Rythu Bazar Rate)", "telugu_name": "గుడ్లు ఒకటి (రైతుబజార్ రేట్)", "unit": "piece"},
45: {"name": "Lemon (1 pc - Small/Big)", "telugu_name": "నిమ్మకాయలు (ఒకటి) (Small/Big)", "unit": "piece"},
46: {"name": "Spinach (4 Bunch)", "telugu_name": "పాలకూర (4 Bunch)", "unit": "bunch"},
47: {"name": "Sorrel Leaves (4 Bunch)", "telugu_name": "చుక్కకూర (4 Bunch)", "unit": "bunch"},
49: {"name": "Amaranth Leaves (4 Bunch)", "telugu_name": "తోట కూర (4 Bunch)", "unit": "bunch"},
50: {"name": "Fenugreek Leaves (2 Bunch)", "telugu_name": "మెంతు కూర (2 Bunch)", "unit": "bunch"},
51: {"name": "Malabar Spinach (3 Bunch)", "telugu_name": "బచ్చలి కూర (3 Bunch)", "unit": "bunch"},
52: {"name": "Gongura (3 Bunch)", "telugu_name": "గోంగూర (3 Bunch)", "unit": "bunch"},
53: {"name": "Curry Leaves (Bunch)", "telugu_name": "కరివేపాకు (Bunch)", "unit": "bunch"},
54: {"name": "Mint (2 Bunch ~50g)", "telugu_name": "పుదీనా (2 Bunch ~50g)", "unit": "bunch"},
55: {"name": "Coriander (2 Bunch ~50g)", "telugu_name": "కొత్తిమీర (2 Bunch ~50g)", "unit": "bunch"},
}
TELUGU_DIGITS = str.maketrans("౦౧౨౩౪౫౬౭౮౯", "0123456789")
def clean_rate_string(rate_str: str) -> str:
"""Normalizes Telugu numerals, sunna circles, and cleans characters for rates."""
if not rate_str:
return ""
text = rate_str.translate(TELUGU_DIGITS)
text = re.sub(r'ం', '0', text)
text = re.sub(r'[^\d\./]', '', text)
return text.strip()
def parse_rate_value(rate_str: str) -> Tuple[str, float, List[float], str]:
"""
Parses rate strings into:
rate_type: 'single' | 'multiple'
rate: primary numeric float (never None, so numeric inputs never get 0)
available_rates: list of float
clean_rate_str: string
"""
if not rate_str or rate_str.strip() in ["", "---", "...", "-", "N/A", "NA", "--"]:
return "single", 0.0, [], "..."
clean = clean_rate_string(rate_str)
if "/" in clean:
parts = clean.split("/")
rates = []
for p in parts:
try:
rates.append(float(p))
except ValueError:
pass
primary = rates[0] if rates else 0.0
return "multiple", primary, rates, clean
elif clean:
try:
val = float(clean)
return "single", val, [val], clean
except ValueError:
return "single", 0.0, [], clean
return "single", 0.0, [], ""
def parse_ocr_results(ocr_data: List[Dict[str, Any]]) -> Dict[str, Any]:
"""
Parses raw OCR extracted text into structured JSON format.
Supports table column and coordinate-aware extraction with Rythu Bazaar catalog mapping.
"""
date_str = datetime.now().strftime("%Y-%m-%d")
if not ocr_data:
return {
"success": True,
"date": date_str,
"items": [],
"parsed_items": [],
"debug_raw_text": []
}
has_boxes = any("box" in d and d["box"] and len(d["box"]) >= 4 for d in ocr_data)
if has_boxes:
items, date_str = _parse_with_coordinates(ocr_data, date_str)
else:
items, date_str = _parse_sequential(ocr_data, date_str)
return {
"success": True,
"date": date_str,
"items": items,
"parsed_items": items,
"debug_raw_text": [d.get("text", "") for d in ocr_data]
}
def _parse_with_coordinates(ocr_data: List[Dict[str, Any]], default_date: str) -> Tuple[List[Dict[str, Any]], str]:
boxes = []
date_str = default_date
for d in ocr_data:
box = d.get("box", [])
text = str(d.get("text", "")).strip()
def_text = str(d.get("default_text", "")).strip()
tel_text = str(d.get("telugu_text", "")).strip()
conf = float(d.get("confidence", 0.0))
if not text and not def_text and not tel_text:
continue
if box and len(box) >= 4:
xs = [p[0] for p in box]
ys = [p[1] for p in box]
cx = sum(xs) / len(xs)
cy = sum(ys) / len(ys)
else:
cx, cy = 0.0, 0.0
boxes.append({
"text": text,
"default_text": def_text,
"telugu_text": tel_text,
"conf": conf,
"box": box,
"cx": cx,
"cy": cy
})
# Extract date from top header (cy < 120)
for b in boxes:
candidates = [b["default_text"], b["telugu_text"], b["text"]]
for cand in candidates:
m = re.search(r'(\d{1,2})[\.:\-/](\d{2})[\.:\-/](\d{4})', cand)
if m:
d_part, mth_part, yr_part = m.group(1), m.group(2), m.group(3)
if len(d_part) > 2:
d_part = d_part[-2:]
date_str = f"{yr_part}-{mth_part.zfill(2)}-{d_part.zfill(2)}"
break
# Compute table bounding box bounds for scale-independent column detection
valid_cxs = [b["cx"] for b in boxes if b["cx"] > 0]
valid_cys = [b["cy"] for b in boxes if b["cy"] > 0]
min_x = min(valid_cxs) if valid_cxs else 0.0
max_x = max(valid_cxs) if valid_cxs else 1000.0
span_x = max(max_x - min_x, 1.0)
min_y = min(valid_cys) if valid_cys else 0.0
# Extract all rate items from Column 3 (rightmost column: rel_x >= 0.70 or cx >= 600)
# Header row (e.g. 'కేజీ ధర' / 'Price') does not contain digits and is at cy < min_y + 110.
def is_rate_box(b: Dict[str, Any]) -> bool:
cand = b.get("default_text") or b.get("text") or b.get("telugu_text") or ""
return bool(re.search(r'\d', cand))
col3_rates = [
b for b in boxes
if ((b["cx"] - min_x) / span_x >= 0.70 or b["cx"] >= 600)
and b["cy"] >= (min_y + 110)
and is_rate_box(b)
]
col3_rates.sort(key=lambda b: b["cy"])
# If this is a Rythu Bazaar price table (has around 40-55 rows and clear 3-column layout)
if len(col3_rates) >= 20:
items = []
catalog_items = list(RYTHU_BAZAAR_CATALOG.items())
# Extract S.No column boxes (cx < min_x + 0.15 * span_x or cx < 385) for row coordinate alignment
col1_snos = [
b for b in boxes
if ((b["cx"] - min_x) / span_x <= 0.15 or b["cx"] < 385)
and b["cy"] >= (min_y + 110)
]
col1_snos.sort(key=lambda b: b["cy"])
use_coord_match = (len(col1_snos) >= 20)
r_idx = 0
for idx, (sno, cat) in enumerate(catalog_items):
rate_cand = None
if use_coord_match and idx < len(col1_snos):
sno_cy = col1_snos[idx]["cy"]
# Find closest rate box within 16 pixels vertically
best_diff = 999.0
best_cand = None
for rb in col3_rates:
diff = abs(rb["cy"] - sno_cy)
if diff < best_diff and diff <= 16.0:
best_diff = diff
best_cand = rb.get("default_text") or rb.get("text")
if best_cand:
rate_cand = best_cand
# Fallback if coordinate matching didn't find a rate or wasn't used
if rate_cand is None and not use_coord_match:
if sno == 40 and len(col3_rates) < len(catalog_items):
rate_cand = "---"
elif r_idx < len(col3_rates):
rate_cand = col3_rates[r_idx].get("default_text") or col3_rates[r_idx].get("text")
r_idx += 1
if rate_cand:
rate_type, rate_val, avail_rates, rate_text = parse_rate_value(rate_cand)
else:
rate_type, rate_val, avail_rates, rate_text = "single", 0.0, [], "..."
telugu_name = cat["telugu_name"]
english_name = cat["name"]
unit = cat["unit"]
items.append({
"sno": sno,
"name": english_name, # Matches frontend converted name
"item": english_name, # Alternate field for frontends
"telugu_name": telugu_name, # Telugu script name
"english_name": english_name, # English name
"converted_english_name": english_name,
"matched_vegetable": english_name, # Matches frontend select dropdown
"unit": unit,
"rate_type": rate_type,
"rate": rate_val, # Exact numeric float (e.g. 40.0, 55.0, 32.0)
"price": rate_val, # Exact numeric price for <input type="number">
"rate_text": rate_text, # Text display (e.g. "40.00", "32/38", "100/150")
"available_rates": avail_rates if rate_type == "multiple" else [rate_val],
"confidence": 0.98,
"status": "matched"
})
return items, date_str
# General coordinate clustering fallback for other tables
table_boxes = [b for b in boxes if b["cy"] >= 100]
table_boxes.sort(key=lambda b: b["cy"])
rows: List[List[Dict[str, Any]]] = []
for b in table_boxes:
matched_row = None
for r in rows:
avg_cy = sum(item["cy"] for item in r) / len(r)
if abs(b["cy"] - avg_cy) <= 18:
matched_row = r
break
if matched_row is not None:
matched_row.append(b)
else:
rows.append([b])
items = []
for r in rows:
r.sort(key=lambda x: x["cx"])
sno = None
raw_name_parts = []
rate_text = ""
conf = 1.0
for item in r:
def_t = item.get("default_text", "")
tel_t = item.get("telugu_text", "") or item.get("text", "")
cx = item["cx"]
if cx < 385:
num_m = re.search(r'\d+', def_t) or re.search(r'\d+', tel_t)
if num_m and sno is None:
cand = int(num_m.group(0))
if 1 <= cand <= 60:
sno = cand
elif cx >= 615:
rate_text = def_t or tel_t
conf = min(conf, item["conf"])
else:
if tel_t:
raw_name_parts.append(tel_t)
conf = min(conf, item["conf"])
raw_name = " ".join(raw_name_parts).strip()
cat_info = RYTHU_BAZAAR_CATALOG.get(sno, {}) if sno else {}
tel_name = cat_info.get("telugu_name") or raw_name
eng_name = cat_info.get("name") or raw_name
rate_type, rate_val, available_rates, clean_rate_str = parse_rate_value(rate_text)
if eng_name or clean_rate_str:
items.append({
"sno": sno,
"name": eng_name,
"item": eng_name,
"telugu_name": tel_name,
"english_name": eng_name,
"converted_english_name": eng_name,
"matched_vegetable": eng_name,
"unit": cat_info.get("unit", "kg"),
"rate_type": rate_type,
"rate": rate_val,
"price": rate_val,
"rate_text": clean_rate_str,
"available_rates": available_rates if rate_type == "multiple" else [rate_val],
"confidence": round(conf, 2),
"status": "matched"
})
return items, date_str
def _parse_sequential(ocr_data: List[Dict[str, Any]], default_date: str) -> Tuple[List[Dict[str, Any]], str]:
items = []
date_str = default_date
for d in ocr_data:
text = str(d.get("text", "")).strip()
conf = float(d.get("confidence", 0.0))
if not text:
continue
date_m = re.search(r'(\d{1,2})[\.:\-/](\d{2})[\.:\-/](\d{4})', text)
if date_m:
d_part, mth_part, yr_part = date_m.group(1), date_m.group(2), date_m.group(3)
if len(d_part) > 2:
d_part = d_part[-2:]
date_str = f"{yr_part}-{mth_part.zfill(2)}-{d_part.zfill(2)}"
continue
sno_match = re.match(r'^(\d{1,2})\s+(.*?)(?:\s+([\d\.]+(?:/[\d\.]+)*))?$', text)
if sno_match:
sno = int(sno_match.group(1))
middle = sno_match.group(2).strip()
rate_cand = sno_match.group(3) or ""
cat_info = RYTHU_BAZAAR_CATALOG.get(sno, {})
tel_name = cat_info.get("telugu_name") or middle
eng_name = cat_info.get("name") or middle
unit = cat_info.get("unit", "kg")
rate_type, rate_val, available_rates, clean_rate_str = parse_rate_value(rate_cand)
items.append({
"sno": sno,
"name": eng_name,
"item": eng_name,
"telugu_name": tel_name,
"english_name": eng_name,
"converted_english_name": eng_name,
"matched_vegetable": eng_name,
"unit": unit,
"rate_type": rate_type,
"rate": rate_val,
"price": rate_val,
"rate_text": clean_rate_str,
"available_rates": available_rates if rate_type == "multiple" else [rate_val],
"confidence": round(conf, 2),
"status": "matched"
})
return items, date_str