import easyocr import numpy as np import re import difflib from PIL import Image from theme_engine import DEFAULT_THEME # Initialize easyocr and spellchecker lazily _reader = None def get_ocr_reader(): global _reader if _reader is None: _reader = easyocr.Reader(['en']) return _reader _spell = None def get_spellchecker(): global _spell if _spell is None: try: from spellchecker import SpellChecker _spell = SpellChecker() # Add some common UI terms so they don't get accidentally "corrected" _spell.word_frequency.load_words(['textarea', 'checkbox', 'radio', 'dropdown', 'login', 'signup', 'submit']) except ImportError: print("pyspellchecker not installed. Skipping NLP correction.") _spell = False return _spell # Comprehensive UI words dictionary for aggressive fuzzy matching against OCR hallucinations UI_WORDS = [ # Core UI "GENDER", "MALE", "FEMALE", "DESCRIPTION", "SUBMIT", "ENTER NAME", "HI GOOD MORNING", "BUTTON", "CHECKBOX", "RADIO", "IMAGE", "TEXT", "INPUT", "LABEL", "SELECT", # Common Form Fields "DEPARTMENT", "ARE YOU DISABLED", "CAPTCHA", "FIRST NAME", "LAST NAME", "EMAIL", "PASSWORD", "PHONE", "ADDRESS", "CITY", "STATE", "ZIP CODE", "COUNTRY", "DATE OF BIRTH", "AGE", "COMPANY", "TITLE", "ROLE", "WEBSITE", "MESSAGE", "COMMENTS", # Common Actions & Toggles "TERMS", "CONDITIONS", "AGREE", "ACCEPT", "DECLINE", "YES", "NO", "UPLOAD", "DOWNLOAD", "SAVE", "CANCEL", "NEXT", "PREVIOUS", "BACK", "SEARCH", "FILTER", "SORT", # Nav & Auth "MENU", "HOME", "ABOUT", "CONTACT", "LOGIN", "LOGOUT", "REGISTER", "SIGN UP", "SIGN IN", "FORGOT PASSWORD", "RESET", "SEND", "ADD", "REMOVE", "DELETE", "EDIT", "UPDATE", "PROFILE", "SETTINGS", "HELP", "SUPPORT", "FAQ" ] class BoundingBox: def __init__(self, cls_name, x, y, w, h): self.cls_name = cls_name self.x = x self.y = y self.w = w self.h = h self.x1 = x - w / 2 self.y1 = y - h / 2 self.x2 = x + w / 2 self.y2 = y + h / 2 self.children = [] self.parent = None self.text_content = "" def contains(self, other): # A box contains another if it covers at least 80% of the other's area # This prevents large slightly-skewed boxes from losing their children ix1 = max(self.x1, other.x1) iy1 = max(self.y1, other.y1) ix2 = min(self.x2, other.x2) iy2 = min(self.y2, other.y2) if ix2 > ix1 and iy2 > iy1: intersection_area = (ix2 - ix1) * (iy2 - iy1) if intersection_area / other.area() > 0.7: return True return False def area(self): return self.w * self.h def __repr__(self): return f"{self.cls_name}({self.x}, {self.y}, {self.w}, {self.h})" def build_dom_tree(boxes): boxes.sort(key=lambda b: b.area(), reverse=True) top_level = [] for i, box in enumerate(boxes): parent_found = False # Search from smallest possible parents to largest for j in range(len(boxes) - 1, -1, -1): if i == j: continue potential_parent = boxes[j] if potential_parent.area() > box.area() and potential_parent.contains(box): potential_parent.children.append(box) box.parent = potential_parent parent_found = True break if not parent_found: top_level.append(box) return optimize_dom_tree(top_level) def optimize_dom_tree(nodes): optimized = [] for node in nodes: node.children = optimize_dom_tree(node.children) # Collapse redundant nested containers (container -> exactly 1 container child) if node.cls_name.lower() == "container" and len(node.children) == 1 and node.children[0].cls_name.lower() == "container": optimized.append(node.children[0]) else: optimized.append(node) return optimized def clean_ocr_text(text): # Remove leading fake circles like "0", "o", "O", "[]" from radio/checkbox text text = text.strip() return re.sub(r'^([oO0\[\]\(\)]\s*)+', '', text) def fuzzy_correct(text): clean_text = re.sub(r'[^A-Z\s]', '', text.upper()).strip() if not clean_text: return text matches = difflib.get_close_matches(clean_text, UI_WORDS, n=1, cutoff=0.5) if matches: return matches[0] # Fallback to spellchecker spell = get_spellchecker() if spell: words = text.split() corrected_words = [] for word in words: clean_word = re.sub(r'[^a-zA-Z]', '', word) if clean_word and clean_word.lower() not in spell: correction = spell.correction(clean_word) if correction: if word.isupper(): corrected_words.append(correction.upper()) elif word.istitle(): corrected_words.append(correction.title()) else: corrected_words.append(correction) else: corrected_words.append(word) else: corrected_words.append(word) return " ".join(corrected_words) return text def group_into_rows(nodes): if not nodes: return [] # Sort nodes by top edge (y1) to process them top-to-bottom sorted_nodes = sorted(nodes, key=lambda n: n.y1) rows = [] current_row = [] for node in sorted_nodes: if not current_row: current_row.append(node) else: row_y1 = min(n.y1 for n in current_row) row_y2 = max(n.y2 for n in current_row) overlap = max(0, min(node.y2, row_y2) - max(node.y1, row_y1)) min_h = min(node.h, row_y2 - row_y1) # If they overlap by at least 30% of the smaller height if min_h > 0 and (overlap / min_h) > 0.3: current_row.append(node) else: rows.append(current_row) current_row = [node] if current_row: rows.append(current_row) # Sort each row horizontally for i in range(len(rows)): rows[i] = sorted(rows[i], key=lambda n: n.x) return rows import html as html_lib def generate_html_node(node, img_width, img_height, theme=None, tw_width="w-full", inline_style=""): if theme is None: theme = DEFAULT_THEME c = theme["colors"] br = theme.get("border_radius", "6px") cls_normalized = node.cls_name.lower() text_upper = node.text_content.upper().strip() if node.text_content else "" is_merged = getattr(node, 'is_merged', False) text_first = getattr(node, 'text_first', False) # 1. Radio/Checkbox Heuristics if cls_normalized in ["radio", "checkbox"] and not is_merged: # If it's a structural label that YOLO hallucinated as a radio if any(word in text_upper for word in ["GENDER", "DEPARTMENT", "DISABLED"]): cls_normalized = "label" # If the radio/checkbox text is literally just a checkmark 'V', make it a checkbox elif text_upper == "V": cls_normalized = "checkbox" # If it has text but no circle/box indicator, and isn't empty (which means it's the actual drawn box) elif text_upper and not re.search(r'([oO0\[\]\(\)vV])', text_upper): # We assume OCR picked up a normal label, so we convert it. # But we leave EMPTY text alone, because those are the actual drawn circles/boxes! cls_normalized = "label" children_html = "" if node.children: rows = group_into_rows(node.children) for row in rows: if len(row) > 1: children_html += '
\n' current_x_pct = 0 for child in row: child_x_pct = (child.x1 / node.w) * 100 gap_pct = child_x_pct - current_x_pct if gap_pct > 1: children_html += f'
\n' child_w_pct = (child.w / node.w) * 100 children_html += generate_html_node(child, node.w, node.h, theme=theme, tw_width="w-full", inline_style=f"width: {child_w_pct}%; flex-shrink: 0;") current_x_pct = child_x_pct + child_w_pct children_html += '
\n' else: child = row[0] child_w_pct = (child.w / node.w) * 100 child_x_pct = (child.x1 / node.w) * 100 children_html += f'
\n' if child_x_pct > 1: children_html += f'
\n' children_html += generate_html_node(child, node.w, node.h, theme=theme, tw_width="w-full", inline_style=f"width: {child_w_pct}%; flex-shrink: 0;") children_html += '
\n' safe_text = html_lib.escape(node.text_content) if node.text_content else "" if cls_normalized == "container": return f'
{children_html}
\n' elif cls_normalized == "radio": text = clean_ocr_text(safe_text) if safe_text else "" radio_style = f"accent-color: {c['accent']};" if text_first: label_html = f'' if text else "" return f'
{label_html}
\n' else: label_html = f'' if text else "" return f'
{label_html}
\n' elif cls_normalized == "checkbox": text = clean_ocr_text(safe_text) if safe_text else "" cb_style = f"accent-color: {c['accent']};" if text_first: label_html = f'' if text else "" return f'
{label_html}
\n' else: label_html = f'' if text else "" return f'
{label_html}
\n' elif cls_normalized == "label": text = safe_text if safe_text else "Label" return f'\n' elif cls_normalized == "toggle": return f'''
\n''' elif "button" in cls_normalized: text = safe_text if safe_text else "Button" if "SELECT" in text_upper or text_upper.endswith(" V"): display_text = text.replace(" v", "").replace(" V", "").strip() if not display_text: display_text = "Select..." return f'''
\n''' else: return f'\n' elif cls_normalized == "text": text = safe_text if safe_text else "Text" height_pct = int((node.h / img_height) * 100) if height_pct > 10: return f'

{text}

\n' elif height_pct > 5: return f'

{text}

\n' else: return f'

{text}

\n' elif cls_normalized == "header" or cls_normalized == "heading": text = safe_text if safe_text else "Header Title" return f'

{text}

\n' elif cls_normalized == "textarea": return f'\n' elif cls_normalized == "input": text = safe_text if safe_text else "Input field..." return f'\n' elif cls_normalized == "image" or cls_normalized == "icon": return f'
Image Area
\n' else: content = children_html if children_html else node.cls_name return f'
{content}
\n' def assign_ocr_to_boxes(img_pil, boxes): try: img_np = np.array(img_pil) reader = get_ocr_reader() ocr_results = reader.readtext(img_np) for box in boxes: normalized_cls = box.cls_name.lower() if normalized_cls not in ["text", "header", "heading", "button", "input", "radio", "checkbox", "label"]: continue box_texts = [] for ocr_box, text, conf in ocr_results: # ocr_box is [[x1, y1], [x2, y1], [x2, y2], [x1, y2]] ox1 = min(p[0] for p in ocr_box) ox2 = max(p[0] for p in ocr_box) oy1 = min(p[1] for p in ocr_box) oy2 = max(p[1] for p in ocr_box) # Check for intersection between OCR box and YOLO box overlap_x = max(0, min(box.x2, ox2) - max(box.x1, ox1)) overlap_y = max(0, min(box.y2, oy2) - max(box.y1, oy1)) if overlap_x > 0 and overlap_y > 0: intersection_area = overlap_x * overlap_y ocr_box_area = (ox2 - ox1) * (oy2 - oy1) # The OCR text box must be at least 40% inside the YOLO box if intersection_area / ocr_box_area > 0.4: box_texts.append((ox1, oy1, text)) if box_texts: # Sort texts by roughly Y, then X for left-to-right reading box_texts.sort(key=lambda t: (t[1] // 20, t[0])) raw_text = " ".join([t[2] for t in box_texts]) box.text_content = fuzzy_correct(raw_text) except Exception as e: print(f"Global OCR/NLP Error: {e}") def parse_yolo_results(results, img_pil, img_width, img_height, class_names, theme=None): if theme is None: theme = DEFAULT_THEME c = theme["colors"] br = theme.get("border_radius", "6px") font = theme.get("font_family", "Inter, system-ui, sans-serif") boxes = [] for r in results: for box in r.boxes: x, y, w, h = box.xywh[0].tolist() cls_idx = int(box.cls[0].item()) cls_name = class_names[cls_idx] b = BoundingBox(cls_name, x, y, w, h) boxes.append(b) assign_ocr_to_boxes(img_pil, boxes) top_level_nodes = build_dom_tree(boxes) html = [f'
'] html.append(f'
') rows = group_into_rows(top_level_nodes) # Spatial Element Merging (Group isolated text with adjacent drawn radio/checkbox circles) merged_rows = [] for row in rows: merged_row = [] i = 0 while i < len(row): node = row[i] # Case 1: Circle (Radio/Checkbox) followed by Text (e.g. [O] MALE) if node.cls_name.lower() in ["radio", "checkbox", "icon", "icon-button", "icon-label"] and not getattr(node, 'is_merged', False): if i + 1 < len(row): next_node = row[i+1] if next_node.cls_name.lower() in ["label", "text", "radio", "checkbox"]: dist = next_node.x1 - node.x2 # Accept any overlap (negative distance) or gap up to 30% of screen if dist < (img_width * 0.30): node.text_content = next_node.text_content node.w = max(node.x2, next_node.x2) - node.x1 node.x2 = max(node.x2, next_node.x2) node.x = (node.x1 + node.x2) / 2 if node.cls_name.lower() not in ["radio", "checkbox"]: node.cls_name = "radio" # Force to radio if it was detected as an icon node.is_merged = True merged_row.append(node) i += 2 continue # Case 2: Text followed by Circle/Box (e.g. ARE YOU DISABLED [V]) if node.cls_name.lower() in ["label", "text", "radio"] and not getattr(node, 'is_merged', False): if i + 1 < len(row): next_node = row[i+1] if next_node.cls_name.lower() in ["checkbox", "radio", "icon", "icon-button", "icon-label"]: dist = next_node.x1 - node.x2 if dist < (img_width * 0.30): original_text = next_node.text_content.upper().strip() if next_node.text_content else "" next_node.text_content = node.text_content next_node.w = max(node.x2, next_node.x2) - node.x1 next_node.x1 = node.x1 next_node.x = (next_node.x1 + next_node.x2) / 2 # The user requested that only the circles are radios, the trailing box should be a checkbox next_node.cls_name = "checkbox" next_node.is_merged = True next_node.text_first = True merged_row.append(next_node) i += 2 continue merged_row.append(node) i += 1 merged_rows.append(merged_row) rows = merged_rows for row in rows: if len(row) > 1: html.append('
') current_x_pct = 0 for node in row: node_x_pct = (node.x1 / img_width) * 100 gap_pct = node_x_pct - current_x_pct if gap_pct > 1: html.append(f'
') node_w_pct = (node.w / img_width) * 100 node_html = generate_html_node(node, img_width, img_height, theme=theme, tw_width="w-full", inline_style=f"width: {node_w_pct}%; flex-shrink: 0;") html.append(f" {node_html}") current_x_pct = node_x_pct + node_w_pct html.append('
') else: node = row[0] node_w_pct = (node.w / img_width) * 100 node_x_pct = (node.x1 / img_width) * 100 html.append('
') if node_x_pct > 1: html.append(f'
') node_html = generate_html_node(node, img_width, img_height, theme=theme, tw_width="w-full", inline_style=f"width: {node_w_pct}%; flex-shrink: 0;") html.append(f" {node_html}") html.append('
') html.append('
') html.append('
') return "\n".join(html)