sketch-to-ui / parser.py
CyborgAB's picture
Upload folder using huggingface_hub
d420bb2 verified
Raw History Blame Contribute Delete
23.1 kB
import easyocr
import numpy as np
import re
import difflib
from PIL import Image
from theme_engine import DEFAULT_THEME
# Initialize easyocr and spellchecker lazily
_reader = None
def get_ocr_reader():
global _reader
if _reader is None:
_reader = easyocr.Reader(['en'])
return _reader
_spell = None
def get_spellchecker():
global _spell
if _spell is None:
try:
from spellchecker import SpellChecker
_spell = SpellChecker()
# Add some common UI terms so they don't get accidentally "corrected"
_spell.word_frequency.load_words(['textarea', 'checkbox', 'radio', 'dropdown', 'login', 'signup', 'submit'])
except ImportError:
print("pyspellchecker not installed. Skipping NLP correction.")
_spell = False
return _spell
# Comprehensive UI words dictionary for aggressive fuzzy matching against OCR hallucinations
UI_WORDS = [
# Core UI
"GENDER", "MALE", "FEMALE", "DESCRIPTION", "SUBMIT",
"ENTER NAME", "HI GOOD MORNING", "BUTTON", "CHECKBOX",
"RADIO", "IMAGE", "TEXT", "INPUT", "LABEL", "SELECT",
# Common Form Fields
"DEPARTMENT", "ARE YOU DISABLED", "CAPTCHA", "FIRST NAME", "LAST NAME",
"EMAIL", "PASSWORD", "PHONE", "ADDRESS", "CITY", "STATE",
"ZIP CODE", "COUNTRY", "DATE OF BIRTH", "AGE", "COMPANY",
"TITLE", "ROLE", "WEBSITE", "MESSAGE", "COMMENTS",
# Common Actions & Toggles
"TERMS", "CONDITIONS", "AGREE", "ACCEPT", "DECLINE",
"YES", "NO", "UPLOAD", "DOWNLOAD", "SAVE", "CANCEL",
"NEXT", "PREVIOUS", "BACK", "SEARCH", "FILTER", "SORT",
# Nav & Auth
"MENU", "HOME", "ABOUT", "CONTACT", "LOGIN", "LOGOUT",
"REGISTER", "SIGN UP", "SIGN IN", "FORGOT PASSWORD",
"RESET", "SEND", "ADD", "REMOVE", "DELETE", "EDIT",
"UPDATE", "PROFILE", "SETTINGS", "HELP", "SUPPORT", "FAQ"
]
class BoundingBox:
def __init__(self, cls_name, x, y, w, h):
self.cls_name = cls_name
self.x = x
self.y = y
self.w = w
self.h = h
self.x1 = x - w / 2
self.y1 = y - h / 2
self.x2 = x + w / 2
self.y2 = y + h / 2
self.children = []
self.parent = None
self.text_content = ""
def contains(self, other):
# A box contains another if it covers at least 80% of the other's area
# This prevents large slightly-skewed boxes from losing their children
ix1 = max(self.x1, other.x1)
iy1 = max(self.y1, other.y1)
ix2 = min(self.x2, other.x2)
iy2 = min(self.y2, other.y2)
if ix2 > ix1 and iy2 > iy1:
intersection_area = (ix2 - ix1) * (iy2 - iy1)
if intersection_area / other.area() > 0.7:
return True
return False
def area(self):
return self.w * self.h
def __repr__(self):
return f"{self.cls_name}({self.x}, {self.y}, {self.w}, {self.h})"
def build_dom_tree(boxes):
boxes.sort(key=lambda b: b.area(), reverse=True)
top_level = []
for i, box in enumerate(boxes):
parent_found = False
# Search from smallest possible parents to largest
for j in range(len(boxes) - 1, -1, -1):
if i == j: continue
potential_parent = boxes[j]
if potential_parent.area() > box.area() and potential_parent.contains(box):
potential_parent.children.append(box)
box.parent = potential_parent
parent_found = True
break
if not parent_found:
top_level.append(box)
return optimize_dom_tree(top_level)
def optimize_dom_tree(nodes):
optimized = []
for node in nodes:
node.children = optimize_dom_tree(node.children)
# Collapse redundant nested containers (container -> exactly 1 container child)
if node.cls_name.lower() == "container" and len(node.children) == 1 and node.children[0].cls_name.lower() == "container":
optimized.append(node.children[0])
else:
optimized.append(node)
return optimized
def clean_ocr_text(text):
# Remove leading fake circles like "0", "o", "O", "[]" from radio/checkbox text
text = text.strip()
return re.sub(r'^([oO0\[\]\(\)]\s*)+', '', text)
def fuzzy_correct(text):
clean_text = re.sub(r'[^A-Z\s]', '', text.upper()).strip()
if not clean_text:
return text
matches = difflib.get_close_matches(clean_text, UI_WORDS, n=1, cutoff=0.5)
if matches:
return matches[0]
# Fallback to spellchecker
spell = get_spellchecker()
if spell:
words = text.split()
corrected_words = []
for word in words:
clean_word = re.sub(r'[^a-zA-Z]', '', word)
if clean_word and clean_word.lower() not in spell:
correction = spell.correction(clean_word)
if correction:
if word.isupper():
corrected_words.append(correction.upper())
elif word.istitle():
corrected_words.append(correction.title())
else:
corrected_words.append(correction)
else:
corrected_words.append(word)
else:
corrected_words.append(word)
return " ".join(corrected_words)
return text
def group_into_rows(nodes):
if not nodes:
return []
# Sort nodes by top edge (y1) to process them top-to-bottom
sorted_nodes = sorted(nodes, key=lambda n: n.y1)
rows = []
current_row = []
for node in sorted_nodes:
if not current_row:
current_row.append(node)
else:
row_y1 = min(n.y1 for n in current_row)
row_y2 = max(n.y2 for n in current_row)
overlap = max(0, min(node.y2, row_y2) - max(node.y1, row_y1))
min_h = min(node.h, row_y2 - row_y1)
# If they overlap by at least 30% of the smaller height
if min_h > 0 and (overlap / min_h) > 0.3:
current_row.append(node)
else:
rows.append(current_row)
current_row = [node]
if current_row:
rows.append(current_row)
# Sort each row horizontally
for i in range(len(rows)):
rows[i] = sorted(rows[i], key=lambda n: n.x)
return rows
import html as html_lib
def generate_html_node(node, img_width, img_height, theme=None, tw_width="w-full", inline_style=""):
if theme is None:
theme = DEFAULT_THEME
c = theme["colors"]
br = theme.get("border_radius", "6px")
cls_normalized = node.cls_name.lower()
text_upper = node.text_content.upper().strip() if node.text_content else ""
is_merged = getattr(node, 'is_merged', False)
text_first = getattr(node, 'text_first', False)
# 1. Radio/Checkbox Heuristics
if cls_normalized in ["radio", "checkbox"] and not is_merged:
# If it's a structural label that YOLO hallucinated as a radio
if any(word in text_upper for word in ["GENDER", "DEPARTMENT", "DISABLED"]):
cls_normalized = "label"
# If the radio/checkbox text is literally just a checkmark 'V', make it a checkbox
elif text_upper == "V":
cls_normalized = "checkbox"
# If it has text but no circle/box indicator, and isn't empty (which means it's the actual drawn box)
elif text_upper and not re.search(r'([oO0\[\]\(\)vV])', text_upper):
# We assume OCR picked up a normal label, so we convert it.
# But we leave EMPTY text alone, because those are the actual drawn circles/boxes!
cls_normalized = "label"
children_html = ""
if node.children:
rows = group_into_rows(node.children)
for row in rows:
if len(row) > 1:
children_html += '<div class="flex flex-row items-center w-full mb-6">\n'
current_x_pct = 0
for child in row:
child_x_pct = (child.x1 / node.w) * 100
gap_pct = child_x_pct - current_x_pct
if gap_pct > 1:
children_html += f'<div style="width: {gap_pct}%; flex-shrink: 0;"></div>\n'
child_w_pct = (child.w / node.w) * 100
children_html += generate_html_node(child, node.w, node.h, theme=theme, tw_width="w-full", inline_style=f"width: {child_w_pct}%; flex-shrink: 0;")
current_x_pct = child_x_pct + child_w_pct
children_html += '</div>\n'
else:
child = row[0]
child_w_pct = (child.w / node.w) * 100
child_x_pct = (child.x1 / node.w) * 100
children_html += f'<div class="flex flex-row items-center w-full mb-6">\n'
if child_x_pct > 1:
children_html += f'<div style="width: {child_x_pct}%; flex-shrink: 0;"></div>\n'
children_html += generate_html_node(child, node.w, node.h, theme=theme, tw_width="w-full", inline_style=f"width: {child_w_pct}%; flex-shrink: 0;")
children_html += '</div>\n'
safe_text = html_lib.escape(node.text_content) if node.text_content else ""
if cls_normalized == "container":
return f'<div class="{tw_width} p-8 flex flex-col" style="border: 2px solid {c["border"]}; border-radius: {br}; {inline_style}">{children_html}</div>\n'
elif cls_normalized == "radio":
text = clean_ocr_text(safe_text) if safe_text else ""
radio_style = f"accent-color: {c['accent']};"
if text_first:
label_html = f'<label class="text-sm font-medium truncate mr-2" style="color: {c["text_secondary"]};">{text}</label>' if text else ""
return f'<div class="{tw_width} flex items-center justify-end" style="{inline_style}">{label_html}<input type="radio" style="{radio_style}" class="w-4 h-4 flex-shrink-0"></div>\n'
else:
label_html = f'<label class="text-sm font-medium truncate ml-2" style="color: {c["text_secondary"]};">{text}</label>' if text else ""
return f'<div class="{tw_width} flex items-center" style="{inline_style}"><input type="radio" style="{radio_style}" class="w-4 h-4 flex-shrink-0">{label_html}</div>\n'
elif cls_normalized == "checkbox":
text = clean_ocr_text(safe_text) if safe_text else ""
cb_style = f"accent-color: {c['accent']};"
if text_first:
label_html = f'<label class="text-sm font-medium truncate mr-2" style="color: {c["text_secondary"]};">{text}</label>' if text else ""
return f'<div class="{tw_width} flex items-center justify-end" style="{inline_style}">{label_html}<input type="checkbox" style="{cb_style}" class="w-4 h-4 flex-shrink-0"></div>\n'
else:
label_html = f'<label class="text-sm font-medium truncate ml-2" style="color: {c["text_secondary"]};">{text}</label>' if text else ""
return f'<div class="{tw_width} flex items-center" style="{inline_style}"><input type="checkbox" style="{cb_style}" class="w-4 h-4 flex-shrink-0">{label_html}</div>\n'
elif cls_normalized == "label":
text = safe_text if safe_text else "Label"
return f'<label class="{tw_width} text-sm font-medium truncate" style="color: {c["text_secondary"]}; {inline_style}">{text}</label>\n'
elif cls_normalized == "toggle":
return f'''<div class="{tw_width} flex items-center" style="{inline_style}">
<label class="relative inline-flex items-center cursor-pointer">
<input type="checkbox" value="" class="sr-only peer">
<div class="w-11 h-6 rounded-full peer peer-checked:after:translate-x-full after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border after:rounded-full after:h-5 after:w-5 after:transition-all flex-shrink-0" style="background: {c['border']};"
onchange="this.style.background=this.previousElementSibling.checked?'{c['accent']}':'{c['border']}'"></div>
</label>
</div>\n'''
elif "button" in cls_normalized:
text = safe_text if safe_text else "Button"
if "SELECT" in text_upper or text_upper.endswith(" V"):
display_text = text.replace(" v", "").replace(" V", "").strip()
if not display_text: display_text = "Select..."
return f'''<div class="{tw_width}" style="{inline_style}">
<select class="w-full text-sm font-medium block px-4 py-2 appearance-none cursor-pointer" style="background: {c['bg_input']}; border: 1px solid {c['border']}; color: {c['text_primary']}; border-radius: {br};">
<option selected>{display_text}</option>
<option value="1">Option 1</option>
<option value="2">Option 2</option>
</select>
</div>\n'''
else:
return f'<button class="{tw_width} font-bold py-2 px-4 transition-colors flex items-center justify-center truncate" style="background: {c["accent"]}; color: {c["button_text"]}; border-radius: {br}; {inline_style}" onmouseover="this.style.background=\'{c["accent_hover"]}\'" onmouseout="this.style.background=\'{c["accent"]}\'">{text}</button>\n'
elif cls_normalized == "text":
text = safe_text if safe_text else "Text"
height_pct = int((node.h / img_height) * 100)
if height_pct > 10:
return f'<h1 class="{tw_width} text-3xl font-bold tracking-tight flex items-center truncate" style="color: {c["text_primary"]}; {inline_style}">{text}</h1>\n'
elif height_pct > 5:
return f'<h2 class="{tw_width} text-xl font-semibold tracking-tight flex items-center truncate" style="color: {c["text_primary"]}; {inline_style}">{text}</h2>\n'
else:
return f'<p class="{tw_width} text-sm flex items-center truncate" style="color: {c["text_secondary"]}; {inline_style}">{text}</p>\n'
elif cls_normalized == "header" or cls_normalized == "heading":
text = safe_text if safe_text else "Header Title"
return f'<h1 class="{tw_width} text-2xl font-bold tracking-tight flex items-center truncate" style="color: {c["text_primary"]}; {inline_style}">{text}</h1>\n'
elif cls_normalized == "textarea":
return f'<textarea class="{tw_width} min-h-[100px] py-2 px-3 focus:outline-none" placeholder="Type here..." style="background: {c["bg_input"]}; border: 1px solid {c["border"]}; border-radius: {br}; color: {c["text_primary"]}; {inline_style}"></textarea>\n'
elif cls_normalized == "input":
text = safe_text if safe_text else "Input field..."
return f'<input type="text" placeholder="{text}" class="{tw_width} py-2 px-3 focus:outline-none" style="background: {c["bg_input"]}; border: 1px solid {c["border"]}; border-radius: {br}; color: {c["text_primary"]}; {inline_style}">\n'
elif cls_normalized == "image" or cls_normalized == "icon":
return f'<div class="{tw_width} min-h-[120px] flex items-center justify-center" style="background: {c["bg_input"]}; border-radius: {br}; border: 2px dashed {c["border"]}; color: {c["text_placeholder"]}; {inline_style}">Image Area</div>\n'
else:
content = children_html if children_html else node.cls_name
return f'<div class="{tw_width} p-2 flex items-center justify-center text-xs truncate" style="background: {c["bg_input"]}; border: 1px solid {c["border"]}; border-radius: {br}; color: {c["text_placeholder"]}; {inline_style}">{content}</div>\n'
def assign_ocr_to_boxes(img_pil, boxes):
try:
img_np = np.array(img_pil)
reader = get_ocr_reader()
ocr_results = reader.readtext(img_np)
for box in boxes:
normalized_cls = box.cls_name.lower()
if normalized_cls not in ["text", "header", "heading", "button", "input", "radio", "checkbox", "label"]:
continue
box_texts = []
for ocr_box, text, conf in ocr_results:
# ocr_box is [[x1, y1], [x2, y1], [x2, y2], [x1, y2]]
ox1 = min(p[0] for p in ocr_box)
ox2 = max(p[0] for p in ocr_box)
oy1 = min(p[1] for p in ocr_box)
oy2 = max(p[1] for p in ocr_box)
# Check for intersection between OCR box and YOLO box
overlap_x = max(0, min(box.x2, ox2) - max(box.x1, ox1))
overlap_y = max(0, min(box.y2, oy2) - max(box.y1, oy1))
if overlap_x > 0 and overlap_y > 0:
intersection_area = overlap_x * overlap_y
ocr_box_area = (ox2 - ox1) * (oy2 - oy1)
# The OCR text box must be at least 40% inside the YOLO box
if intersection_area / ocr_box_area > 0.4:
box_texts.append((ox1, oy1, text))
if box_texts:
# Sort texts by roughly Y, then X for left-to-right reading
box_texts.sort(key=lambda t: (t[1] // 20, t[0]))
raw_text = " ".join([t[2] for t in box_texts])
box.text_content = fuzzy_correct(raw_text)
except Exception as e:
print(f"Global OCR/NLP Error: {e}")
def parse_yolo_results(results, img_pil, img_width, img_height, class_names, theme=None):
if theme is None:
theme = DEFAULT_THEME
c = theme["colors"]
br = theme.get("border_radius", "6px")
font = theme.get("font_family", "Inter, system-ui, sans-serif")
boxes = []
for r in results:
for box in r.boxes:
x, y, w, h = box.xywh[0].tolist()
cls_idx = int(box.cls[0].item())
cls_name = class_names[cls_idx]
b = BoundingBox(cls_name, x, y, w, h)
boxes.append(b)
assign_ocr_to_boxes(img_pil, boxes)
top_level_nodes = build_dom_tree(boxes)
html = [f'<div class="w-full min-h-screen p-8 flex flex-col items-center antialiased" style="background: {c["bg_page"]}; font-family: {font};">']
html.append(f' <div class="w-full max-w-4xl shadow-2xl p-8 flex flex-col" style="background: {c["bg_card"]}; border: 1px solid {c["border"]}; border-radius: {br};">')
rows = group_into_rows(top_level_nodes)
# Spatial Element Merging (Group isolated text with adjacent drawn radio/checkbox circles)
merged_rows = []
for row in rows:
merged_row = []
i = 0
while i < len(row):
node = row[i]
# Case 1: Circle (Radio/Checkbox) followed by Text (e.g. [O] MALE)
if node.cls_name.lower() in ["radio", "checkbox", "icon", "icon-button", "icon-label"] and not getattr(node, 'is_merged', False):
if i + 1 < len(row):
next_node = row[i+1]
if next_node.cls_name.lower() in ["label", "text", "radio", "checkbox"]:
dist = next_node.x1 - node.x2
# Accept any overlap (negative distance) or gap up to 30% of screen
if dist < (img_width * 0.30):
node.text_content = next_node.text_content
node.w = max(node.x2, next_node.x2) - node.x1
node.x2 = max(node.x2, next_node.x2)
node.x = (node.x1 + node.x2) / 2
if node.cls_name.lower() not in ["radio", "checkbox"]:
node.cls_name = "radio" # Force to radio if it was detected as an icon
node.is_merged = True
merged_row.append(node)
i += 2
continue
# Case 2: Text followed by Circle/Box (e.g. ARE YOU DISABLED [V])
if node.cls_name.lower() in ["label", "text", "radio"] and not getattr(node, 'is_merged', False):
if i + 1 < len(row):
next_node = row[i+1]
if next_node.cls_name.lower() in ["checkbox", "radio", "icon", "icon-button", "icon-label"]:
dist = next_node.x1 - node.x2
if dist < (img_width * 0.30):
original_text = next_node.text_content.upper().strip() if next_node.text_content else ""
next_node.text_content = node.text_content
next_node.w = max(node.x2, next_node.x2) - node.x1
next_node.x1 = node.x1
next_node.x = (next_node.x1 + next_node.x2) / 2
# The user requested that only the circles are radios, the trailing box should be a checkbox
next_node.cls_name = "checkbox"
next_node.is_merged = True
next_node.text_first = True
merged_row.append(next_node)
i += 2
continue
merged_row.append(node)
i += 1
merged_rows.append(merged_row)
rows = merged_rows
for row in rows:
if len(row) > 1:
html.append(' <div class="flex flex-row items-center w-full mb-6">')
current_x_pct = 0
for node in row:
node_x_pct = (node.x1 / img_width) * 100
gap_pct = node_x_pct - current_x_pct
if gap_pct > 1:
html.append(f' <div style="width: {gap_pct}%; flex-shrink: 0;"></div>')
node_w_pct = (node.w / img_width) * 100
node_html = generate_html_node(node, img_width, img_height, theme=theme, tw_width="w-full", inline_style=f"width: {node_w_pct}%; flex-shrink: 0;")
html.append(f" {node_html}")
current_x_pct = node_x_pct + node_w_pct
html.append(' </div>')
else:
node = row[0]
node_w_pct = (node.w / img_width) * 100
node_x_pct = (node.x1 / img_width) * 100
html.append(' <div class="flex flex-row items-center w-full mb-6">')
if node_x_pct > 1:
html.append(f' <div style="width: {node_x_pct}%; flex-shrink: 0;"></div>')
node_html = generate_html_node(node, img_width, img_height, theme=theme, tw_width="w-full", inline_style=f"width: {node_w_pct}%; flex-shrink: 0;")
html.append(f" {node_html}")
html.append(' </div>')
html.append(' </div>')
html.append('</div>')
return "\n".join(html)