Spaces:
Sleeping
Sleeping
Download parser.py from CyborgAB/sketch-to-ui: direct link, hf CLI and curl.
- Browser
- Download file 23.1 kB
-
https://huggingface.co/spaces/CyborgAB/sketch-to-ui/resolve/main/parser.py
- Command line
-
hf download hf://spaces/CyborgAB/sketch-to-ui/parser.py
-
curl -L -o parser.py https://huggingface.co/spaces/CyborgAB/sketch-to-ui/resolve/main/parser.py
23.1 kB
| import easyocr | |
| import numpy as np | |
| import re | |
| import difflib | |
| from PIL import Image | |
| from theme_engine import DEFAULT_THEME | |
| # Initialize easyocr and spellchecker lazily | |
| _reader = None | |
| def get_ocr_reader(): | |
| global _reader | |
| if _reader is None: | |
| _reader = easyocr.Reader(['en']) | |
| return _reader | |
| _spell = None | |
| def get_spellchecker(): | |
| global _spell | |
| if _spell is None: | |
| try: | |
| from spellchecker import SpellChecker | |
| _spell = SpellChecker() | |
| # Add some common UI terms so they don't get accidentally "corrected" | |
| _spell.word_frequency.load_words(['textarea', 'checkbox', 'radio', 'dropdown', 'login', 'signup', 'submit']) | |
| except ImportError: | |
| print("pyspellchecker not installed. Skipping NLP correction.") | |
| _spell = False | |
| return _spell | |
| # Comprehensive UI words dictionary for aggressive fuzzy matching against OCR hallucinations | |
| UI_WORDS = [ | |
| # Core UI | |
| "GENDER", "MALE", "FEMALE", "DESCRIPTION", "SUBMIT", | |
| "ENTER NAME", "HI GOOD MORNING", "BUTTON", "CHECKBOX", | |
| "RADIO", "IMAGE", "TEXT", "INPUT", "LABEL", "SELECT", | |
| # Common Form Fields | |
| "DEPARTMENT", "ARE YOU DISABLED", "CAPTCHA", "FIRST NAME", "LAST NAME", | |
| "EMAIL", "PASSWORD", "PHONE", "ADDRESS", "CITY", "STATE", | |
| "ZIP CODE", "COUNTRY", "DATE OF BIRTH", "AGE", "COMPANY", | |
| "TITLE", "ROLE", "WEBSITE", "MESSAGE", "COMMENTS", | |
| # Common Actions & Toggles | |
| "TERMS", "CONDITIONS", "AGREE", "ACCEPT", "DECLINE", | |
| "YES", "NO", "UPLOAD", "DOWNLOAD", "SAVE", "CANCEL", | |
| "NEXT", "PREVIOUS", "BACK", "SEARCH", "FILTER", "SORT", | |
| # Nav & Auth | |
| "MENU", "HOME", "ABOUT", "CONTACT", "LOGIN", "LOGOUT", | |
| "REGISTER", "SIGN UP", "SIGN IN", "FORGOT PASSWORD", | |
| "RESET", "SEND", "ADD", "REMOVE", "DELETE", "EDIT", | |
| "UPDATE", "PROFILE", "SETTINGS", "HELP", "SUPPORT", "FAQ" | |
| ] | |
| class BoundingBox: | |
| def __init__(self, cls_name, x, y, w, h): | |
| self.cls_name = cls_name | |
| self.x = x | |
| self.y = y | |
| self.w = w | |
| self.h = h | |
| self.x1 = x - w / 2 | |
| self.y1 = y - h / 2 | |
| self.x2 = x + w / 2 | |
| self.y2 = y + h / 2 | |
| self.children = [] | |
| self.parent = None | |
| self.text_content = "" | |
| def contains(self, other): | |
| # A box contains another if it covers at least 80% of the other's area | |
| # This prevents large slightly-skewed boxes from losing their children | |
| ix1 = max(self.x1, other.x1) | |
| iy1 = max(self.y1, other.y1) | |
| ix2 = min(self.x2, other.x2) | |
| iy2 = min(self.y2, other.y2) | |
| if ix2 > ix1 and iy2 > iy1: | |
| intersection_area = (ix2 - ix1) * (iy2 - iy1) | |
| if intersection_area / other.area() > 0.7: | |
| return True | |
| return False | |
| def area(self): | |
| return self.w * self.h | |
| def __repr__(self): | |
| return f"{self.cls_name}({self.x}, {self.y}, {self.w}, {self.h})" | |
| def build_dom_tree(boxes): | |
| boxes.sort(key=lambda b: b.area(), reverse=True) | |
| top_level = [] | |
| for i, box in enumerate(boxes): | |
| parent_found = False | |
| # Search from smallest possible parents to largest | |
| for j in range(len(boxes) - 1, -1, -1): | |
| if i == j: continue | |
| potential_parent = boxes[j] | |
| if potential_parent.area() > box.area() and potential_parent.contains(box): | |
| potential_parent.children.append(box) | |
| box.parent = potential_parent | |
| parent_found = True | |
| break | |
| if not parent_found: | |
| top_level.append(box) | |
| return optimize_dom_tree(top_level) | |
| def optimize_dom_tree(nodes): | |
| optimized = [] | |
| for node in nodes: | |
| node.children = optimize_dom_tree(node.children) | |
| # Collapse redundant nested containers (container -> exactly 1 container child) | |
| if node.cls_name.lower() == "container" and len(node.children) == 1 and node.children[0].cls_name.lower() == "container": | |
| optimized.append(node.children[0]) | |
| else: | |
| optimized.append(node) | |
| return optimized | |
| def clean_ocr_text(text): | |
| # Remove leading fake circles like "0", "o", "O", "[]" from radio/checkbox text | |
| text = text.strip() | |
| return re.sub(r'^([oO0\[\]\(\)]\s*)+', '', text) | |
| def fuzzy_correct(text): | |
| clean_text = re.sub(r'[^A-Z\s]', '', text.upper()).strip() | |
| if not clean_text: | |
| return text | |
| matches = difflib.get_close_matches(clean_text, UI_WORDS, n=1, cutoff=0.5) | |
| if matches: | |
| return matches[0] | |
| # Fallback to spellchecker | |
| spell = get_spellchecker() | |
| if spell: | |
| words = text.split() | |
| corrected_words = [] | |
| for word in words: | |
| clean_word = re.sub(r'[^a-zA-Z]', '', word) | |
| if clean_word and clean_word.lower() not in spell: | |
| correction = spell.correction(clean_word) | |
| if correction: | |
| if word.isupper(): | |
| corrected_words.append(correction.upper()) | |
| elif word.istitle(): | |
| corrected_words.append(correction.title()) | |
| else: | |
| corrected_words.append(correction) | |
| else: | |
| corrected_words.append(word) | |
| else: | |
| corrected_words.append(word) | |
| return " ".join(corrected_words) | |
| return text | |
| def group_into_rows(nodes): | |
| if not nodes: | |
| return [] | |
| # Sort nodes by top edge (y1) to process them top-to-bottom | |
| sorted_nodes = sorted(nodes, key=lambda n: n.y1) | |
| rows = [] | |
| current_row = [] | |
| for node in sorted_nodes: | |
| if not current_row: | |
| current_row.append(node) | |
| else: | |
| row_y1 = min(n.y1 for n in current_row) | |
| row_y2 = max(n.y2 for n in current_row) | |
| overlap = max(0, min(node.y2, row_y2) - max(node.y1, row_y1)) | |
| min_h = min(node.h, row_y2 - row_y1) | |
| # If they overlap by at least 30% of the smaller height | |
| if min_h > 0 and (overlap / min_h) > 0.3: | |
| current_row.append(node) | |
| else: | |
| rows.append(current_row) | |
| current_row = [node] | |
| if current_row: | |
| rows.append(current_row) | |
| # Sort each row horizontally | |
| for i in range(len(rows)): | |
| rows[i] = sorted(rows[i], key=lambda n: n.x) | |
| return rows | |
| import html as html_lib | |
| def generate_html_node(node, img_width, img_height, theme=None, tw_width="w-full", inline_style=""): | |
| if theme is None: | |
| theme = DEFAULT_THEME | |
| c = theme["colors"] | |
| br = theme.get("border_radius", "6px") | |
| cls_normalized = node.cls_name.lower() | |
| text_upper = node.text_content.upper().strip() if node.text_content else "" | |
| is_merged = getattr(node, 'is_merged', False) | |
| text_first = getattr(node, 'text_first', False) | |
| # 1. Radio/Checkbox Heuristics | |
| if cls_normalized in ["radio", "checkbox"] and not is_merged: | |
| # If it's a structural label that YOLO hallucinated as a radio | |
| if any(word in text_upper for word in ["GENDER", "DEPARTMENT", "DISABLED"]): | |
| cls_normalized = "label" | |
| # If the radio/checkbox text is literally just a checkmark 'V', make it a checkbox | |
| elif text_upper == "V": | |
| cls_normalized = "checkbox" | |
| # If it has text but no circle/box indicator, and isn't empty (which means it's the actual drawn box) | |
| elif text_upper and not re.search(r'([oO0\[\]\(\)vV])', text_upper): | |
| # We assume OCR picked up a normal label, so we convert it. | |
| # But we leave EMPTY text alone, because those are the actual drawn circles/boxes! | |
| cls_normalized = "label" | |
| children_html = "" | |
| if node.children: | |
| rows = group_into_rows(node.children) | |
| for row in rows: | |
| if len(row) > 1: | |
| children_html += '<div class="flex flex-row items-center w-full mb-6">\n' | |
| current_x_pct = 0 | |
| for child in row: | |
| child_x_pct = (child.x1 / node.w) * 100 | |
| gap_pct = child_x_pct - current_x_pct | |
| if gap_pct > 1: | |
| children_html += f'<div style="width: {gap_pct}%; flex-shrink: 0;"></div>\n' | |
| child_w_pct = (child.w / node.w) * 100 | |
| children_html += generate_html_node(child, node.w, node.h, theme=theme, tw_width="w-full", inline_style=f"width: {child_w_pct}%; flex-shrink: 0;") | |
| current_x_pct = child_x_pct + child_w_pct | |
| children_html += '</div>\n' | |
| else: | |
| child = row[0] | |
| child_w_pct = (child.w / node.w) * 100 | |
| child_x_pct = (child.x1 / node.w) * 100 | |
| children_html += f'<div class="flex flex-row items-center w-full mb-6">\n' | |
| if child_x_pct > 1: | |
| children_html += f'<div style="width: {child_x_pct}%; flex-shrink: 0;"></div>\n' | |
| children_html += generate_html_node(child, node.w, node.h, theme=theme, tw_width="w-full", inline_style=f"width: {child_w_pct}%; flex-shrink: 0;") | |
| children_html += '</div>\n' | |
| safe_text = html_lib.escape(node.text_content) if node.text_content else "" | |
| if cls_normalized == "container": | |
| return f'<div class="{tw_width} p-8 flex flex-col" style="border: 2px solid {c["border"]}; border-radius: {br}; {inline_style}">{children_html}</div>\n' | |
| elif cls_normalized == "radio": | |
| text = clean_ocr_text(safe_text) if safe_text else "" | |
| radio_style = f"accent-color: {c['accent']};" | |
| if text_first: | |
| label_html = f'<label class="text-sm font-medium truncate mr-2" style="color: {c["text_secondary"]};">{text}</label>' if text else "" | |
| return f'<div class="{tw_width} flex items-center justify-end" style="{inline_style}">{label_html}<input type="radio" style="{radio_style}" class="w-4 h-4 flex-shrink-0"></div>\n' | |
| else: | |
| label_html = f'<label class="text-sm font-medium truncate ml-2" style="color: {c["text_secondary"]};">{text}</label>' if text else "" | |
| return f'<div class="{tw_width} flex items-center" style="{inline_style}"><input type="radio" style="{radio_style}" class="w-4 h-4 flex-shrink-0">{label_html}</div>\n' | |
| elif cls_normalized == "checkbox": | |
| text = clean_ocr_text(safe_text) if safe_text else "" | |
| cb_style = f"accent-color: {c['accent']};" | |
| if text_first: | |
| label_html = f'<label class="text-sm font-medium truncate mr-2" style="color: {c["text_secondary"]};">{text}</label>' if text else "" | |
| return f'<div class="{tw_width} flex items-center justify-end" style="{inline_style}">{label_html}<input type="checkbox" style="{cb_style}" class="w-4 h-4 flex-shrink-0"></div>\n' | |
| else: | |
| label_html = f'<label class="text-sm font-medium truncate ml-2" style="color: {c["text_secondary"]};">{text}</label>' if text else "" | |
| return f'<div class="{tw_width} flex items-center" style="{inline_style}"><input type="checkbox" style="{cb_style}" class="w-4 h-4 flex-shrink-0">{label_html}</div>\n' | |
| elif cls_normalized == "label": | |
| text = safe_text if safe_text else "Label" | |
| return f'<label class="{tw_width} text-sm font-medium truncate" style="color: {c["text_secondary"]}; {inline_style}">{text}</label>\n' | |
| elif cls_normalized == "toggle": | |
| return f'''<div class="{tw_width} flex items-center" style="{inline_style}"> | |
| <label class="relative inline-flex items-center cursor-pointer"> | |
| <input type="checkbox" value="" class="sr-only peer"> | |
| <div class="w-11 h-6 rounded-full peer peer-checked:after:translate-x-full after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border after:rounded-full after:h-5 after:w-5 after:transition-all flex-shrink-0" style="background: {c['border']};" | |
| onchange="this.style.background=this.previousElementSibling.checked?'{c['accent']}':'{c['border']}'"></div> | |
| </label> | |
| </div>\n''' | |
| elif "button" in cls_normalized: | |
| text = safe_text if safe_text else "Button" | |
| if "SELECT" in text_upper or text_upper.endswith(" V"): | |
| display_text = text.replace(" v", "").replace(" V", "").strip() | |
| if not display_text: display_text = "Select..." | |
| return f'''<div class="{tw_width}" style="{inline_style}"> | |
| <select class="w-full text-sm font-medium block px-4 py-2 appearance-none cursor-pointer" style="background: {c['bg_input']}; border: 1px solid {c['border']}; color: {c['text_primary']}; border-radius: {br};"> | |
| <option selected>{display_text}</option> | |
| <option value="1">Option 1</option> | |
| <option value="2">Option 2</option> | |
| </select> | |
| </div>\n''' | |
| else: | |
| return f'<button class="{tw_width} font-bold py-2 px-4 transition-colors flex items-center justify-center truncate" style="background: {c["accent"]}; color: {c["button_text"]}; border-radius: {br}; {inline_style}" onmouseover="this.style.background=\'{c["accent_hover"]}\'" onmouseout="this.style.background=\'{c["accent"]}\'">{text}</button>\n' | |
| elif cls_normalized == "text": | |
| text = safe_text if safe_text else "Text" | |
| height_pct = int((node.h / img_height) * 100) | |
| if height_pct > 10: | |
| return f'<h1 class="{tw_width} text-3xl font-bold tracking-tight flex items-center truncate" style="color: {c["text_primary"]}; {inline_style}">{text}</h1>\n' | |
| elif height_pct > 5: | |
| return f'<h2 class="{tw_width} text-xl font-semibold tracking-tight flex items-center truncate" style="color: {c["text_primary"]}; {inline_style}">{text}</h2>\n' | |
| else: | |
| return f'<p class="{tw_width} text-sm flex items-center truncate" style="color: {c["text_secondary"]}; {inline_style}">{text}</p>\n' | |
| elif cls_normalized == "header" or cls_normalized == "heading": | |
| text = safe_text if safe_text else "Header Title" | |
| return f'<h1 class="{tw_width} text-2xl font-bold tracking-tight flex items-center truncate" style="color: {c["text_primary"]}; {inline_style}">{text}</h1>\n' | |
| elif cls_normalized == "textarea": | |
| return f'<textarea class="{tw_width} min-h-[100px] py-2 px-3 focus:outline-none" placeholder="Type here..." style="background: {c["bg_input"]}; border: 1px solid {c["border"]}; border-radius: {br}; color: {c["text_primary"]}; {inline_style}"></textarea>\n' | |
| elif cls_normalized == "input": | |
| text = safe_text if safe_text else "Input field..." | |
| return f'<input type="text" placeholder="{text}" class="{tw_width} py-2 px-3 focus:outline-none" style="background: {c["bg_input"]}; border: 1px solid {c["border"]}; border-radius: {br}; color: {c["text_primary"]}; {inline_style}">\n' | |
| elif cls_normalized == "image" or cls_normalized == "icon": | |
| return f'<div class="{tw_width} min-h-[120px] flex items-center justify-center" style="background: {c["bg_input"]}; border-radius: {br}; border: 2px dashed {c["border"]}; color: {c["text_placeholder"]}; {inline_style}">Image Area</div>\n' | |
| else: | |
| content = children_html if children_html else node.cls_name | |
| return f'<div class="{tw_width} p-2 flex items-center justify-center text-xs truncate" style="background: {c["bg_input"]}; border: 1px solid {c["border"]}; border-radius: {br}; color: {c["text_placeholder"]}; {inline_style}">{content}</div>\n' | |
| def assign_ocr_to_boxes(img_pil, boxes): | |
| try: | |
| img_np = np.array(img_pil) | |
| reader = get_ocr_reader() | |
| ocr_results = reader.readtext(img_np) | |
| for box in boxes: | |
| normalized_cls = box.cls_name.lower() | |
| if normalized_cls not in ["text", "header", "heading", "button", "input", "radio", "checkbox", "label"]: | |
| continue | |
| box_texts = [] | |
| for ocr_box, text, conf in ocr_results: | |
| # ocr_box is [[x1, y1], [x2, y1], [x2, y2], [x1, y2]] | |
| ox1 = min(p[0] for p in ocr_box) | |
| ox2 = max(p[0] for p in ocr_box) | |
| oy1 = min(p[1] for p in ocr_box) | |
| oy2 = max(p[1] for p in ocr_box) | |
| # Check for intersection between OCR box and YOLO box | |
| overlap_x = max(0, min(box.x2, ox2) - max(box.x1, ox1)) | |
| overlap_y = max(0, min(box.y2, oy2) - max(box.y1, oy1)) | |
| if overlap_x > 0 and overlap_y > 0: | |
| intersection_area = overlap_x * overlap_y | |
| ocr_box_area = (ox2 - ox1) * (oy2 - oy1) | |
| # The OCR text box must be at least 40% inside the YOLO box | |
| if intersection_area / ocr_box_area > 0.4: | |
| box_texts.append((ox1, oy1, text)) | |
| if box_texts: | |
| # Sort texts by roughly Y, then X for left-to-right reading | |
| box_texts.sort(key=lambda t: (t[1] // 20, t[0])) | |
| raw_text = " ".join([t[2] for t in box_texts]) | |
| box.text_content = fuzzy_correct(raw_text) | |
| except Exception as e: | |
| print(f"Global OCR/NLP Error: {e}") | |
| def parse_yolo_results(results, img_pil, img_width, img_height, class_names, theme=None): | |
| if theme is None: | |
| theme = DEFAULT_THEME | |
| c = theme["colors"] | |
| br = theme.get("border_radius", "6px") | |
| font = theme.get("font_family", "Inter, system-ui, sans-serif") | |
| boxes = [] | |
| for r in results: | |
| for box in r.boxes: | |
| x, y, w, h = box.xywh[0].tolist() | |
| cls_idx = int(box.cls[0].item()) | |
| cls_name = class_names[cls_idx] | |
| b = BoundingBox(cls_name, x, y, w, h) | |
| boxes.append(b) | |
| assign_ocr_to_boxes(img_pil, boxes) | |
| top_level_nodes = build_dom_tree(boxes) | |
| html = [f'<div class="w-full min-h-screen p-8 flex flex-col items-center antialiased" style="background: {c["bg_page"]}; font-family: {font};">'] | |
| html.append(f' <div class="w-full max-w-4xl shadow-2xl p-8 flex flex-col" style="background: {c["bg_card"]}; border: 1px solid {c["border"]}; border-radius: {br};">') | |
| rows = group_into_rows(top_level_nodes) | |
| # Spatial Element Merging (Group isolated text with adjacent drawn radio/checkbox circles) | |
| merged_rows = [] | |
| for row in rows: | |
| merged_row = [] | |
| i = 0 | |
| while i < len(row): | |
| node = row[i] | |
| # Case 1: Circle (Radio/Checkbox) followed by Text (e.g. [O] MALE) | |
| if node.cls_name.lower() in ["radio", "checkbox", "icon", "icon-button", "icon-label"] and not getattr(node, 'is_merged', False): | |
| if i + 1 < len(row): | |
| next_node = row[i+1] | |
| if next_node.cls_name.lower() in ["label", "text", "radio", "checkbox"]: | |
| dist = next_node.x1 - node.x2 | |
| # Accept any overlap (negative distance) or gap up to 30% of screen | |
| if dist < (img_width * 0.30): | |
| node.text_content = next_node.text_content | |
| node.w = max(node.x2, next_node.x2) - node.x1 | |
| node.x2 = max(node.x2, next_node.x2) | |
| node.x = (node.x1 + node.x2) / 2 | |
| if node.cls_name.lower() not in ["radio", "checkbox"]: | |
| node.cls_name = "radio" # Force to radio if it was detected as an icon | |
| node.is_merged = True | |
| merged_row.append(node) | |
| i += 2 | |
| continue | |
| # Case 2: Text followed by Circle/Box (e.g. ARE YOU DISABLED [V]) | |
| if node.cls_name.lower() in ["label", "text", "radio"] and not getattr(node, 'is_merged', False): | |
| if i + 1 < len(row): | |
| next_node = row[i+1] | |
| if next_node.cls_name.lower() in ["checkbox", "radio", "icon", "icon-button", "icon-label"]: | |
| dist = next_node.x1 - node.x2 | |
| if dist < (img_width * 0.30): | |
| original_text = next_node.text_content.upper().strip() if next_node.text_content else "" | |
| next_node.text_content = node.text_content | |
| next_node.w = max(node.x2, next_node.x2) - node.x1 | |
| next_node.x1 = node.x1 | |
| next_node.x = (next_node.x1 + next_node.x2) / 2 | |
| # The user requested that only the circles are radios, the trailing box should be a checkbox | |
| next_node.cls_name = "checkbox" | |
| next_node.is_merged = True | |
| next_node.text_first = True | |
| merged_row.append(next_node) | |
| i += 2 | |
| continue | |
| merged_row.append(node) | |
| i += 1 | |
| merged_rows.append(merged_row) | |
| rows = merged_rows | |
| for row in rows: | |
| if len(row) > 1: | |
| html.append(' <div class="flex flex-row items-center w-full mb-6">') | |
| current_x_pct = 0 | |
| for node in row: | |
| node_x_pct = (node.x1 / img_width) * 100 | |
| gap_pct = node_x_pct - current_x_pct | |
| if gap_pct > 1: | |
| html.append(f' <div style="width: {gap_pct}%; flex-shrink: 0;"></div>') | |
| node_w_pct = (node.w / img_width) * 100 | |
| node_html = generate_html_node(node, img_width, img_height, theme=theme, tw_width="w-full", inline_style=f"width: {node_w_pct}%; flex-shrink: 0;") | |
| html.append(f" {node_html}") | |
| current_x_pct = node_x_pct + node_w_pct | |
| html.append(' </div>') | |
| else: | |
| node = row[0] | |
| node_w_pct = (node.w / img_width) * 100 | |
| node_x_pct = (node.x1 / img_width) * 100 | |
| html.append(' <div class="flex flex-row items-center w-full mb-6">') | |
| if node_x_pct > 1: | |
| html.append(f' <div style="width: {node_x_pct}%; flex-shrink: 0;"></div>') | |
| node_html = generate_html_node(node, img_width, img_height, theme=theme, tw_width="w-full", inline_style=f"width: {node_w_pct}%; flex-shrink: 0;") | |
| html.append(f" {node_html}") | |
| html.append(' </div>') | |
| html.append(' </div>') | |
| html.append('</div>') | |
| return "\n".join(html) | |