Harden layout generation and image mapping

screencoder/block_parsor.py

@@ -12,9 +12,22 @@ PROMPT_LIST = [
     ("navigation", "Please output the minimum bounding box of the navigation. Please output the bounding box in the format of <bbox>x1 y1 x2 y2</bbox>. Avoid the blank space in the navigation."),
     ("main content", "Please output the minimum bounding box of the main content. Please output the bounding box in the format of <bbox>x1 y1 x2 y2</bbox>. Avoid the blank space in the main content."),
 ]
-PROMPT_MERGE = "Return the bounding boxes of the sidebar, main content, header, and navigation in this webpage screenshot. Please only return the corresponding bounding boxes. Note: 1. The areas should not overlap; 2. All text information and other content should be framed inside; 3. Try to keep it compact without leaving a lot of blank space; 4. Output a label and the corresponding bounding box for each line."
+PROMPT_MERGE = """Partition the entire webpage screenshot into exactly four rectangular regions: sidebar, header, navigation, and main content.
+
+Use a normalized 0-1000 coordinate system, where the top-left corner of the full screenshot is (0, 0) and the bottom-right corner is (1000, 1000).
+
+Treat this as a gap-free page partition rather than four independent object detections:
+1. The four regions must not overlap.
+2. Their union must cover the entire visible webpage.
+3. Adjacent regions must share exact boundaries, with no uncovered gaps between them.
+4. Extend each region through internal whitespace until it reaches the neighboring region. Do not crop tightly around only the visible text, icons, or controls.
+5. Every visible text item, icon, image, control, and UI component must belong to one of the four regions.
+6. Return exactly four lines and no explanation. Use this exact format for every line:
+label: <bbox>x1 y1 x2 y2</bbox>
+"""
 BBOX_TAG_START = "<bbox>"
 BBOX_TAG_END = "</bbox>"
+EXPECTED_REGION_NAMES = {"sidebar", "header", "navigation", "main content"}
 
 def get_args():
     parser = argparse.ArgumentParser(description="Parses bounding boxes from an image using a vision model.")
@@ -62,6 +75,51 @@ def parse_bboxes(bbox_input: str) -> dict[str, tuple[int, int, int, int]]:
     print("Final parsed bboxes:", bboxes)
     return bboxes
 
+
+def normalize_gap_free_partition(
+    bboxes: dict[str, tuple[int, int, int, int]], edge_tolerance: int = 1
+) -> dict[str, tuple[int, int, int, int]]:
+    """Snap tiny outer-edge drift, then require an exact four-box page partition."""
+    if set(bboxes) != EXPECTED_REGION_NAMES:
+        raise ValueError(
+            f"gap-free partition requires exactly {sorted(EXPECTED_REGION_NAMES)}"
+        )
+
+    normalized = {}
+    for name, (x1, y1, x2, y2) in bboxes.items():
+        if 0 <= x1 <= edge_tolerance:
+            x1 = 0
+        if 0 <= y1 <= edge_tolerance:
+            y1 = 0
+        if 1000 - edge_tolerance <= x2 <= 1000:
+            x2 = 1000
+        if 1000 - edge_tolerance <= y2 <= 1000:
+            y2 = 1000
+        if not (0 <= x1 < x2 <= 1000 and 0 <= y1 < y2 <= 1000):
+            raise ValueError(f"gap-free partition has invalid box for {name}")
+        normalized[name] = (x1, y1, x2, y2)
+
+    boxes = list(normalized.items())
+    for index, (name_a, box_a) in enumerate(boxes):
+        ax1, ay1, ax2, ay2 = box_a
+        for name_b, box_b in boxes[index + 1 :]:
+            bx1, by1, bx2, by2 = box_b
+            overlap_w = max(0, min(ax2, bx2) - max(ax1, bx1))
+            overlap_h = max(0, min(ay2, by2) - max(ay1, by1))
+            if overlap_w * overlap_h > 0:
+                raise ValueError(
+                    f"gap-free partition overlaps: {name_a} and {name_b}"
+                )
+
+    total_area = sum(
+        (x2 - x1) * (y2 - y1) for x1, y1, x2, y2 in normalized.values()
+    )
+    if total_area != 1_000_000:
+        raise ValueError(
+            f"gap-free partition covers {total_area}/1000000 normalized cells"
+        )
+    return normalized
+
 def draw_bboxes(image_path: str, bboxes: dict[str, tuple[int, int, int, int]], output_path: str) -> str:
     """Draws normalized (0-1000) bboxes on an image for visualization."""
     image = cv2.imread(image_path)
@@ -229,7 +287,8 @@ def main():
     
     image_path = os.path.join(tmp_dir, f"{run_id}.png")
     # api_path = os.path.join(base_dir, "doubao_api.txt")
-    json_output_path = os.path.join(tmp_dir, f"{run_id}_bboxes.json")
+    json_output_path = os.path.join(tmp_dir, f"{run_id}_region_bboxes.json")
+    legacy_json_output_path = os.path.join(tmp_dir, f"{run_id}_bboxes.json")
     annotated_image_output_path = os.path.join(tmp_dir, f"{run_id}_with_bboxes.png")
 
     # Debug: Print all paths and check if files exist
@@ -250,6 +309,7 @@ def main():
         print(f"Error: Input image not found at {image_path}")
         # Create empty json file so the pipeline doesn't break
         save_bboxes_to_json({}, json_output_path)
+        save_bboxes_to_json({}, legacy_json_output_path)
         exit(1)
     
     api_key = os.environ.get('API_key')
@@ -257,6 +317,7 @@ def main():
         print(f"Error: API key not found in environment variable 'API_key'")
         # Create empty json file so the pipeline doesn't break
         save_bboxes_to_json({}, json_output_path)
+        save_bboxes_to_json({}, legacy_json_output_path)
         exit(1)
 
     print(f"--- Starting BBox Parsing for run_id: {run_id} ---")
@@ -268,23 +329,32 @@ def main():
         print(f"Error: Failed to encode image {image_path}")
         # Create empty json file so the pipeline doesn't break
         save_bboxes_to_json({}, json_output_path)
+        save_bboxes_to_json({}, legacy_json_output_path)
         exit(1)
     
     bbox_content = client.ask(PROMPT_MERGE, base64_image)
+    raw_response_path = os.path.join(tmp_dir, f"{run_id}_bbox_raw.txt")
+    with open(raw_response_path, "w", encoding="utf-8") as f:
+        f.write(bbox_content or "")
+    print(f"Raw bbox response saved to: {raw_response_path}")
     bboxes = parse_bboxes(bbox_content)
     
     if bboxes:
+        bboxes = normalize_gap_free_partition(bboxes)
+        print("Validated exact gap-free 0-1000 partition.")
         print("\n--- Resolving containment issues ---")
         bboxes = resolve_containment(bboxes)
         print("--- Containment resolved ---")
         
         print(f"\n--- Detection Complete for run_id: {run_id} ---")
         save_bboxes_to_json(bboxes, json_output_path)
+        save_bboxes_to_json(bboxes, legacy_json_output_path)
         draw_bboxes(image_path, bboxes, annotated_image_output_path)
     else:
         print(f"\nNo valid bounding box coordinates found for run_id: {run_id}")
         # Still create an empty json file so the pipeline doesn't break
         save_bboxes_to_json({}, json_output_path)
+        save_bboxes_to_json({}, legacy_json_output_path)
 
 if __name__ == "__main__":
-    main()
+    main()

screencoder/html_generator.py

@@ -9,6 +9,8 @@ import os
 
 # This dictionary holds the user's instructions for the current run.
 user_instruction = {"sidebar": "", "header": "", "navigation": "", "main content": ""}
+TARGET_VIEWPORT_WIDTH = 1280
+TARGET_VIEWPORT_HEIGHT = 720
 
 def get_args():
     parser = argparse.ArgumentParser(description="Generates an HTML layout from bounding box data.")
@@ -18,6 +20,13 @@ def get_args():
 
 def get_prompt_dict(instructions):
     """Dynamically creates the prompt dictionary with the user's instructions."""
+    image_contract = """
+对于截图中需要从原图裁剪并回填的真实位图（例如视频缩略图、照片、头像），请使用同时带有
+\"screencoder-image-placeholder bg-gray-400\" 两个 class 的纯灰色占位元素，并为它设置明确、非零的宽度和高度。
+不要将纯装饰色块、背景、分割线、按钮或 SVG 图标标记为 screencoder-image-placeholder。
+生成内容会被放入当前区域的边界框中：请优先使用 w-full、h-full、flex、grid 和相对尺寸，避免根据原始截图像素写入夸张的固定宽高或字号。
+可以使用 Tailwind CSS 3 的 utility class。所有元素必须留在当前容器内，不得溢出。
+"""
     # return {
     #     "sidebar": f"""This is a screenshot of a container. Please fill in a complete HTML and tail-wind CSS code to accurately reproduce the given container. Please note that the layout, icon style, size, and text information of all blocks need to be basically consistent with the original screenshot based on the user's additional conditions. User instruction: {instructions["sidebar"]}. The following is the code for filling in:
     # <div>
@@ -41,7 +50,7 @@ def get_prompt_dict(instructions):
     # only return the code within the <div> and </div> tags""",
     # }
     return {
-        "sidebar": f"""这是一个container的截图。这是用户给的额外要求：{instructions["sidebar"]}请填写一段完整的HTML和tail-wind CSS代码以准确再现给定的容器。请注意所有组块的排版、图标样式、大小、文字信息需要在用户额外条件的基础上与原始截图基本保持一致。对于小图标，请保持生成的svg图标和原图一致。请使用相同大小、位置的纯灰色图像块并标记为"bg-gray-400"代替大的图片，注意代码中的大小和位置信息。以下是供填写的代码：
+        "sidebar": f"""这是一个container的截图。这是用户给的额外要求：{instructions["sidebar"]}请填写一段完整的HTML和tail-wind CSS代码以准确再现给定的容器。请注意所有组块的排版、图标样式、大小、文字信息需要在用户额外条件的基础上与原始截图基本保持一致。对于小图标，请保持生成的svg图标和原图一致。{image_contract}以下是供填写的代码：
 
         <div>
         your code here
@@ -49,7 +58,7 @@ def get_prompt_dict(instructions):
 
         只需返回<div>和</div>标签内的代码""",
 
-        "header": f"""这是一个container的截图。这是用户给的额外要求：{instructions["header"]}请填写一段完整的HTML和tail-wind CSS代码以准确再现给定的容器。请注意所有组块在boundary box中的相对位置、排版、文字信息、颜色需要在用户额外条件的基础上与原始截图基本保持一致。请保持生成的svg图标和原图一致。请使用相同大小、位置的纯灰色图像块并标记为"bg-gray-400"代替图片，注意代码中的大小和位置信息。以下是供填写的代码：
+        "header": f"""这是一个container的截图。这是用户给的额外要求：{instructions["header"]}请填写一段完整的HTML和tail-wind CSS代码以准确再现给定的容器。请注意所有组块在boundary box中的相对位置、排版、文字信息、颜色需要在用户额外条件的基础上与原始截图基本保持一致。请保持生成的svg图标和原图一致。{image_contract}以下是供填写的代码：
 
         <div>
         your code here
@@ -57,7 +66,7 @@ def get_prompt_dict(instructions):
 
         只需返回<div>和</div>标签内的代码""",
 
-        "navigation": f"""这是一个container的截图。这是用户给的额外要求：{instructions["navigation"]}请填写一段完整的HTML和tail-wind CSS代码以准确再现给定的容器。请注意所有组块的在boundary box中的相对位置、文字排版、颜色需要在用户额外条件的基础上与原始截图基本保持一致。图像请你直接使用原始截图中一致的图标。请使用相同大小、位置的纯灰色图像块并标记为"bg-gray-400"代替图片，注意代码中的大小和位置信息。以下是供填写的代码：
+        "navigation": f"""这是一个container的截图。这是用户给的额外要求：{instructions["navigation"]}请填写一段完整的HTML和tail-wind CSS代码以准确再现给定的容器。请注意所有组块的在boundary box中的相对位置、文字排版、颜色需要在用户额外条件的基础上与原始截图基本���持一致。图像请你直接使用原始截图中一致的图标。{image_contract}以下是供填写的代码：
 
         <div>
         your code here
@@ -65,7 +74,7 @@ def get_prompt_dict(instructions):
 
         只需返回<div>和</div>标签内的代码""",
 
-        "main content": f"""这是一个container的截图。这是用户给的额外要求：{instructions["main content"]}请填写一段完整的HTML和tail-wind CSS代码以准确再现给定的容器。截图中显示的图像请使用相同大小、位置的纯灰色图像块并标记为"bg-gray-400"代替图片，注意代码中其大小和位置信息，不需要识别其中的文字信息。请注意所有组块在boundary box中的相对位置、排版、文字信息、颜色需要在用户额外条件的基础上与原始截图基本保持一致。以下是供填写的代码：
+        "main content": f"""这是一个container的截图。这是用户给的额外要求：{instructions["main content"]}请填写一段完整的HTML和tail-wind CSS代码以准确再现给定的容器。{image_contract}截图中的位图内容不需要识别其中的文字信息。请注意所有组块在boundary box中的相对位置、排版、文字信息、颜色需要在用户额外条件的基础上与原始截图基本保持一致。以下是供填写的代码：
 
         <div>
         your code here
@@ -74,6 +83,31 @@ def get_prompt_dict(instructions):
         只需返回<div>和</div>标签内的代码"""
     }
 
+
+def add_render_geometry_contract(prompt, bbox, root_bbox):
+    """Tell the model the CSS-pixel box its crop will occupy in the final page."""
+    root_width = max(1, root_bbox[2] - root_bbox[0])
+    root_height = max(1, root_bbox[3] - root_bbox[1])
+    target_width = max(
+        1, round((bbox[2] - bbox[0]) / root_width * TARGET_VIEWPORT_WIDTH)
+    )
+    target_height = max(
+        1, round((bbox[3] - bbox[1]) / root_height * TARGET_VIEWPORT_HEIGHT)
+    )
+    max_icon = max(12, min(24, target_width // 2, int(target_height * 0.67)))
+    max_font = max(10, min(16, target_width // 4, target_height // 3))
+    max_padding = max(1, min(8, target_width // 10, target_height // 6))
+
+    return f"""{prompt}
+
+【最终渲染几何约束】
+当前区域在最终 1280×720 viewport 中只有约 {target_width}×{target_height} CSS px。
+必须按照这个最终尺寸设计，不要按输入裁剪图的原始像素尺寸设计。
+最外层必须使用 w-full h-full overflow-hidden box-border；所有后代元素都必须留在区域边界内。
+最大图标边长 {max_icon}px，最大字号 {max_font}px，单侧 padding 不超过 {max_padding}px；必要时使用 Tailwind 3 arbitrary values（如 w-[20px]、text-[11px]）。
+如果空间不足，优先省略次要文字或减少间距，禁止通过溢出、超大固定尺寸或超出边界来容纳内容。
+"""
+
 def generate_code(bbox_tree, img_path, bot, instructions):
     """Generates code for each leaf node in the bounding box tree."""
     img = Image.open(img_path)
@@ -87,7 +121,9 @@ def generate_code(bbox_tree, img_path, bot, instructions):
             
             node_type = node.get("type")
             if node_type and node_type in prompt_dict:
-                prompt = prompt_dict[node_type]
+                prompt = add_render_geometry_contract(
+                    prompt_dict[node_type], bbox, bbox_tree["bbox"]
+                )
                 try:
                     code = bot.ask(prompt, encode_image(cropped_img))
                     code_dict[node["id"]] = code
@@ -133,8 +169,16 @@ def generate_html(bbox_tree, output_file):
                 width: 100%;
                 height: 100%;
             }
+            .box[data-region-type="header"],
+            .box[data-region-type="navigation"] {
+                line-height: 1 !important;
+            }
+            .box[data-region-type="header"] *,
+            .box[data-region-type="navigation"] * {
+                line-height: 1 !important;
+            }
         </style>
-        <link href="https://cdn.jsdelivr.net/npm/tailwindcss@2.2.19/dist/tailwind.min.css" rel="stylesheet">
+        <script src="https://cdn.tailwindcss.com"></script>
     </head>
     <body>
         <div class="container">
@@ -156,7 +200,8 @@ def generate_html(bbox_tree, output_file):
         width = (bbox[2] - bbox[0]) / parent_width * 100
         height = (bbox[3] - bbox[1]) / parent_height * 100
 
-        html = f'<div id="{node_id}" class="box" style="left: {left}%; top: {top}%; width: {width}%; height: {height}%;">'
+        region_type = node.get("type", "container")
+        html = f'<div id="{node_id}" data-region-type="{region_type}" class="box" style="left: {left}%; top: {top}%; width: {width}%; height: {height}%;">'
         if children:
             html += '<div class="container">'
             current_width = bbox[2] - bbox[0]
@@ -195,7 +240,9 @@ def generate_code_parallel(bbox_tree, img_path, bot, instructions):
                 # Select prompt based on node type
                 if "type" in node:
                     if node["type"] in prompt_dict:
-                        prompt = prompt_dict[node["type"]]
+                        prompt = add_render_geometry_contract(
+                            prompt_dict[node["type"]], bbox, bbox_tree["bbox"]
+                        )
                     else:
                         print(f"Unknown component type: {node['type']}")
                         code_dict[node["id"]] = f"<!-- Unknown component type: {node['type']} -->"
@@ -265,7 +312,9 @@ def main():
     output_dir = os.path.join(base_dir, 'data', 'output', args.run_id)
     os.makedirs(output_dir, exist_ok=True)
 
-    input_json_path = os.path.join(tmp_dir, f"{args.run_id}_bboxes.json")
+    region_json_path = os.path.join(tmp_dir, f"{args.run_id}_region_bboxes.json")
+    legacy_json_path = os.path.join(tmp_dir, f"{args.run_id}_bboxes.json")
+    input_json_path = region_json_path if os.path.exists(region_json_path) else legacy_json_path
     img_path = os.path.join(tmp_dir, f"{args.run_id}.png")
     output_html_path = os.path.join(output_dir, f"{args.run_id}_layout.html")
 
@@ -309,6 +358,21 @@ def main():
     # Model id is resolved by Doubao from ARK_MODEL_ID, with a safe default fallback.
     bot = Doubao(api_key)
     code_dict = generate_code_parallel(root, img_path, bot, user_instruction)
+    raw_generation_path = os.path.join(
+        output_dir, f"{args.run_id}_generation_raw.json"
+    )
+    with open(raw_generation_path, "w", encoding="utf-8") as f:
+        json.dump(
+            {
+                "model": bot.model,
+                "instructions": user_instruction,
+                "responses": code_dict,
+            },
+            f,
+            indent=2,
+            ensure_ascii=False,
+        )
+    print(f"Raw region generations saved to: {raw_generation_path}")
     code_substitution(output_html_path, code_dict)
 
     print(f"HTML layout with generated content saved to {os.path.basename(output_html_path)}")

screencoder/image_box_detection.py

@@ -59,7 +59,7 @@ def extract_bboxes_from_html_fallback(html_path: Path):
         
         # Extract placeholder bboxes
         placeholder_bboxes = []
-        placeholder_images = soup.find_all(class_='bg-gray-400')
+        placeholder_images = soup.select('.screencoder-image-placeholder')
         for i, img in enumerate(placeholder_images):
             # For fallback, we'll use a simple approach
             # In a real scenario, you'd need to parse the actual layout
@@ -75,8 +75,7 @@ def extract_bboxes_from_html_fallback(html_path: Path):
         return region_bboxes, placeholder_bboxes, 1280, 720
         
     except Exception as e:
-        print(f"Error in fallback HTML parsing: {e}")
-        return [], [], 1280, 720
+        raise RuntimeError(f"Fallback HTML parsing failed: {e}") from e
 
 # ---------- Main logic ----------
 async def extract_bboxes_from_html(html_path: Path):
@@ -97,19 +96,17 @@ async def extract_bboxes_from_html(html_path: Path):
                     browser = await p.chromium.launch(headless=True)
                 else:
                     print(f"Failed to install browser dependencies: {result.stderr}")
-                    # Return empty results to continue the pipeline
-                    return [], [], 1280, 720
+                    raise RuntimeError(f"Playwright browser install failed: {result.stderr}")
             except Exception as install_error:
                 print(f"Failed to install browser dependencies: {install_error}")
-                # Return empty results to continue the pipeline
-                return [], [], 1280, 720
+                raise RuntimeError(f"Playwright browser setup failed: {install_error}") from install_error
         
         try:
             ctx = await browser.new_context(
                 viewport={"width": 1280, "height": 720},
             )
             page = await ctx.new_page()
-            await page.goto(html_path.resolve().as_uri())
+            await page.goto(html_path.resolve().as_uri(), wait_until="networkidle")
 
             metrics = await page.evaluate("""
                 () => {
@@ -121,14 +118,26 @@ async def extract_bboxes_from_html(html_path: Path):
 
                     const placeholder_bboxes = [];
                     let ph_id_counter = 0;
-                    const all_potential_placeholders = document.querySelectorAll('.bg-gray-400');
+                    const dedicated = document.querySelectorAll('.screencoder-image-placeholder');
+                    const all_potential_placeholders = dedicated.length
+                        ? dedicated
+                        : document.querySelectorAll('.bg-gray-400');
+                    const candidate_placeholder_count = all_potential_placeholders.length;
+                    let invalid_placeholder_count = 0;
+                    let skipped_svg_count = 0;
+                    let outside_region_count = 0;
 
                     for (const el of all_potential_placeholders) {
                         // Apply the same filters as before
-                        if (el.tagName === 'SVG') continue;
-                        if (el.innerText && el.innerText.trim() !== '') continue;
-                        
+                        if (el.tagName === 'SVG') {
+                            skipped_svg_count += 1;
+                            continue;
+                        }
                         const el_rect = el.getBoundingClientRect();
+                        if (el_rect.width <= 0 || el_rect.height <= 0) {
+                            invalid_placeholder_count += 1;
+                            continue;
+                        }
                         const el_center = { x: el_rect.left + el_rect.width / 2, y: el_rect.top + el_rect.height / 2 };
                         
                         // Find which region this placeholder is inside
@@ -151,6 +160,8 @@ async def extract_bboxes_from_html(html_path: Path):
                                 h: el_rect.height,
                                 region_id: containing_region_id
                             });
+                        } else {
+                            outside_region_count += 1;
                         }
                     }
 
@@ -159,17 +170,36 @@ async def extract_bboxes_from_html(html_path: Path):
                         region_bboxes, 
                         placeholder_bboxes, 
                         layout_width: layout_rect.width, 
-                        layout_height: layout_rect.height 
+                        layout_height: layout_rect.height,
+                        candidate_placeholder_count,
+                        invalid_placeholder_count,
+                        skipped_svg_count,
+                        outside_region_count
                     };
                 }
             """)
+            if metrics['invalid_placeholder_count'] > 0:
+                raise RuntimeError(
+                    f"{metrics['invalid_placeholder_count']} placeholder(s) have zero rendered size"
+                )
+            if (
+                metrics['candidate_placeholder_count'] != len(metrics['placeholder_bboxes'])
+                or metrics['skipped_svg_count'] > 0
+                or metrics['outside_region_count'] > 0
+            ):
+                raise RuntimeError(
+                    "Placeholder contract violation: "
+                    f"candidates={metrics['candidate_placeholder_count']}, "
+                    f"detected={len(metrics['placeholder_bboxes'])}, "
+                    f"svg={metrics['skipped_svg_count']}, "
+                    f"outside_region={metrics['outside_region_count']}"
+                )
             await browser.close()
             return metrics['region_bboxes'], metrics['placeholder_bboxes'], metrics['layout_width'], metrics['layout_height']
         except Exception as e:
             print(f"Error during browser operation: {e}")
             await browser.close()
-            # Return empty results to continue the pipeline
-            return [], [], 1280, 720
+            raise RuntimeError(f"Playwright rendering failed: {e}") from e
 
 
 def draw_bboxes_on_image(img, region_bboxes, placeholder_bboxes):
@@ -225,7 +255,7 @@ def main():
     
     html_path = output_dir / f"{run_id}_layout.html"
     screenshot_path = tmp_dir / f"{run_id}.png"
-    output_json_path = tmp_dir / f"{run_id}_bboxes.json"
+    output_json_path = tmp_dir / f"{run_id}_placeholder_bboxes.json"
     debug_image_path = tmp_dir / f"debug_gray_bboxes_{run_id}.png"
 
     if not html_path.exists():
@@ -251,10 +281,7 @@ def main():
         )
         print("Successfully extracted bboxes using Playwright")
     except Exception as e:
-        print(f"Playwright failed: {e}")
-        print("Falling back to HTML parsing method...")
-        region_bboxes, placeholder_bboxes, layout_width, layout_height = extract_bboxes_from_html_fallback(html_path)
-        print("Successfully extracted bboxes using fallback method")
+        sys.exit(f"Error: Playwright bbox extraction failed: {e}")
     
     if not placeholder_bboxes:
         # This is not necessarily an error; some UIs might not have placeholders.

screencoder/image_replacer.py

@@ -99,7 +99,10 @@ def main():
     soup = BeautifulSoup(html_content, 'html.parser')
 
     # 1. Find all placeholder elements by their class, in document order.
-    placeholder_elements = soup.find_all(class_='bg-gray-400')
+    placeholder_elements = soup.find_all(class_='screencoder-image-placeholder')
+    if not placeholder_elements:
+        # Compatibility for artifacts generated before the dedicated marker existed.
+        placeholder_elements = soup.find_all(class_='bg-gray-400')
 
     # 2. Get the placeholder IDs from the mapping file in the correct, sorted order.
     def natural_sort_key(s):
@@ -115,16 +118,15 @@ def main():
     
     # 3. Check for count mismatches
     if len(placeholder_elements) != len(ordered_placeholder_ids):
-        print(f"Warning: Mismatch in counts! Found {len(placeholder_elements)} placeholder images in HTML, but {len(ordered_placeholder_ids)} mappings.")
+        sys.exit(
+            "Error: Placeholder/mapping count mismatch: "
+            f"html={len(placeholder_elements)}, mappings={len(ordered_placeholder_ids)}"
+        )
     else:
         print(f"Found {len(placeholder_elements)} placeholder images to replace.")
 
     # 4. Iterate through both lists, create a proper <img> tag, and replace the placeholder.
     for i, ph_element in enumerate(placeholder_elements):
-        if i >= len(ordered_placeholder_ids):
-            print(f"Warning: More placeholder images in HTML than mappings. Stopping at image {i+1}.")
-            break
-        
         ph_id = ordered_placeholder_ids[i]
         # Fix: Use the correct relative path from HTML file to image directory
         relative_img_path = f"{crop_dir.name}/{ph_id}.png"

screencoder/mapping.py

@@ -235,7 +235,9 @@ def main():
     base_dir = Path(__file__).parent.resolve()
     tmp_dir = base_dir / 'data' / 'tmp' / run_id
     
-    gray_json_path = tmp_dir / f"{run_id}_bboxes.json"
+    placeholder_json_path = tmp_dir / f"{run_id}_placeholder_bboxes.json"
+    legacy_json_path = tmp_dir / f"{run_id}_bboxes.json"
+    gray_json_path = placeholder_json_path if placeholder_json_path.exists() else legacy_json_path
     uied_json_path = tmp_dir / "ip" / f"{run_id}.json"
     mapping_output_path = tmp_dir / f"mapping_full_{run_id}.json"
     debug_src_path = tmp_dir / f"{run_id}.png"
@@ -314,6 +316,12 @@ def main():
     # 6. Report and save results
     print(f"Successfully created {total_mappings_count} one-to-one mappings out of {total_placeholders_count} placeholders.")
 
+    if total_mappings_count != total_placeholders_count:
+        raise RuntimeError(
+            "Incomplete placeholder mapping: "
+            f"mapped={total_mappings_count}, placeholders={total_placeholders_count}"
+        )
+
     mapping_output_path.write_text(json.dumps(final_results, indent=2, ensure_ascii=False))
     print(f"Mapping data written to {mapping_output_path}")