fix(article): improve image extraction for lazy-loaded and JSON-LD images

article_rewriter.py

@@ -519,33 +519,93 @@ def scrape_article(url: str) -> dict:
 
     # --- Extract images ---
     images = []
-    # Try og:image first
+
+    # 1. Try JSON-LD schema thumbnailUrl (most reliable for WordPress/Yoast SEO)
+    for script in soup.find_all("script", type="application/ld+json"):
+        if script.string:
+            try:
+                data = json.loads(script.string)
+                # Handle both single object and @graph array
+                if isinstance(data, dict):
+                    if "thumbnailUrl" in data:
+                        img_url = data["thumbnailUrl"]
+                        if not img_url.startswith("http"):
+                            img_url = urljoin(url, img_url)
+                        images.append(img_url)
+                        print(f"   🖼️  Found image in JSON-LD thumbnailUrl: {img_url}")
+                    # Check image object
+                    if "image" in data and isinstance(data["image"], dict) and "url" in data["image"]:
+                        img_url = data["image"]["url"]
+                        if not img_url.startswith("http"):
+                            img_url = urljoin(url, img_url)
+                        if img_url not in images:
+                            images.append(img_url)
+                            print(f"   🖼️  Found image in JSON-LD image.url: {img_url}")
+                elif isinstance(data, list):
+                    for item in data:
+                        if isinstance(item, dict):
+                            if "thumbnailUrl" in item:
+                                img_url = item["thumbnailUrl"]
+                                if not img_url.startswith("http"):
+                                    img_url = urljoin(url, img_url)
+                                if img_url not in images:
+                                    images.append(img_url)
+                                    print(f"   🖼️  Found image in JSON-LD graph thumbnailUrl")
+                            if "image" in item and isinstance(item["image"], dict) and "url" in item["image"]:
+                                img_url = item["image"]["url"]
+                                if not img_url.startswith("http"):
+                                    img_url = urljoin(url, img_url)
+                                if img_url not in images:
+                                    images.append(img_url)
+                                    print(f"   🖼️  Found image in JSON-LD graph image.url")
+            except (json.JSONDecodeError, KeyError, TypeError):
+                pass
+
+    # 2. Try og:image
     og_image = soup.find("meta", property="og:image")
     if og_image and og_image.get("content"):
         img_url = og_image["content"]
         if not img_url.startswith("http"):
             img_url = urljoin(url, img_url)
-        images.append(img_url)
+        if img_url not in images:
+            images.append(img_url)
+            print(f"   🖼️  Found og:image: {img_url}")
 
-    # Get images from article body
+    # 3. Get images from article body (handle lazy-loading attributes)
     if content_elem:
         for img in content_elem.find_all("img"):
-            src = img.get("src") or img.get("data-src") or img.get("data-lazy-src")
-            if src and not src.startswith("data:image"):
-                if not src.startswith("http"):
-                    src = urljoin(url, src)
-                # Skip tiny icons, avatars, tracking pixels (Increased size thresholds to drop ad blocks/thumbnails)
-                width = img.get("width", "")
-                height = img.get("height", "")
-                try:
-                    if width and int(width) < 300:
-                        continue
-                    if height and int(height) < 200:
-                        continue
-                except ValueError:
-                    pass
-                if src not in images:
-                    images.append(src)
+            # Check multiple lazy-load attribute patterns (order matters: real src first, then lazy attrs)
+            src = None
+            for attr in ["src", "data-src", "data-lazy-src", "data-original", "data-srcset"]:
+                val = img.get(attr, "")
+                if attr == "data-srcset":
+                    # Take first URL from srcset (format: "url 1024w, url2 300w")
+                    if val:
+                        src = val.split()[0] if val.split() else None
+                elif val and not val.startswith("data:image"):
+                    src = val
+                    break
+
+            if not src:
+                continue
+
+            if not src.startswith("http"):
+                src = urljoin(url, src)
+
+            # Skip tiny icons, avatars, tracking pixels
+            width = img.get("width", "")
+            height = img.get("height", "")
+            try:
+                if width and int(width) < 300:
+                    continue
+                if height and int(height) < 200:
+                    continue
+            except ValueError:
+                pass
+
+            if src not in images:
+                images.append(src)
+                print(f"   🖼️  Found image in article body: {src[:80]}...")
 
     # --- Extract embedded media (videos, social posts, audio) ---
     embeds = []