Add application file

.env.example

@@ -0,0 +1,12 @@
+# API Keys
+GROQ_API_KEY=your_groq_api_key_here
+
+# Network Configuration (optional, defaults to 5050)
+PORT=5050
+
+# CORS Configuration (comma-separated list of allowed frontend domains)
+# e.g., DIFFLENS_ORIGINS=https://your-username.github.io,http://localhost:5000
+DIFFLENS_ORIGINS=http://localhost:5000
+
+# Override for Public asset URLs (optional, defaults to auto-detecting the request host)
+# DIFFLENS_PUBLIC_URL=https://your-huggingface-space.hf.space

.gitignore

@@ -0,0 +1,18 @@
+# Local environment and secrets
+.env
+.secret_key
+
+# Python virtual environments
+.venv/
+venv/
+env/
+ENV/
+
+# Python cache files
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+
+# Local uploads & temporary folders
+uploads/

api.py

@@ -0,0 +1,154 @@
+import os
+import time
+import uuid
+from dotenv import load_dotenv
+load_dotenv()
+
+from flask import Flask, request, send_from_directory, jsonify, session
+from flask_cors import CORS
+from diff import compare_images
+from diff_ai import compare_images_ai
+from describe import describe_change
+from auth import (
+    authenticate, login_required, get_secret_key, current_user,
+)
+
+app = Flask(__name__)
+app.config["SECRET_KEY"] = get_secret_key()
+app.config["SESSION_COOKIE_HTTPONLY"] = True
+app.config["SESSION_COOKIE_SAMESITE"] = "Lax"
+
+# Allow the frontend origin to send cookies. For local dev we list the dev
+# origins; in production set DIFFLENS_ORIGINS env var to your domain.
+_origins_env = os.environ.get("DIFFLENS_ORIGINS")
+if _origins_env:
+    _origins = [o.strip() for o in _origins_env.split(",") if o.strip()]
+else:
+    _origins = [
+        "http://localhost:8000", "http://127.0.0.1:8000",
+        "http://localhost:5000", "http://127.0.0.1:5000",
+        "http://localhost:3000", "http://127.0.0.1:3000"
+    ]
+CORS(app, resources={r"/*": {"origins": _origins}}, supports_credentials=True)
+
+# Use /tmp/uploads on Linux/production environments (e.g. Hugging Face Spaces)
+if os.name != "nt" or os.environ.get("HF_HUB_DISABLE_SYMLINKS_WARNING"):
+    UPLOAD_FOLDER = "/tmp/uploads"
+else:
+    UPLOAD_FOLDER = "../uploads"
+os.makedirs(UPLOAD_FOLDER, exist_ok=True)
+
+
+def cleanup_old_files(folder, max_age_seconds=900):
+    """Clean up files in the given folder that are older than max_age_seconds."""
+    try:
+        now = time.time()
+        for filename in os.listdir(folder):
+            filepath = os.path.join(folder, filename)
+            if os.path.isfile(filepath):
+                stat = os.stat(filepath)
+                if now - stat.st_mtime > max_age_seconds:
+                    try:
+                        os.remove(filepath)
+                    except Exception as e:
+                        print(f"[app] Failed to remove {filename}: {e}")
+    except Exception as e:
+        print(f"[app] Cleanup failed: {e}")
+
+
+@app.route("/")
+def home():
+    return "Backend running"
+
+
+# ---------------- AUTH ----------------
+
+@app.route("/auth/login", methods=["POST", "OPTIONS"])
+def login():
+    if request.method == "OPTIONS":
+        return ("", 204)
+    data = request.get_json(silent=True) or {}
+    username = (data.get("username") or "").strip()
+    password = data.get("password") or ""
+    if not username or not password:
+        return jsonify({"error": "missing credentials"}), 400
+    if not authenticate(username, password):
+        return jsonify({"error": "invalid credentials"}), 401
+    session["user"] = username
+    session.permanent = True
+    return jsonify({"user": username})
+
+
+@app.route("/auth/logout", methods=["POST", "OPTIONS"])
+def logout():
+    if request.method == "OPTIONS":
+        return ("", 204)
+    session.clear()
+    return jsonify({"ok": True})
+
+
+@app.route("/auth/me", methods=["GET", "OPTIONS"])
+def me():
+    if request.method == "OPTIONS":
+        return ("", 204)
+    return jsonify({"user": current_user()})
+
+
+# ---------------- DIFF ----------------
+
+@app.route("/compare", methods=["POST", "OPTIONS"])
+def compare():
+    if request.method == "OPTIONS":
+        return jsonify({"status": "ok"}), 200
+
+    # Clean up old temp uploads
+    cleanup_old_files(UPLOAD_FOLDER)
+
+    if "image1" not in request.files or "image2" not in request.files:
+        return jsonify({"error": "Missing images"}), 400
+
+    img1 = request.files["image1"]
+    img2 = request.files["image2"]
+
+    # Use UUID to prevent image name collisions from concurrent users
+    req_id = str(uuid.uuid4())
+    path1 = os.path.join(UPLOAD_FOLDER, f"{req_id}_img1.jpg")
+    path2 = os.path.join(UPLOAD_FOLDER, f"{req_id}_img2.jpg")
+    img1.save(path1)
+    img2.save(path2)
+
+    output_path = os.path.join(UPLOAD_FOLDER, f"{req_id}_result.png")
+    heatmap_path = os.path.join(UPLOAD_FOLDER, f"{req_id}_heatmap.png")
+
+    try:
+        result_data = compare_images_ai(path1, path2, output_path, heatmap_path)
+    except Exception as e:
+        print(f"[app] AI pipeline failed ({e}); falling back to classical")
+        result_data = compare_images(path1, path2, output_path, heatmap_path)
+        result_data["pipeline"] = "classical"
+
+    description = describe_change(path1, path2, result_data)
+
+    # Construct the base URL dynamically so Hugging Face Space works without hardcoded domains
+    public_base = os.environ.get("DIFFLENS_PUBLIC_URL")
+    if not public_base:
+        proto = request.headers.get("X-Forwarded-Proto", request.scheme)
+        public_base = f"{proto}://{request.host}"
+    public_base = public_base.rstrip("/")
+
+    return jsonify({
+        "result":  f"{public_base}/uploads/{req_id}_result.png",
+        "heatmap": f"{public_base}/uploads/{req_id}_heatmap.png",
+        "metrics": result_data,
+        "description": description,
+    })
+
+
+@app.route('/uploads/<filename>')
+def uploaded_file(filename):
+    return send_from_directory(UPLOAD_FOLDER, filename)
+
+
+if __name__ == "__main__":
+    port = int(os.environ.get("PORT", 5050))
+    app.run(debug=True, host="0.0.0.0", port=port)

app.py

@@ -0,0 +1,128 @@
+import os
+import sys
+import tempfile
+import cv2
+import dotenv
+
+# Load environment variables (e.g. GROQ_API_KEY)
+dotenv.load_dotenv()
+
+# Add app.py directory and backend/ directory to system path so imports work in all layouts
+current_dir = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(current_dir)
+sys.path.append(os.path.join(current_dir, "backend"))
+
+from diff_ai import compare_images_ai
+from diff import compare_images
+from describe import describe_change
+
+import gradio as gr
+
+def gradio_compare(img1, img2):
+    if img1 is None or img2 is None:
+        return None, None, "Please upload both images.", {}
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        path1 = os.path.join(tmpdir, "img1.jpg")
+        path2 = os.path.join(tmpdir, "img2.jpg")
+        output_path = os.path.join(tmpdir, "result.png")
+        heatmap_path = os.path.join(tmpdir, "heatmap.png")
+
+        # Convert RGB inputs (Gradio) to BGR (OpenCV)
+        img1_bgr = cv2.cvtColor(img1, cv2.COLOR_RGB2BGR)
+        img2_bgr = cv2.cvtColor(img2, cv2.COLOR_RGB2BGR)
+
+        # Save images
+        cv2.imwrite(path1, img1_bgr)
+        cv2.imwrite(path2, img2_bgr)
+
+        # Run comparison pipeline
+        pipeline_used = "AI Pipeline"
+        try:
+            result_data = compare_images_ai(path1, path2, output_path, heatmap_path)
+        except Exception as e:
+            print(f"[Gradio] AI pipeline failed ({e}); falling back to classical")
+            result_data = compare_images(path1, path2, output_path, heatmap_path)
+            result_data["pipeline"] = "classical"
+            pipeline_used = "Classical Pipeline"
+
+        # Generate description
+        try:
+            description = describe_change(path1, path2, result_data)
+        except Exception as e:
+            description = f"Error generating description: {e}"
+
+        # Load output images (convert BGR back to RGB for Gradio display)
+        result_img = None
+        heatmap_img = None
+
+        if os.path.exists(output_path):
+            result_img = cv2.cvtColor(cv2.imread(output_path), cv2.COLOR_BGR2RGB)
+        if os.path.exists(heatmap_path):
+            heatmap_img = cv2.cvtColor(cv2.imread(heatmap_path), cv2.COLOR_BGR2RGB)
+
+        # Format metrics dictionary
+        metrics = {
+            "Pipeline Used": pipeline_used,
+            "SSIM Score (Similarity)": f"{result_data.get('ssim_score', 0):.4f}" if isinstance(result_data.get('ssim_score'), (int, float)) else "N/A",
+            "LPIPS Distance (Perceptual)": f"{result_data.get('lpips_max', 0):.4f}" if isinstance(result_data.get('lpips_max'), (int, float)) else "N/A",
+            "Detected Object Changes": result_data.get("object_changes", "0"),
+            "Detected Crack Count": result_data.get("crack_count", "0"),
+            "Change Severity": str(result_data.get("severity", "none")).upper(),
+            "Alignment Failed": "Yes" if result_data.get("alignment_failed") else "No"
+        }
+
+        return result_img, heatmap_img, description, metrics
+
+# UI Theme and Layout Design
+theme = gr.themes.Soft(
+    primary_hue="indigo",
+    secondary_hue="slate",
+    neutral_hue="slate"
+)
+
+css_styling = """
+.container {
+    max-width: 1200px;
+    margin: 0 auto;
+}
+.header {
+    text-align: center;
+    margin-bottom: 2rem;
+}
+"""
+
+with gr.Blocks() as demo:
+    gr.Markdown(
+        """
+        # 🔍 DiffLens AI — Visual Comparison & Change Analysis
+        Compare **Before** and **After** images of structures, scenes, or objects. The system aligns the images, detects change zones, runs segmentation (SAM2) and semantic filtering (DINOv2), and writes a detailed natural language explanation of changes.
+        """,
+        elem_classes="header"
+    )
+    
+    with gr.Row():
+        with gr.Column(scale=1):
+            gr.Markdown("### Input Images")
+            img1_input = gr.Image(label="Before Image", type="numpy")
+            img2_input = gr.Image(label="After Image", type="numpy")
+            submit_btn = gr.Button("Compare & Analyze", variant="primary", size="lg")
+            
+        with gr.Column(scale=1):
+            gr.Markdown("### Results")
+            with gr.Tabs():
+                with gr.TabItem("Visual Change Detection"):
+                    result_img_output = gr.Image(label="Change Boundaries (Bbox & Segments)")
+                    heatmap_img_output = gr.Image(label="Difference Heatmap")
+                with gr.TabItem("AI Interpretation & Metrics"):
+                    desc_output = gr.Textbox(label="AI Explanation of What Changed", lines=6)
+                    metrics_output = gr.JSON(label="Quantitative Comparison Metrics")
+                    
+    submit_btn.click(
+        fn=gradio_compare,
+        inputs=[img1_input, img2_input],
+        outputs=[result_img_output, heatmap_img_output, desc_output, metrics_output]
+    )
+
+if __name__ == "__main__":
+    demo.launch(theme=theme, css=css_styling)

auth.py

@@ -0,0 +1,87 @@
+"""
+Simple username + password authentication for DiffLens.
+
+User credentials are stored in users.json next to this file:
+  {"users": {"alice": "$2b$12$..."}}
+
+The hash is bcrypt. Sessions use Flask's signed cookie (SECRET_KEY must be
+set in the environment for production).
+"""
+import json
+import os
+import secrets
+from functools import wraps
+
+import bcrypt
+from flask import session, jsonify, request
+
+USERS_FILE = os.path.join(os.path.dirname(__file__), "users.json")
+
+
+def _load_users():
+    if not os.path.exists(USERS_FILE):
+        return {}
+    with open(USERS_FILE, "r") as f:
+        data = json.load(f) or {}
+    return data.get("users", {})
+
+
+def _save_users(users):
+    with open(USERS_FILE, "w") as f:
+        json.dump({"users": users}, f, indent=2)
+
+
+def hash_password(plain):
+    return bcrypt.hashpw(plain.encode("utf-8"), bcrypt.gensalt(12)).decode("ascii")
+
+
+def verify_password(plain, hashed):
+    try:
+        return bcrypt.checkpw(plain.encode("utf-8"), hashed.encode("utf-8"))
+    except Exception:
+        return False
+
+
+def create_user(username, password):
+    users = _load_users()
+    users[username] = hash_password(password)
+    _save_users(users)
+
+
+def authenticate(username, password):
+    users = _load_users()
+    h = users.get(username)
+    if not h:
+        return False
+    return verify_password(password, h)
+
+
+def get_secret_key():
+    """Returns the Flask SECRET_KEY. Reads from env, falls back to a file
+    that's auto-generated on first run so the key persists across restarts."""
+    key = os.environ.get("DIFFLENS_SECRET_KEY")
+    if key:
+        return key
+    path = os.path.join(os.path.dirname(__file__), ".secret_key")
+    if os.path.exists(path):
+        with open(path, "r") as f:
+            return f.read().strip()
+    key = secrets.token_urlsafe(48)
+    with open(path, "w") as f:
+        f.write(key)
+    os.chmod(path, 0o600)
+    return key
+
+
+def login_required(view):
+    """Decorator that returns 401 for unauthenticated requests."""
+    @wraps(view)
+    def wrapper(*args, **kwargs):
+        if not session.get("user"):
+            return jsonify({"error": "unauthorized"}), 401
+        return view(*args, **kwargs)
+    return wrapper
+
+
+def current_user():
+    return session.get("user")

create_user.py

@@ -0,0 +1,16 @@
+"""
+CLI: create or update a DiffLens user.
+
+Usage:
+  python3 create_user.py <username> <password>
+"""
+import sys
+from auth import create_user
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        print("Usage: python3 create_user.py <username> <password>")
+        sys.exit(1)
+    username, password = sys.argv[1], sys.argv[2]
+    create_user(username, password)
+    print(f"User '{username}' created/updated.")

describe.py

@@ -0,0 +1,168 @@
+"""
+VLM-based change description via Ollama (Llama 3.2 Vision).
+
+Llama 3.2 Vision (mllama) only accepts ONE image per request, so we
+concatenate the before and after images into a single side-by-side
+composite with labels and send that.
+
+Requires:
+  - Ollama running locally on port 11434
+  - llama3.2-vision model pulled (`ollama pull llama3.2-vision`)
+"""
+import base64
+import io
+import json
+import os
+import urllib.request
+import urllib.error
+
+import cv2
+import numpy as np
+
+OLLAMA_URL = "http://localhost:11434/api/generate"
+OLLAMA_MODEL = "llama3.2-vision:latest"
+GROQ_URL = "https://api.groq.com/openai/v1/chat/completions"
+GROQ_MODEL = "meta-llama/llama-4-scout-17b-16e-instruct"
+TIMEOUT_SECONDS = 180
+
+
+def _make_side_by_side(img1_path, img2_path, max_height=384):
+    """Build a single composite image with BEFORE | AFTER labels.
+    Returns base64-encoded PNG, or None on failure."""
+    a = cv2.imread(img1_path)
+    b = cv2.imread(img2_path)
+    if a is None or b is None:
+        return None
+
+    # Normalize heights
+    def resize_to_height(img, h):
+        scale = h / img.shape[0]
+        return cv2.resize(img, (int(round(img.shape[1] * scale)), h),
+                          interpolation=cv2.INTER_AREA)
+
+    a = resize_to_height(a, max_height)
+    b = resize_to_height(b, max_height)
+
+    # Header strip with labels
+    header_h = 50
+    total_w = a.shape[1] + b.shape[1] + 20  # 20px gap between
+    header = np.full((header_h, total_w, 3), 30, dtype=np.uint8)
+    cv2.putText(header, "BEFORE", (a.shape[1] // 2 - 70, 35),
+                cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 255, 255), 2)
+    cv2.putText(header, "AFTER", (a.shape[1] + 20 + b.shape[1] // 2 - 60, 35),
+                cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 255, 255), 2)
+
+    # Glue panels: a, gap, b
+    gap = np.full((max_height, 20, 3), 30, dtype=np.uint8)
+    body = np.hstack([a, gap, b])
+    composite = np.vstack([header, body])
+
+    ok, buf = cv2.imencode(".png", composite)
+    if not ok:
+        return None
+    return base64.b64encode(buf.tobytes()).decode("ascii")
+
+
+def describe_change(img1_path, img2_path, metrics):
+    """Returns a plain-English description string, or None on failure.
+    Uses Groq API if GROQ_API_KEY is in env, else falls back to local Ollama."""
+    groq_api_key = os.environ.get("GROQ_API_KEY")
+
+    try:
+        composite_b64 = _make_side_by_side(img1_path, img2_path)
+        if composite_b64 is None:
+            return None
+
+        prompt_lines = [
+            "This single image contains TWO photos placed side by side.",
+            "The LEFT photo is labeled BEFORE. The RIGHT photo is labeled AFTER.",
+            "Describe ONLY what physically changed from BEFORE to AFTER in 1-3 short sentences.",
+            "Be specific about objects (color, shape, position).",
+            "If nothing meaningful changed, say 'No visible changes.'",
+            "Do not list things that are the same in both photos.",
+        ]
+        if metrics:
+            added   = metrics.get("added")
+            removed = metrics.get("removed")
+            if added is not None or removed is not None:
+                prompt_lines.append(
+                    f"\nDetector hint: added={added}, removed={removed}."
+                )
+        prompt = "\n".join(prompt_lines)
+
+        if groq_api_key:
+            # --- Use Groq API ---
+            payload = {
+                "model": GROQ_MODEL,
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {"type": "text", "text": prompt},
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": f"data:image/png;base64,{composite_b64}"
+                                }
+                            }
+                        ]
+                    }
+                ],
+                "temperature": 0.2,
+                "max_tokens": 120
+            }
+            req = urllib.request.Request(
+                GROQ_URL,
+                data=json.dumps(payload).encode("utf-8"),
+                headers={
+                    "Content-Type": "application/json",
+                    "Authorization": f"Bearer {groq_api_key}",
+                    "User-Agent": "Mozilla/5.0"
+                },
+            )
+            with urllib.request.urlopen(req, timeout=TIMEOUT_SECONDS) as resp:
+                body = json.loads(resp.read().decode("utf-8"))
+            text = (body["choices"][0]["message"]["content"] or "").strip()
+            print("[describe] generated description using Groq API")
+            return text or None
+        else:
+            # --- Fallback to local Ollama ---
+            payload = {
+                "model": OLLAMA_MODEL,
+                "prompt": prompt,
+                "images": [composite_b64],
+                "stream": False,
+                "options": {
+                    "temperature": 0.2,
+                    "num_predict": 120,
+                    "num_ctx": 2048,
+                },
+                "keep_alive": "10m",
+            }
+            req = urllib.request.Request(
+                OLLAMA_URL,
+                data=json.dumps(payload).encode("utf-8"),
+                headers={"Content-Type": "application/json"},
+            )
+            with urllib.request.urlopen(req, timeout=TIMEOUT_SECONDS) as resp:
+                body = json.loads(resp.read().decode("utf-8"))
+            text = (body.get("response") or "").strip()
+            print("[describe] generated description using local Ollama")
+            return text or None
+
+    except urllib.error.HTTPError as e:
+        try:
+            err_body = e.read().decode("utf-8")
+            print(f"[describe] Groq API HTTPError {e.code}: {err_body}")
+        except Exception:
+            print(f"[describe] Groq API HTTPError {e}: (could not read body)")
+        return None
+    except urllib.error.URLError as e:
+        if groq_api_key:
+            print(f"[describe] Groq API error/unreachable: {e}")
+        else:
+            print(f"[describe] Ollama unreachable: {e}")
+        return None
+    except Exception as e:
+        print(f"[describe] error: {e}")
+        return None

diff.py

@@ -0,0 +1,347 @@
+import cv2
+import numpy as np
+from skimage.metrics import structural_similarity as ssim
+
+# -----------------------------
+# LPIPS — perceptual similarity (lazy-loaded so first request pays the cost)
+# -----------------------------
+_lpips_model = None
+_lpips_torch = None
+
+def _get_lpips():
+    """Lazy-load LPIPS + torch on first use. Returns (model, torch) or (None, None)
+    if the dependencies aren't installed — pipeline still works without it."""
+    global _lpips_model, _lpips_torch
+    if _lpips_model is not None:
+        return _lpips_model, _lpips_torch
+    try:
+        import torch
+        import lpips
+        _lpips_torch = torch
+        # 'alex' is the fastest backbone; 'vgg' is more accurate but slower.
+        _lpips_model = lpips.LPIPS(net='alex', verbose=False).eval()
+    except Exception:
+        _lpips_model = False  # sentinel: tried and failed
+        _lpips_torch = None
+    return _lpips_model, _lpips_torch
+
+
+def lpips_distance(patch1_bgr, patch2_bgr):
+    """Returns LPIPS perceptual distance in [0, ~1]. Higher = more different.
+    Returns None if LPIPS isn't available."""
+    model, torch = _get_lpips()
+    if not model:
+        return None
+
+    def to_tensor(bgr):
+        rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
+        # LPIPS expects 64x64+ float in [-1, 1], shape (1,3,H,W)
+        rgb = cv2.resize(rgb, (128, 128))
+        t = torch.from_numpy(rgb).float().permute(2, 0, 1) / 127.5 - 1.0
+        return t.unsqueeze(0)
+
+    with torch.no_grad():
+        d = model(to_tensor(patch1_bgr), to_tensor(patch2_bgr))
+    return float(d.item())
+
+
+# -----------------------------
+# ALIGNMENT (ORB homography + ECC sub-pixel refinement)
+# -----------------------------
+def align_images(img1, img2):
+    gray1 = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)
+    gray2 = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY)
+
+    orb = cv2.ORB_create(3000)
+    kp1, des1 = orb.detectAndCompute(gray1, None)
+    kp2, des2 = orb.detectAndCompute(gray2, None)
+
+    aligned = img2  # fallback
+
+    if des1 is not None and des2 is not None:
+        matcher = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True)
+        matches = matcher.match(des1, des2)
+
+        if len(matches) >= 10:
+            matches = sorted(matches, key=lambda x: x.distance)[:50]
+            pts1 = np.float32([kp1[m.queryIdx].pt for m in matches])
+            pts2 = np.float32([kp2[m.trainIdx].pt for m in matches])
+            matrix, _ = cv2.findHomography(pts2, pts1, cv2.RANSAC)
+
+            if matrix is not None:
+                aligned = cv2.warpPerspective(
+                    img2, matrix, (img1.shape[1], img1.shape[0])
+                )
+
+    # ---- ECC refinement, self-validating ----
+    # Only keep the ECC result if it actually IMPROVED alignment
+    # (measured by SSIM of the aligned images). Otherwise discard.
+    try:
+        g1 = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)
+        g2_before = cv2.cvtColor(aligned, cv2.COLOR_BGR2GRAY)
+        before_score, _ = ssim(g1, g2_before, full=True)
+
+        warp = np.eye(3, 3, dtype=np.float32)
+        criteria = (
+            cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT,
+            100,
+            1e-5,
+        )
+        _, warp = cv2.findTransformECC(
+            g1, g2_before, warp, cv2.MOTION_HOMOGRAPHY, criteria, None, 5
+        )
+        candidate = cv2.warpPerspective(
+            aligned,
+            warp,
+            (img1.shape[1], img1.shape[0]),
+            flags=cv2.INTER_LINEAR + cv2.WARP_INVERSE_MAP,
+        )
+        g2_after = cv2.cvtColor(candidate, cv2.COLOR_BGR2GRAY)
+        after_score, _ = ssim(g1, g2_after, full=True)
+
+        if after_score > before_score:
+            aligned = candidate
+    except cv2.error:
+        # ECC can fail to converge — keep ORB-only result in that case
+        pass
+
+    return aligned
+
+
+# -----------------------------
+# SSIM-BASED STRUCTURAL DIFF
+# -----------------------------
+def ssim_diff(gray1, gray2):
+    """Returns a uint8 'difference' map where bright = structurally different."""
+    score, diff_map = ssim(gray1, gray2, full=True)
+    # ssim returns [-1, 1] where 1 == identical; invert + scale to 0..255
+    diff_map = (1.0 - diff_map) * 255.0
+    return score, diff_map.astype(np.uint8)
+
+
+# -----------------------------
+# CRACK DETECTION (IMPROVED)
+# -----------------------------
+def detect_cracks(gray_img):
+    enhanced = cv2.equalizeHist(gray_img)
+
+    edges = cv2.Canny(enhanced, 40, 120)
+
+    kernel = np.ones((3,3), np.uint8)
+    edges = cv2.morphologyEx(edges, cv2.MORPH_CLOSE, kernel, iterations=1)
+
+    return edges
+
+
+# -----------------------------
+# MAIN FUNCTION
+# -----------------------------
+def compare_images(img1_path, img2_path, output_path, heatmap_path):
+
+    img1 = cv2.imread(img1_path)
+    img2 = cv2.imread(img2_path)
+
+    img2 = cv2.resize(img2, (img1.shape[1], img1.shape[0]))
+
+    # 🔥 ALIGN
+    aligned_img2 = align_images(img1, img2)
+
+    gray1 = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)
+    gray2 = cv2.cvtColor(aligned_img2, cv2.COLOR_BGR2GRAY)
+
+    # Moderate blur — enough to absorb pixel noise / alignment jitter, but
+    # small enough that handwritten letters and other tiny changes survive.
+    blur1 = cv2.GaussianBlur(gray1, (5, 5), 0)
+    blur2 = cv2.GaussianBlur(gray2, (5, 5), 0)
+
+    # -----------------------------
+    # DIFF — SSIM with a large window is the workhorse here.
+    # A big win=21 window means a single new pixel doesn't trigger a hit;
+    # only regions where the *local structure* genuinely changed do.
+    # -----------------------------
+    ssim_score, struct_diff = ssim_diff(blur1, blur2)
+
+    diff = struct_diff
+    heatmap = cv2.applyColorMap(diff, cv2.COLORMAP_JET)
+
+    # -----------------------------
+    # OBJECT DETECTION — shape-aware filtering
+    # -----------------------------
+    _, thresh = cv2.threshold(diff, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
+
+    # OPEN first: erodes thin alignment-edge ribbons away entirely.
+    # Then CLOSE: reconnects fragments of the real new object.
+    # Smaller OPEN kernel preserves small changes (handwritten letters, marks).
+    # Median blur first — single best speckle killer for textured surfaces
+    # (wood grain, carpet, asphalt) where SSIM picks up tiny texture jitter.
+    thresh = cv2.medianBlur(thresh, 7)
+
+    open_k  = np.ones((7, 7), np.uint8)
+    close_k = np.ones((15, 15), np.uint8)
+    thresh = cv2.morphologyEx(thresh, cv2.MORPH_OPEN,  open_k,  iterations=2)
+    thresh = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, close_k, iterations=2)
+
+    result = aligned_img2.copy()
+    img_area = gray1.shape[0] * gray1.shape[1]
+
+    # ---- Global alignment-failure guard ----
+    # If the diff covers a huge portion of the image, alignment is broken
+    # (camera moved/rotated too much between shots). Bail out cleanly
+    # instead of drawing one giant useless box.
+    diff_ratio = cv2.countNonZero(thresh) / img_area
+    alignment_failed = diff_ratio > 0.70
+
+    contours, _ = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+    if alignment_failed:
+        contours = []  # skip detection entirely
+    # Allow tiny changes (single letters, marks). LPIPS verification below
+    # is what keeps false positives out at this size.
+    min_area = max(400, img_area * 0.0002)  # ~0.02% of image
+
+    # Score each contour and only keep "blob-shaped" ones (real objects),
+    # not "ribbon-shaped" ones (edge jitter).
+    candidates = []
+    for cnt in contours:
+        area = cv2.contourArea(cnt)
+        if area < min_area:
+            continue
+
+        x, y, w, h = cv2.boundingRect(cnt)
+
+        # Solidity = contour area / convex hull area. Real objects ≈ 0.6+.
+        # Thin ribbons along edges have very low solidity.
+        hull_area = cv2.contourArea(cv2.convexHull(cnt))
+        solidity = area / hull_area if hull_area > 0 else 0
+
+        # Extent = contour area / bounding-box area. Slivers ≈ low extent.
+        extent = area / (w * h) if w * h > 0 else 0
+
+        # Aspect ratio sanity: skip extreme slivers
+        aspect = max(w, h) / max(1, min(w, h))
+
+        # LARGE-BLOB BYPASS: any region covering >2% of the image is
+        # almost certainly a real change (added/removed object). Trust it
+        # unconditionally — don't apply shape filters that could reject
+        # irregular shapes (e.g. paper partially occluded by a plant).
+        is_large = area > img_area * 0.02
+
+        # Relaxed thresholds — small handwritten marks can be irregular.
+        # LPIPS does the final perceptual gate.
+        if not is_large and (solidity < 0.30 or extent < 0.18 or aspect > 12):
+            continue
+
+        candidates.append((area, x, y, w, h))
+
+    # Keep only the top few biggest blobs — the "real" changes.
+    candidates.sort(reverse=True)
+    candidates = candidates[:15]  # raise cap; LPIPS will prune below
+
+    # -----------------------------
+    # LPIPS PERCEPTUAL VERIFICATION
+    # For each surviving candidate region, check that the patch in img2 is
+    # *perceptually* different from the same region in img1. This kills the
+    # last layer of false positives (lighting shifts, residual jitter on
+    # textured surfaces) that survived the SSIM + shape filter.
+    # -----------------------------
+    LPIPS_THRESHOLD = 0.15  # tuned: <0.10 = ~identical, >0.20 = clearly different
+    verified = []
+    lpips_scores = []
+
+    for area, x, y, w, h in candidates:
+        # Pad the crop slightly so LPIPS sees context, not just the object edge
+        pad = 10
+        x0 = max(0, x - pad)
+        y0 = max(0, y - pad)
+        x1 = min(img1.shape[1], x + w + pad)
+        y1 = min(img1.shape[0], y + h + pad)
+
+        patch1 = img1[y0:y1, x0:x1]
+        patch2 = aligned_img2[y0:y1, x0:x1]
+        if patch1.size == 0 or patch2.size == 0:
+            continue
+
+        d = lpips_distance(patch1, patch2)
+
+        # If LPIPS is unavailable (None), trust the upstream pipeline and keep.
+        # If available, only keep candidates that are perceptually different.
+        if d is None or d >= LPIPS_THRESHOLD:
+            verified.append((area, x, y, w, h))
+            if d is not None:
+                lpips_scores.append(d)
+
+    object_changes = len(verified)
+
+    for _, x, y, w, h in verified:
+        cv2.rectangle(result, (x, y), (x + w, y + h), (0, 0, 255), 3)
+
+    # -----------------------------
+    # CRACK DETECTION (ONLY IN CHANGED AREA)
+    # -----------------------------
+    cracks = detect_cracks(gray2)
+
+    # 🔥 FIX 4 APPLIED
+    cracks = cv2.bitwise_and(cracks, cracks, mask=thresh)
+
+    crack_contours, _ = cv2.findContours(cracks, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+    crack_count = 0
+    total_crack_area = 0
+    max_crack_area = 0
+
+    # Only run crack analysis if NO new objects were detected. When a real
+    # object appears, every edge of it would otherwise be flagged as a "crack".
+    if object_changes == 0:
+        for cnt in crack_contours:
+            area = cv2.contourArea(cnt)
+
+            if 80 < area < 1500:
+                x, y, w, h = cv2.boundingRect(cnt)
+
+                if w < 60 and h < 60:
+                    crack_count += 1
+                    total_crack_area += area
+                    max_crack_area = max(max_crack_area, area)
+
+                    cv2.rectangle(result, (x, y), (x+w, y+h), (255,0,0), 2)
+
+    # -----------------------------
+    # SEVERITY SCORING
+    # >=1 real object change → HIGH. Otherwise → NONE.
+    # -----------------------------
+    if object_changes >= 1 or crack_count >= 1:
+        severity = "HIGH"
+    else:
+        severity = "NONE"
+
+    # -----------------------------
+    # DRAW LABEL
+    # -----------------------------
+    color = (0,255,0)
+
+    if severity == "MEDIUM":
+        color = (0,255,255)
+    elif severity == "HIGH":
+        color = (0,0,255)
+
+    label = f"Severity: {severity}"
+    if alignment_failed:
+        label = "ALIGNMENT FAILED — retake from same angle"
+        color = (0, 165, 255)
+    cv2.putText(result, label, (20, 40),
+                cv2.FONT_HERSHEY_SIMPLEX, 1, color, 3)
+
+    # Save outputs
+    cv2.imwrite(output_path, result)
+    cv2.imwrite(heatmap_path, heatmap)
+
+    return {
+        "crack_count": crack_count,
+        "total_crack_area": int(total_crack_area),
+        "max_crack_area": int(max_crack_area),
+        "object_changes": object_changes,
+        "ssim_score": float(ssim_score),
+        "lpips_max": float(max(lpips_scores)) if lpips_scores else None,
+        "lpips_available": lpips_scores != [] or _get_lpips()[0] is not False,
+        "alignment_failed": bool(alignment_failed),
+        "severity": severity
+    }

diff_ai copy.py

@@ -0,0 +1,622 @@
+"""
+AI-first image comparison pipeline (change-region-driven).
+
+Strategy (different from naive segment-everything-then-match):
+  1. LoFTR (kornia) aligns img2 onto img1
+  2. SSIM produces a binary "change mask" — where pixels actually differ
+  3. Connected components on the change mask give us discrete change regions
+  4. For each region:
+       a. Crop a context patch from both images
+       b. DINOv2 verifies perceptual difference (kills shadow/lighting noise)
+       c. SAM2 in PROMPT mode (point prompt at the region centroid) returns
+          a tight object mask in img2
+       d. Edge-density heuristic classifies as ADDED vs REMOVED
+  5. REMOVED and ADDED objects are matched against each other to detect
+     MOVED objects, using a globally-greedy bipartite match over DINOv2
+     similarity (see _match_moved_objects)
+  6. Draw color-coded bounding boxes from the SAM masks
+
+Why this is better than the previous attempt:
+  - No SAM auto-mask-generation (10x faster — only prompted on change regions)
+  - No cross-image DINO matching (no phantom added/removed pairs)
+  - Change mask is the source of truth; SAM and DINO act as refinement layers
+  - Pipeline degrades gracefully: if SAM/DINO fail, the change-region bboxes
+    are still usable as fallback output
+
+All models are lazy-loaded. Falls back to classical pipeline (diff.py) if any
+required model is unavailable — see app.py for the wiring.
+"""
+import cv2
+import numpy as np
+from skimage.metrics import structural_similarity as ssim
+
+# =====================================================================
+# CONFIG / CONSTANTS
+# =====================================================================
+
+# Drawing colors (BGR)
+COLOR_ADDED   = (0, 200,   0)
+COLOR_REMOVED = (0,   0, 220)
+COLOR_MOVED   = (255, 255,  0)   # cyan in BGR  (drawn on both from & to boxes)
+
+LOFTR_LONG_EDGE = 640
+MAX_LONG_EDGE = 960
+
+DINO_SIM_THRESHOLD = 0.92   # same-location patches: >= this => unchanged, drop
+MOVED_SIM_THRESHOLD = 0.75  # cross-location patches: >= this => same object, moved
+EDGE_DENSITY_THRESHOLD = 1.0
+
+REGION_PAD = 12       # padding when cropping a change region for verification
+MOVE_MATCH_PAD = 12   # padding when cropping removed/added objects for move-matching
+
+
+# =====================================================================
+# LAZY MODEL LOADERS
+# =====================================================================
+
+_loftr = None
+
+def _get_loftr():
+    """Lazy-load kornia LoFTR (transformer dense matcher)."""
+    global _loftr
+    if _loftr is not None:
+        return _loftr
+    try:
+        import torch
+        import kornia.feature as KF
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        matcher = KF.LoFTR(pretrained="outdoor").eval().to(device)
+        _loftr = {"matcher": matcher, "torch": torch, "device": device}
+        print("[diff_ai] LoFTR loaded")
+    except Exception as e:
+        print(f"[diff_ai] LoFTR unavailable: {e}")
+        _loftr = False
+    return _loftr
+
+
+_sam = None
+
+def _get_sam():
+    """Lazy-load ultralytics SAM2 (used in prompted mode, not auto-mask)."""
+    global _sam
+    if _sam is not None:
+        return _sam
+    try:
+        from ultralytics import SAM
+        _sam = SAM("sam2_t.pt")  # ~150MB, auto-downloads
+        print("[diff_ai] SAM2 loaded")
+    except Exception as e:
+        print(f"[diff_ai] SAM2 unavailable: {e}")
+        _sam = False
+    return _sam
+
+
+_dinov2 = None
+
+def _get_dinov2():
+    """Lazy-load DINOv2 via HuggingFace transformers (Python 3.9 compatible)."""
+    global _dinov2
+    if _dinov2 is not None:
+        return _dinov2
+    try:
+        import torch
+        from transformers import AutoModel, AutoImageProcessor
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        processor = AutoImageProcessor.from_pretrained(
+            "facebook/dinov2-small", use_fast=True
+        )
+        model = AutoModel.from_pretrained("facebook/dinov2-small").eval().to(device)
+        _dinov2 = {
+            "model": model, "processor": processor,
+            "torch": torch, "device": device,
+        }
+        print("[diff_ai] DINOv2 loaded")
+    except Exception as e:
+        print(f"[diff_ai] DINOv2 unavailable: {e}")
+        _dinov2 = False
+    return _dinov2
+
+
+# =====================================================================
+# STAGE 1 — LoFTR ALIGNMENT
+# =====================================================================
+
+def _to_loftr_tensor(bgr, torch_mod, device):
+    gray = cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY)
+    h, w = gray.shape
+    scale = LOFTR_LONG_EDGE / max(h, w)
+    nh, nw = int(round(h * scale)), int(round(w * scale))
+    nh = max(8, nh - nh % 8)
+    nw = max(8, nw - nw % 8)
+    resized = cv2.resize(gray, (nw, nh))
+    t = torch_mod.from_numpy(resized).float()[None, None] / 255.0
+    return t.to(device), scale
+
+
+def align_loftr(img1_bgr, img2_bgr):
+    """Warp img2 onto img1 using LoFTR matches + RANSAC homography.
+
+    Returns (aligned_bgr, valid_mask) on success, or (None, None) on
+    failure. valid_mask is a uint8 mask (255 = valid, 0 = invalid) the
+    same size as img1, marking which pixels of `aligned_bgr` actually
+    came from img2 vs. the black border warpPerspective fills in when
+    the two images aren't framed identically. Callers should restrict
+    any pixel-level comparison to valid_mask == 255, otherwise that
+    black border gets treated as a huge fake "removed" region."""
+    bundle = _get_loftr()
+    if not bundle:
+        return None, None
+    matcher = bundle["matcher"]
+    torch   = bundle["torch"]
+    device  = bundle["device"]
+
+    try:
+        t1, s1 = _to_loftr_tensor(img1_bgr, torch, device)
+        t2, s2 = _to_loftr_tensor(img2_bgr, torch, device)
+        with torch.no_grad():
+            corr = matcher({"image0": t1, "image1": t2})
+
+        kp1 = corr["keypoints0"].cpu().numpy()
+        kp2 = corr["keypoints1"].cpu().numpy()
+        confidence = corr["confidence"].cpu().numpy()
+
+        mask = confidence > 0.5
+        kp1 = kp1[mask] / s1
+        kp2 = kp2[mask] / s2
+
+        if len(kp1) < 10:
+            print(f"[diff_ai] LoFTR: only {len(kp1)} confident matches")
+            return None, None
+
+        H, _ = cv2.findHomography(kp2, kp1, cv2.RANSAC, 5.0)
+        if H is None:
+            return None, None
+
+        out_size = (img1_bgr.shape[1], img1_bgr.shape[0])
+        aligned = cv2.warpPerspective(img2_bgr, H, out_size)
+
+        # Warp an all-white mask through the same H to find which pixels
+        # of `aligned` are real img2 content vs. the black fill border.
+        valid = np.full(img2_bgr.shape[:2], 255, dtype=np.uint8)
+        valid_mask = cv2.warpPerspective(valid, H, out_size)
+        # Erode a bit so interpolation-blurred edge pixels near the
+        # border aren't counted as valid either.
+        valid_mask = cv2.erode(valid_mask, np.ones((9, 9), np.uint8))
+
+        return aligned, valid_mask
+    except Exception as e:
+        print(f"[diff_ai] LoFTR alignment error: {e}")
+        return None, None
+
+
+# =====================================================================
+# STAGE 2 — CHANGE REGION EXTRACTION (SSIM + connected components)
+# =====================================================================
+
+def extract_change_regions(img1_bgr, aligned_bgr, valid_mask=None):
+    """Returns (heatmap_bgr, list of {bbox, area, centroid}, binmask,
+    alignment_failed). Each region is a connected blob of pixels that
+    significantly differ.
+
+    valid_mask (optional): uint8 mask from align_loftr marking which
+    pixels of aligned_bgr are real warped content vs. warpPerspective's
+    black fill border. When provided, the border is excluded from the
+    change mask entirely so it can never be picked up as a fake
+    "removed" region, and low valid coverage also counts toward
+    alignment_failed."""
+    g1 = cv2.cvtColor(img1_bgr,   cv2.COLOR_BGR2GRAY)
+    g2 = cv2.cvtColor(aligned_bgr, cv2.COLOR_BGR2GRAY)
+    g1 = cv2.GaussianBlur(g1, (5, 5), 0)
+    g2 = cv2.GaussianBlur(g2, (5, 5), 0)
+
+    _, diff = ssim(g1, g2, full=True)
+    diff_u8 = np.clip((1.0 - diff) * 255.0, 0, 255).astype(np.uint8)
+    heatmap = cv2.applyColorMap(diff_u8, cv2.COLORMAP_JET)
+
+    # Otsu binarize + clean noise
+    _, binmask = cv2.threshold(diff_u8, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
+    binmask = cv2.medianBlur(binmask, 7)
+    open_k  = np.ones((5, 5), np.uint8)
+    close_k = np.ones((15, 15), np.uint8)
+    binmask = cv2.morphologyEx(binmask, cv2.MORPH_OPEN,  open_k,  iterations=1)
+    binmask = cv2.morphologyEx(binmask, cv2.MORPH_CLOSE, close_k, iterations=2)
+
+    img_area = img1_bgr.shape[0] * img1_bgr.shape[1]
+
+    if valid_mask is not None:
+        # Never let the warpPerspective border be treated as a change
+        binmask = cv2.bitwise_and(binmask, valid_mask)
+        # If a large chunk of the frame has no real img2 coverage at all,
+        # the homography barely overlaps the frame — treat as a failure
+        # the same way a too-large diff_ratio would be.
+        coverage_ratio = cv2.countNonZero(valid_mask) / img_area
+        if coverage_ratio < 0.5:
+            print(f"[diff_ai] low valid coverage after warp ({coverage_ratio:.2f}) — "
+                  f"treating as alignment failure")
+            return heatmap, [], binmask, True
+
+    # Global alignment-failure guard
+    diff_ratio = cv2.countNonZero(binmask) / img_area
+    alignment_failed = diff_ratio > 0.55
+
+    # Connected components → individual change regions
+    n_labels, labels, stats, centroids = cv2.connectedComponentsWithStats(
+        binmask, connectivity=8
+    )
+
+    min_area = max(400, int(img_area * 0.0005))   # >= 0.05% of image
+    max_area = int(img_area * 0.6)                # < 60% (skip background)
+
+    regions = []
+    if not alignment_failed:
+        for i in range(1, n_labels):  # skip label 0 (background)
+            x = int(stats[i, cv2.CC_STAT_LEFT])
+            y = int(stats[i, cv2.CC_STAT_TOP])
+            w = int(stats[i, cv2.CC_STAT_WIDTH])
+            h = int(stats[i, cv2.CC_STAT_HEIGHT])
+            area = int(stats[i, cv2.CC_STAT_AREA])
+            if area < min_area or area > max_area:
+                continue
+            cx = float(centroids[i, 0])
+            cy = float(centroids[i, 1])
+            regions.append({
+                "bbox": (x, y, w, h),
+                "area": area,
+                "centroid": (cx, cy),
+            })
+
+    return heatmap, regions, binmask, alignment_failed
+
+
+# =====================================================================
+# STAGE 3 — DINOv2 PERCEPTUAL VERIFICATION
+# =====================================================================
+
+def dinov2_similarity(patch1_bgr, patch2_bgr):
+    """Returns cosine similarity in [0, 1] between two patches.
+    Higher = more perceptually similar. Used to drop change regions caused
+    by lighting/shadow rather than real object changes, and to match
+    removed/added objects for MOVED detection."""
+    bundle = _get_dinov2()
+    if not bundle:
+        return None
+    model     = bundle["model"]
+    processor = bundle["processor"]
+    torch     = bundle["torch"]
+    device    = bundle["device"]
+
+    if patch1_bgr.size == 0 or patch2_bgr.size == 0:
+        return None
+    try:
+        rgb1 = cv2.cvtColor(patch1_bgr, cv2.COLOR_BGR2RGB)
+        rgb2 = cv2.cvtColor(patch2_bgr, cv2.COLOR_BGR2RGB)
+        inputs = processor(images=[rgb1, rgb2], return_tensors="pt").to(device)
+        with torch.no_grad():
+            out = model(**inputs)
+        feats = out.last_hidden_state[:, 0]            # CLS token, (2, D)
+        feats = feats / feats.norm(dim=1, keepdim=True)
+        sim = float((feats[0] * feats[1]).sum().item())
+        return sim
+    except Exception as e:
+        print(f"[diff_ai] DINOv2 sim error: {e}")
+        return None
+
+
+# =====================================================================
+# STAGE 4 — SAM2 PROMPTED SEGMENTATION
+# =====================================================================
+
+def _sam_prompt_single(img_bgr, point_xy):
+    """Run SAM2 with one point prompt. Returns binary uint8 mask or None."""
+    sam = _get_sam()
+    if not sam:
+        return None
+    try:
+        results = sam(
+            img_bgr,
+            points=[[float(point_xy[0]), float(point_xy[1])]],
+            labels=[1],
+            verbose=False,
+        )
+        if not results:
+            return None
+        r = results[0]
+        if r.masks is None or len(r.masks.data) == 0:
+            return None
+        masks_np = r.masks.data.cpu().numpy()
+        areas = [int((m > 0.5).sum()) for m in masks_np]
+        if not areas:
+            return None
+        best_idx = int(np.argmax(areas))
+        return (masks_np[best_idx] > 0.5).astype(np.uint8)
+    except Exception:
+        return None
+
+
+def sam_mask_at_point(img_bgr, point_xy, fallback_bbox=None):
+    """Run SAM2 with a single positive point prompt. Returns the largest
+    returned mask as a binary uint8 array, or None on failure.
+
+    If the centroid prompt produces an empty/noisy mask and fallback_bbox
+    is provided, tries additional points across the region and picks the
+    largest mask (multi-point fallback for edge cases where the centroid
+    lands on background — e.g. ring-shaped change regions)."""
+    mask = _sam_prompt_single(img_bgr, point_xy)
+    if mask is not None and int(mask.sum()) > 100:
+        return mask
+
+    if fallback_bbox is not None:
+        x, y, w, h = fallback_bbox
+        margin = 0.25
+        candidates = [
+            (x + w * margin, y + h * margin),
+            (x + w * (1.0 - margin), y + h * margin),
+            (x + w * margin, y + h * (1.0 - margin)),
+            (x + w * (1.0 - margin), y + h * (1.0 - margin)),
+            (x + w * 0.5, y + h * 0.5),
+        ]
+        best_mask = None
+        best_area = 0
+        for px, py in candidates:
+            m = _sam_prompt_single(img_bgr, (px, py))
+            if m is not None:
+                a = int(m.sum())
+                if a > best_area:
+                    best_area = a
+                    best_mask = m
+        if best_mask is not None:
+            return best_mask
+
+    return mask
+
+
+def bbox_from_mask(mask):
+    """Tight bbox (x, y, w, h) around a binary mask, or None if empty."""
+    ys, xs = np.where(mask > 0)
+    if len(ys) == 0:
+        return None
+    x0, x1 = int(xs.min()), int(xs.max())
+    y0, y1 = int(ys.min()), int(ys.max())
+    return (x0, y0, x1 - x0 + 1, y1 - y0 + 1)
+
+
+# =====================================================================
+# STAGE 5 — ADDED vs REMOVED CLASSIFICATION
+# =====================================================================
+
+def classify_added_or_removed(patch1_bgr, patch2_bgr,
+                               edge_threshold=EDGE_DENSITY_THRESHOLD):
+    """Edge-density heuristic. The image with more edges in this region
+    is the one that has the 'object'.
+    Returns 'added' (img2 has more), 'removed' (img1 has more),
+    or None if both patches have near-zero edge variance (noise)."""
+    g1 = cv2.cvtColor(patch1_bgr, cv2.COLOR_BGR2GRAY)
+    g2 = cv2.cvtColor(patch2_bgr, cv2.COLOR_BGR2GRAY)
+    e1 = float(cv2.Laplacian(g1, cv2.CV_64F).var())
+    e2 = float(cv2.Laplacian(g2, cv2.CV_64F).var())
+    if e1 < edge_threshold and e2 < edge_threshold:
+        return None
+    return "added" if e2 >= e1 else "removed"
+
+
+# =====================================================================
+# STAGE 6 — MOVED-OBJECT MATCHING (removed <-> added)
+# =====================================================================
+
+def _crop_patch(img_bgr, bbox, pad):
+    """Crop bbox from img_bgr with padding, clipped to image bounds."""
+    x, y, w, h = bbox
+    x0 = max(0, x - pad)
+    y0 = max(0, y - pad)
+    x1 = min(img_bgr.shape[1], x + w + pad)
+    y1 = min(img_bgr.shape[0], y + h + pad)
+    return img_bgr[y0:y1, x0:x1]
+
+
+def _bboxes_overlap(bbox_a, bbox_b):
+    """AABB intersection test. Overlapping removed/added regions are
+    treated as an in-place change rather than a move, so they're never
+    considered as a MOVED pair."""
+    ax, ay, aw, ah = bbox_a
+    bx, by, bw, bh = bbox_b
+    return ax < bx + bw and ax + aw > bx and ay < by + bh and ay + ah > by
+
+
+def _match_moved_objects(removed_objects, added_objects, img1_bgr, aligned_bgr,
+                          pad=MOVE_MATCH_PAD, sim_threshold=MOVED_SIM_THRESHOLD):
+    """Match REMOVED objects against ADDED objects to find MOVED objects.
+
+    Fix vs. the previous implementation: the old version iterated
+    `removed_objects` in (arbitrary, scan-order-derived) list order and let
+    each removed object greedily grab its own single best-matching added
+    object before moving to the next. That's order-dependent — an earlier
+    removed object could claim a mediocre match and starve a later removed
+    object of the added object it should have matched to, and swapping the
+    order of `removed_objects` could change the result.
+
+    This version computes the *entire* removed x added similarity matrix
+    up front (skipping overlapping pairs), then commits pairs in
+    descending similarity order — the single best candidate pair anywhere
+    in the matrix is matched first, then the next best remaining pair, and
+    so on. This is order-independent and much closer to an optimal
+    bipartite matching than the old per-item greedy approach.
+
+    Returns (moved_objects, surviving_removed, surviving_added).
+    """
+    if not removed_objects or not added_objects:
+        return [], list(removed_objects), list(added_objects)
+
+    candidates = []  # (similarity, removed_idx, added_idx)
+    for ri, rem in enumerate(removed_objects):
+        rem_patch = _crop_patch(img1_bgr, rem["bbox"], pad)
+        for ai, add in enumerate(added_objects):
+            if _bboxes_overlap(rem["bbox"], add["bbox"]):
+                continue
+            add_patch = _crop_patch(aligned_bgr, add["bbox"], pad)
+            sim = dinov2_similarity(rem_patch, add_patch)
+            if sim is not None and sim >= sim_threshold:
+                candidates.append((sim, ri, ai))
+
+    # Best pairs first, order-independent
+    candidates.sort(key=lambda c: c[0], reverse=True)
+
+    matched_removed = set()
+    matched_added = set()
+    moved_objects = []
+    for sim, ri, ai in candidates:
+        if ri in matched_removed or ai in matched_added:
+            continue
+        matched_removed.add(ri)
+        matched_added.add(ai)
+        moved_objects.append({
+            "from": removed_objects[ri],
+            "to": added_objects[ai],
+            "similarity": sim,
+        })
+        print(f"  MOVED: from {removed_objects[ri]['centroid']} "
+              f"-> to {added_objects[ai]['centroid']}  sim={sim:.3f}")
+
+    surviving_removed = [r for i, r in enumerate(removed_objects) if i not in matched_removed]
+    surviving_added   = [a for i, a in enumerate(added_objects)   if i not in matched_added]
+
+    for i, rem in enumerate(removed_objects):
+        if i not in matched_removed:
+            print(f"  REMOVED (unmatched) @{rem['centroid']}")
+
+    return moved_objects, surviving_removed, surviving_added
+
+
+# =====================================================================
+# DRAWING
+# =====================================================================
+
+def _draw_box(img, bbox, color, label):
+    x, y, w, h = bbox
+    cv2.rectangle(img, (x, y), (x + w, y + h), color, 3)
+    (tw, th), _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2)
+    cv2.rectangle(img, (x, y - th - 8), (x + tw + 8, y), color, -1)
+    cv2.putText(img, label, (x + 4, y - 4),
+                cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2)
+
+
+# =====================================================================
+# MAIN ENTRY
+# =====================================================================
+
+def compare_images_ai(img1_path, img2_path, output_path, heatmap_path):
+    """End-to-end change-region-driven AI pipeline. Raises RuntimeError
+    if any required model is unavailable."""
+    if not _get_loftr() or not _get_sam() or not _get_dinov2():
+        raise RuntimeError("AI pipeline unavailable — required model missing")
+
+    img1 = cv2.imread(img1_path)
+    img2 = cv2.imread(img2_path)
+    if img1 is None or img2 is None:
+        raise RuntimeError("Failed to read input images")
+
+    # Normalize sizes + downscale to keep models fast
+    img2 = cv2.resize(img2, (img1.shape[1], img1.shape[0]))
+
+    h0, w0 = img1.shape[:2]
+    if max(h0, w0) > MAX_LONG_EDGE:
+        s = MAX_LONG_EDGE / max(h0, w0)
+        new_size = (int(w0 * s), int(h0 * s))
+        img1 = cv2.resize(img1, new_size, interpolation=cv2.INTER_AREA)
+        img2 = cv2.resize(img2, new_size, interpolation=cv2.INTER_AREA)
+
+    # 1) LoFTR alignment
+    aligned, valid_mask = align_loftr(img1, img2)
+    if aligned is None:
+        raise RuntimeError("LoFTR alignment failed")
+
+    # 2) Change region extraction (SSIM + CC), excluding the warp border
+    heatmap, regions, _, alignment_failed = extract_change_regions(img1, aligned, valid_mask)
+    if alignment_failed:
+        regions = []
+    print(f"[diff_ai] {len(regions)} change region(s) before verification")
+
+    # 3) For each region: DINOv2 verify + SAM2 prompted segmentation
+    added_objects = []
+    removed_objects = []
+
+    for r in regions:
+        x, y, w, h = r["bbox"]
+        x0 = max(0, x - REGION_PAD)
+        y0 = max(0, y - REGION_PAD)
+        x1 = min(img1.shape[1], x + w + REGION_PAD)
+        y1 = min(img1.shape[0], y + h + REGION_PAD)
+        patch1 = img1[y0:y1, x0:x1]
+        patch2 = aligned[y0:y1, x0:x1]
+
+        # 3a) DINOv2 perceptual gate
+        sim = dinov2_similarity(patch1, patch2)
+        if sim is not None and sim >= DINO_SIM_THRESHOLD:
+            print(f"  region @{r['centroid']}: DINOv2 sim={sim:.3f} — perceptually same, dropping")
+            continue
+
+        # 3b) Classify as added vs removed via edge density
+        label = classify_added_or_removed(patch1, patch2)
+        if label is None:
+            print(f"  region @{r['centroid']}: edge variance too low on both sides, skipping")
+            continue
+
+        # 3c) SAM2 prompted at the centroid for a clean object mask
+        target_img = aligned if label == "added" else img1
+        mask = sam_mask_at_point(target_img, r["centroid"], r["bbox"])
+        tight_bbox = bbox_from_mask(mask) if mask is not None else r["bbox"]
+
+        entry = {
+            "bbox": tight_bbox,
+            "centroid": r["centroid"],
+            "dino_sim": sim,
+            "mask": mask,
+            "source": target_img,
+        }
+        if label == "added":
+            added_objects.append(entry)
+        else:
+            removed_objects.append(entry)
+
+        print(f"  region @{r['centroid']}: sim={sim} -> {label}")
+
+    # 4) MOVED object detection: globally-greedy match REMOVED <-> ADDED
+    moved_objects, removed_objects, added_objects = _match_moved_objects(
+        removed_objects, added_objects, img1, aligned
+    )
+
+    n_moved   = len(moved_objects)
+    n_added   = len(added_objects)
+    n_removed = len(removed_objects)
+    n_total   = n_added + n_removed + n_moved
+    severity  = "HIGH" if n_total > 0 else "NONE"
+
+    # 5) Draw boxes
+    result_img = aligned.copy()
+
+    for a in added_objects:
+        _draw_box(result_img, a["bbox"], COLOR_ADDED, "ADDED")
+    for r in removed_objects:
+        _draw_box(result_img, r["bbox"], COLOR_REMOVED, "REMOVED")
+    for m in moved_objects:
+        _draw_box(result_img, m["from"]["bbox"], COLOR_MOVED, "MOVED")
+        _draw_box(result_img, m["to"]["bbox"],   COLOR_MOVED, "MOVED")
+
+    # Draw severity label in top-left
+    label = f"{severity}  +{n_added}  -{n_removed}  ~{n_moved}"
+    cv2.putText(result_img, label, (20, 42),
+                cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 255, 255), 5)
+    cv2.putText(result_img, label, (20, 42),
+                cv2.FONT_HERSHEY_SIMPLEX, 1.0, (0, 0, 0), 2)
+
+    cv2.imwrite(output_path, result_img)
+    cv2.imwrite(heatmap_path, heatmap)
+
+    return {
+        "pipeline": "ai",
+        "added": n_added,
+        "removed": n_removed,
+        "moved": n_moved,
+        "object_changes": n_total,
+        "severity": severity,
+        "alignment_failed": alignment_failed,
+    }

diff_ai.py

@@ -0,0 +1,640 @@
+"""
+AI-first image comparison pipeline (change-region-driven).
+
+Strategy (different from naive segment-everything-then-match):
+  1. LoFTR (kornia) aligns img2 onto img1
+  2. SSIM produces a binary "change mask" — where pixels actually differ
+  3. Connected components on the change mask give us discrete change regions
+  4. For each region:
+       a. Crop a context patch from both images
+       b. DINOv2 verifies perceptual difference (kills shadow/lighting noise)
+       c. SAM2 in PROMPT mode (point prompt at the region centroid) returns
+          a tight object mask in img2
+       d. Edge-density heuristic classifies as ADDED vs REMOVED
+  5. Draw color-coded bounding boxes / masks (falling back to the raw change
+     bbox if SAM couldn't produce a mask, so a single failed SAM call never
+     causes a real change to vanish from the output)
+
+Why this is better than the previous attempt:
+  - No SAM auto-mask-generation (10x faster — only prompted on change regions)
+  - No cross-image DINO matching (no phantom added/removed pairs)
+  - Change mask is the source of truth; SAM and DINO act as refinement layers
+  - Pipeline degrades gracefully: if SAM/DINO fail, the change-region bboxes
+    are still drawn as fallback output (not silently dropped)
+  - DINOv2 embeddings are batched: the perceptual gate embeds all regions in
+    one forward pass, and moved-object matching embeds each patch once and
+    compares via a similarity matrix instead of one forward pass per pair
+
+All models are lazy-loaded (thread-safely). Falls back to classical pipeline
+(diff.py) if any required model is unavailable — see app.py for the wiring.
+"""
+import threading
+
+import cv2
+import numpy as np
+from skimage.metrics import structural_similarity as ssim
+
+# =====================================================================
+# TUNABLE CONSTANTS
+# =====================================================================
+
+# Drawing colors (BGR)
+COLOR_ADDED   = (0, 200,   0)
+COLOR_REMOVED = (0,   0, 220)
+COLOR_MOVED   = (255, 255,  0)   # cyan in BGR  (drawn on both from & to boxes)
+
+LOFTR_LONG_EDGE   = 640          # working resolution for LoFTR matching
+LOFTR_MIN_MATCHES = 10           # below this, alignment is considered failed
+LOFTR_CONF_THRESH = 0.5
+
+MAX_LONG_EDGE = 960              # cap on working image size for the pipeline
+
+CHANGE_MIN_AREA_FRAC = 0.0005    # region must be >= this fraction of image area
+CHANGE_MAX_AREA_FRAC = 0.6       # region must be < this fraction (skip background)
+ALIGNMENT_FAIL_DIFF_RATIO = 0.55 # if more of the image "changed" than this, alignment likely failed
+
+DINO_SIM_THRESHOLD  = 0.85       # >= this means perceptually identical → drop region
+MOVED_SIM_THRESHOLD = 0.75       # >= this cosine sim links a removed/added pair as "moved"
+REGION_PAD = 12                  # px padding around a region when cropping context patches
+
+EDGE_VARIANCE_THRESHOLD = 1.0    # Laplacian-variance floor for added/removed classification
+
+# SAM mask is considered too small/noisy below this fraction of the prompt
+# image's area (relative, so it scales with input resolution).
+SAM_MIN_MASK_AREA_FRAC = 0.0005
+
+_MODEL_LOCK = threading.Lock()
+
+
+# =====================================================================
+# LAZY MODEL LOADERS (thread-safe)
+# =====================================================================
+
+_loftr = None
+
+def _get_loftr():
+    """Lazy-load kornia LoFTR (transformer dense matcher)."""
+    global _loftr
+    if _loftr is not None:
+        return _loftr
+    with _MODEL_LOCK:
+        if _loftr is not None:
+            return _loftr
+        try:
+            import torch
+            import kornia.feature as KF
+            device = "cuda" if torch.cuda.is_available() else "cpu"
+            matcher = KF.LoFTR(pretrained="outdoor").eval().to(device)
+            _loftr = {"matcher": matcher, "torch": torch, "device": device}
+            print("[diff_ai] LoFTR loaded")
+        except Exception as e:
+            print(f"[diff_ai] LoFTR unavailable: {e}")
+            _loftr = False
+    return _loftr
+
+
+_sam = None
+
+def _get_sam():
+    """Lazy-load ultralytics SAM2 (used in prompted mode, not auto-mask)."""
+    global _sam
+    if _sam is not None:
+        return _sam
+    with _MODEL_LOCK:
+        if _sam is not None:
+            return _sam
+        try:
+            import torch
+            from ultralytics import SAM
+            model = SAM("sam2_t.pt")  # ~150MB, auto-downloads
+            device = "cuda" if torch.cuda.is_available() else "cpu"
+            try:
+                model.to(device)
+            except Exception:
+                pass  # some ultralytics versions take device per-call instead
+            _sam = {"model": model, "device": device}
+            print(f"[diff_ai] SAM2 loaded on {device}")
+        except Exception as e:
+            print(f"[diff_ai] SAM2 unavailable: {e}")
+            _sam = False
+    return _sam
+
+
+_dinov2 = None
+
+def _get_dinov2():
+    """Lazy-load DINOv2 via HuggingFace transformers (Python 3.9 compatible)."""
+    global _dinov2
+    if _dinov2 is not None:
+        return _dinov2
+    with _MODEL_LOCK:
+        if _dinov2 is not None:
+            return _dinov2
+        try:
+            import torch
+            from transformers import AutoModel, AutoImageProcessor
+            device = "cuda" if torch.cuda.is_available() else "cpu"
+            processor = AutoImageProcessor.from_pretrained(
+                "facebook/dinov2-small", use_fast=True
+            )
+            model = AutoModel.from_pretrained("facebook/dinov2-small").eval().to(device)
+            _dinov2 = {
+                "model": model, "processor": processor,
+                "torch": torch, "device": device,
+            }
+            print("[diff_ai] DINOv2 loaded")
+        except Exception as e:
+            print(f"[diff_ai] DINOv2 unavailable: {e}")
+            _dinov2 = False
+    return _dinov2
+
+
+# =====================================================================
+# STAGE 1 — LoFTR ALIGNMENT
+# =====================================================================
+
+def _to_loftr_tensor(bgr, torch_mod, device):
+    gray = cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY)
+    h, w = gray.shape
+    scale = LOFTR_LONG_EDGE / max(h, w)
+    nh, nw = int(round(h * scale)), int(round(w * scale))
+    # Round down to a multiple of 8 (LoFTR requirement) without ever hitting 0.
+    nh = max(8, nh - nh % 8)
+    nw = max(8, nw - nw % 8)
+    resized = cv2.resize(gray, (nw, nh))
+    t = torch_mod.from_numpy(resized).float()[None, None] / 255.0
+    return t.to(device), (nw / w, nh / h)  # per-axis scale, resize isn't isotropic-safe
+
+
+def align_loftr(img1_bgr, img2_bgr):
+    """Warp img2 onto img1 using LoFTR matches + RANSAC homography."""
+    bundle = _get_loftr()
+    if not bundle:
+        return None
+    matcher = bundle["matcher"]
+    torch   = bundle["torch"]
+    device  = bundle["device"]
+
+    try:
+        t1, (sx1, sy1) = _to_loftr_tensor(img1_bgr, torch, device)
+        t2, (sx2, sy2) = _to_loftr_tensor(img2_bgr, torch, device)
+        with torch.no_grad():
+            corr = matcher({"image0": t1, "image1": t2})
+
+        kp1 = corr["keypoints0"].cpu().numpy()
+        kp2 = corr["keypoints1"].cpu().numpy()
+        confidence = corr["confidence"].cpu().numpy()
+
+        mask = confidence > LOFTR_CONF_THRESH
+        kp1 = kp1[mask] / np.array([sx1, sy1])
+        kp2 = kp2[mask] / np.array([sx2, sy2])
+
+        if len(kp1) < LOFTR_MIN_MATCHES:
+            print(f"[diff_ai] LoFTR: only {len(kp1)} confident matches")
+            return None
+
+        H, _ = cv2.findHomography(kp2, kp1, cv2.RANSAC, 5.0)
+        if H is None:
+            return None
+
+        return cv2.warpPerspective(
+            img2_bgr, H, (img1_bgr.shape[1], img1_bgr.shape[0])
+        )
+    except Exception as e:
+        print(f"[diff_ai] LoFTR alignment error: {e}")
+        return None
+
+
+# =====================================================================
+# STAGE 2 — CHANGE REGION EXTRACTION (SSIM + connected components)
+# =====================================================================
+
+def extract_change_regions(img1_bgr, aligned_bgr):
+    """Returns (heatmap_bgr, list of {bbox, area, centroid}, binmask, alignment_failed).
+    Each region is a connected blob of pixels that significantly differ."""
+    g1 = cv2.cvtColor(img1_bgr,   cv2.COLOR_BGR2GRAY)
+    g2 = cv2.cvtColor(aligned_bgr, cv2.COLOR_BGR2GRAY)
+    g1 = cv2.GaussianBlur(g1, (5, 5), 0)
+    g2 = cv2.GaussianBlur(g2, (5, 5), 0)
+
+    _, diff = ssim(g1, g2, full=True)
+    diff_u8 = np.clip((1.0 - diff) * 255.0, 0, 255).astype(np.uint8)
+    heatmap = cv2.applyColorMap(diff_u8, cv2.COLORMAP_JET)
+
+    # Otsu binarize + clean noise
+    _, binmask = cv2.threshold(diff_u8, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
+    binmask = cv2.medianBlur(binmask, 7)
+    open_k  = np.ones((5, 5), np.uint8)
+    close_k = np.ones((15, 15), np.uint8)
+    binmask = cv2.morphologyEx(binmask, cv2.MORPH_OPEN,  open_k,  iterations=1)
+    binmask = cv2.morphologyEx(binmask, cv2.MORPH_CLOSE, close_k, iterations=2)
+
+    # Global alignment-failure guard
+    img_area = img1_bgr.shape[0] * img1_bgr.shape[1]
+    diff_ratio = cv2.countNonZero(binmask) / img_area
+    alignment_failed = diff_ratio > ALIGNMENT_FAIL_DIFF_RATIO
+
+    # Connected components → individual change regions
+    n_labels, labels, stats, centroids = cv2.connectedComponentsWithStats(
+        binmask, connectivity=8
+    )
+
+    min_area = max(400, int(img_area * CHANGE_MIN_AREA_FRAC))
+    max_area = int(img_area * CHANGE_MAX_AREA_FRAC)
+
+    regions = []
+    if not alignment_failed:
+        for i in range(1, n_labels):  # skip label 0 (background)
+            x = int(stats[i, cv2.CC_STAT_LEFT])
+            y = int(stats[i, cv2.CC_STAT_TOP])
+            w = int(stats[i, cv2.CC_STAT_WIDTH])
+            h = int(stats[i, cv2.CC_STAT_HEIGHT])
+            area = int(stats[i, cv2.CC_STAT_AREA])
+            if area < min_area or area > max_area:
+                continue
+            cx = float(centroids[i, 0])
+            cy = float(centroids[i, 1])
+            regions.append({
+                "bbox": (x, y, w, h),
+                "area": area,
+                "centroid": (cx, cy),
+            })
+
+    return heatmap, regions, binmask, alignment_failed
+
+
+# =====================================================================
+# STAGE 3 — DINOv2 PERCEPTUAL VERIFICATION (batched)
+# =====================================================================
+
+def dinov2_embed_batch(patches_bgr):
+    """Embed a list of BGR patches in a single forward pass.
+    Returns an (N, D) L2-normalized numpy array, or None if the model is
+    unavailable or every patch is empty. Empty patches get an all-zero
+    embedding (guaranteed cosine sim of 0 with anything)."""
+    bundle = _get_dinov2()
+    if not bundle:
+        return None
+    model, processor = bundle["model"], bundle["processor"]
+    torch, device = bundle["torch"], bundle["device"]
+
+    valid_idx = [i for i, p in enumerate(patches_bgr) if p is not None and p.size > 0]
+    if not valid_idx:
+        return None
+
+    try:
+        rgb_imgs = [cv2.cvtColor(patches_bgr[i], cv2.COLOR_BGR2RGB) for i in valid_idx]
+        inputs = processor(images=rgb_imgs, return_tensors="pt").to(device)
+        with torch.no_grad():
+            out = model(**inputs)
+        feats = out.last_hidden_state[:, 0]  # CLS token, (n_valid, D)
+        feats = feats / feats.norm(dim=1, keepdim=True)
+        feats = feats.cpu().numpy()
+    except Exception as e:
+        print(f"[diff_ai] DINOv2 batch embed error: {e}")
+        return None
+
+    dim = feats.shape[1]
+    result = np.zeros((len(patches_bgr), dim), dtype=feats.dtype)
+    for slot, i in enumerate(valid_idx):
+        result[i] = feats[slot]
+    return result
+
+
+def dinov2_similarity(patch1_bgr, patch2_bgr):
+    """Cosine similarity in [-1, 1] between two patches (single-pair
+    convenience wrapper around the batched embedder). Higher = more
+    perceptually similar."""
+    embs = dinov2_embed_batch([patch1_bgr, patch2_bgr])
+    if embs is None:
+        return None
+    if not np.any(embs[0]) or not np.any(embs[1]):
+        return None
+    return float(np.dot(embs[0], embs[1]))
+
+
+# =====================================================================
+# STAGE 4 — SAM2 PROMPTED SEGMENTATION
+# =====================================================================
+
+def _sam_prompt_single(img_bgr, point_xy):
+    """Run SAM2 with one point prompt. Returns binary uint8 mask or None."""
+    bundle = _get_sam()
+    if not bundle:
+        return None
+    sam = bundle["model"]
+    try:
+        results = sam(
+            img_bgr,
+            points=[[float(point_xy[0]), float(point_xy[1])]],
+            labels=[1],
+            verbose=False,
+        )
+        if not results:
+            return None
+        r = results[0]
+        if r.masks is None or len(r.masks.data) == 0:
+            return None
+        masks_np = r.masks.data.cpu().numpy()
+        areas = [int((m > 0.5).sum()) for m in masks_np]
+        if not areas:
+            return None
+        best_idx = int(np.argmax(areas))
+        return (masks_np[best_idx] > 0.5).astype(np.uint8)
+    except Exception as e:
+        print(f"[diff_ai] SAM2 prompt error @{point_xy}: {e}")
+        return None
+
+
+def sam_mask_at_point(img_bgr, point_xy, fallback_bbox=None):
+    """Run SAM2 with a single positive point prompt. Returns the largest
+    returned mask as a binary uint8 array, or None on failure.
+
+    If the centroid prompt produces an empty/noisy mask and fallback_bbox
+    is provided, tries additional points across the region and picks the
+    largest mask (multi-point fallback for edge cases where the centroid
+    lands on background — e.g. ring-shaped change regions)."""
+    img_area = img_bgr.shape[0] * img_bgr.shape[1]
+    min_mask_area = max(50, int(img_area * SAM_MIN_MASK_AREA_FRAC))
+
+    mask = _sam_prompt_single(img_bgr, point_xy)
+    if mask is not None and int(mask.sum()) > min_mask_area:
+        return mask
+
+    if fallback_bbox is not None:
+        x, y, w, h = fallback_bbox
+        margin = 0.25
+        candidates = [
+            (x + w * margin, y + h * margin),
+            (x + w * (1.0 - margin), y + h * margin),
+            (x + w * margin, y + h * (1.0 - margin)),
+            (x + w * (1.0 - margin), y + h * (1.0 - margin)),
+            (x + w * 0.5, y + h * 0.5),
+        ]
+        best_mask = None
+        best_area = 0
+        for px, py in candidates:
+            m = _sam_prompt_single(img_bgr, (px, py))
+            if m is not None:
+                a = int(m.sum())
+                if a > best_area:
+                    best_area = a
+                    best_mask = m
+        if best_mask is not None:
+            return best_mask
+
+    return mask  # possibly None, possibly small — caller falls back to bbox
+
+
+def bbox_from_mask(mask):
+    """Tight bbox (x, y, w, h) around a binary mask, or None if empty."""
+    if mask is None:
+        return None
+    ys, xs = np.where(mask > 0)
+    if len(ys) == 0:
+        return None
+    x0, x1 = int(xs.min()), int(xs.max())
+    y0, y1 = int(ys.min()), int(ys.max())
+    return (x0, y0, x1 - x0 + 1, y1 - y0 + 1)
+
+
+# =====================================================================
+# STAGE 5 — ADDED vs REMOVED CLASSIFICATION
+# =====================================================================
+
+def classify_added_or_removed(patch1_bgr, patch2_bgr,
+                               edge_threshold=EDGE_VARIANCE_THRESHOLD):
+    """Edge-density heuristic. The image with more edges in this region
+    is the one that has the 'object'.
+    Returns 'added' (img2 has more), 'removed' (img1 has more),
+    or None if both patches have near-zero edge variance (noise)."""
+    g1 = cv2.cvtColor(patch1_bgr, cv2.COLOR_BGR2GRAY)
+    g2 = cv2.cvtColor(patch2_bgr, cv2.COLOR_BGR2GRAY)
+    e1 = float(cv2.Laplacian(g1, cv2.CV_64F).var())
+    e2 = float(cv2.Laplacian(g2, cv2.CV_64F).var())
+    if e1 < edge_threshold and e2 < edge_threshold:
+        return None
+    return "added" if e2 >= e1 else "removed"
+
+
+# =====================================================================
+# DRAWING
+# =====================================================================
+
+def _draw_box(img, bbox, color, label=None):
+    x, y, w, h = bbox
+    cv2.rectangle(img, (x, y), (x + w, y + h), color, 3)
+    if label:
+        (tw, th), _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2)
+        ty = max(th + 8, y)
+        cv2.rectangle(img, (x, ty - th - 8), (x + tw + 8, ty), color, -1)
+        cv2.putText(img, label, (x + 4, ty - 4),
+                    cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2)
+
+
+def _composite_object(base, object_img, mask, bbox, color):
+    """Paste object pixels from object_img onto a darkened base using mask,
+    with a colored outline. If mask is missing/empty (SAM failed or was
+    unavailable), fall back to drawing the raw change-region bbox so the
+    change is never silently dropped from the output. Modifies base in place."""
+    if mask is not None and int(mask.sum()) > 0:
+        mask_bin = mask.astype(bool)
+        base[mask_bin] = object_img[mask_bin]
+        contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+        cv2.drawContours(base, contours, -1, color, 2)
+    elif bbox is not None:
+        x, y, w, h = bbox
+        base[y:y + h, x:x + w] = object_img[y:y + h, x:x + w]
+        _draw_box(base, bbox, color)
+    return base
+
+
+# =====================================================================
+# MAIN ENTRY
+# =====================================================================
+
+def compare_images_ai(img1_path, img2_path, output_path, heatmap_path):
+    """End-to-end change-region-driven AI pipeline. Raises RuntimeError
+    if any required model is unavailable."""
+    if not _get_loftr() or not _get_sam() or not _get_dinov2():
+        raise RuntimeError("AI pipeline unavailable — required model missing")
+
+    img1 = cv2.imread(img1_path)
+    img2 = cv2.imread(img2_path)
+    if img1 is None or img2 is None:
+        raise RuntimeError("Failed to read input images")
+
+    if img1.shape[:2] != img2.shape[:2]:
+        print(f"[diff_ai] warning: input sizes differ {img1.shape[:2]} vs "
+              f"{img2.shape[:2]}; resizing img2 to match img1 (may distort)")
+        img2 = cv2.resize(img2, (img1.shape[1], img1.shape[0]))
+
+    h0, w0 = img1.shape[:2]
+    if max(h0, w0) > MAX_LONG_EDGE:
+        s = MAX_LONG_EDGE / max(h0, w0)
+        new_size = (int(w0 * s), int(h0 * s))
+        img1 = cv2.resize(img1, new_size, interpolation=cv2.INTER_AREA)
+        img2 = cv2.resize(img2, new_size, interpolation=cv2.INTER_AREA)
+
+    # 1) LoFTR alignment
+    aligned = align_loftr(img1, img2)
+    if aligned is None:
+        raise RuntimeError("LoFTR alignment failed")
+
+    # 2) Change region extraction (SSIM + CC)
+    heatmap, regions, _, alignment_failed = extract_change_regions(img1, aligned)
+    if alignment_failed:
+        regions = []
+    print(f"[diff_ai] {len(regions)} change region(s) before verification")
+
+    # 3) Batched DINOv2 perceptual gate across all regions at once
+    patches1, patches2 = [], []
+    for r in regions:
+        x, y, w, h = r["bbox"]
+        x0 = max(0, x - REGION_PAD); y0 = max(0, y - REGION_PAD)
+        x1 = min(img1.shape[1], x + w + REGION_PAD)
+        y1 = min(img1.shape[0], y + h + REGION_PAD)
+        r["patch_bounds"] = (x0, y0, x1, y1)
+        patches1.append(img1[y0:y1, x0:x1])
+        patches2.append(aligned[y0:y1, x0:x1])
+
+    emb1 = dinov2_embed_batch(patches1)
+    emb2 = dinov2_embed_batch(patches2)
+
+    added_objects = []
+    removed_objects = []
+
+    for i, r in enumerate(regions):
+        x0, y0, x1, y1 = r["patch_bounds"]
+        patch1, patch2 = patches1[i], patches2[i]
+
+        # 3a) DINOv2 perceptual gate
+        sim = None
+        if emb1 is not None and emb2 is not None and np.any(emb1[i]) and np.any(emb2[i]):
+            sim = float(np.dot(emb1[i], emb2[i]))
+        if sim is not None and sim >= DINO_SIM_THRESHOLD:
+            print(f"  region @{r['centroid']}: DINOv2 sim={sim:.3f} — perceptually same, dropping")
+            continue
+
+        # 3b) Classify as added vs removed via edge density
+        label = classify_added_or_removed(patch1, patch2)
+        if label is None:
+            print(f"  region @{r['centroid']}: edge variance too low on both sides, skipping")
+            continue
+
+        # 3c) SAM2 prompted at the centroid for a clean object mask
+        target_img = aligned if label == "added" else img1
+        mask = sam_mask_at_point(target_img, r["centroid"], r["bbox"])
+        tight_bbox = bbox_from_mask(mask) or r["bbox"]
+
+        entry = {
+            "bbox": tight_bbox,
+            "centroid": r["centroid"],
+            "dino_sim": sim,
+            "mask": mask,
+            "source": target_img,
+        }
+        if label == "added":
+            added_objects.append(entry)
+        else:
+            removed_objects.append(entry)
+
+        print(f"  region @{r['centroid']}: sim={sim} -> {label}"
+              f"{' (no SAM mask, using bbox fallback)' if mask is None else ''}")
+
+    # 4) MOVED object detection: match REMOVED <-> ADDED pairs.
+    # Embed every candidate patch once, then compare via a similarity matrix
+    # instead of one DINOv2 forward pass per (removed, added) pair.
+    rem_patches, add_patches = [], []
+    for rem in removed_objects:
+        rx, ry, rw, rh = rem["bbox"]
+        r_x0 = max(0, rx - REGION_PAD); r_y0 = max(0, ry - REGION_PAD)
+        r_x1 = min(img1.shape[1], rx + rw + REGION_PAD)
+        r_y1 = min(img1.shape[0], ry + rh + REGION_PAD)
+        rem_patches.append(img1[r_y0:r_y1, r_x0:r_x1])
+    for add in added_objects:
+        ax, ay, aw, ah = add["bbox"]
+        a_x0 = max(0, ax - REGION_PAD); a_y0 = max(0, ay - REGION_PAD)
+        a_x1 = min(aligned.shape[1], ax + aw + REGION_PAD)
+        a_y1 = min(aligned.shape[0], ay + ah + REGION_PAD)
+        add_patches.append(aligned[a_y0:a_y1, a_x0:a_x1])
+
+    rem_emb = dinov2_embed_batch(rem_patches) if rem_patches else None
+    add_emb = dinov2_embed_batch(add_patches) if add_patches else None
+
+    moved_objects = []
+    surviving_removed = []
+    remaining_added = list(range(len(added_objects)))
+
+    if rem_emb is not None and add_emb is not None and len(remaining_added) > 0:
+        sim_matrix = rem_emb @ add_emb.T  # (R, A), both rows L2-normalized
+    else:
+        sim_matrix = None
+
+    for ri, rem in enumerate(removed_objects):
+        rx, ry, rw, rh = rem["bbox"]
+        best_sim, best_j = -1.0, None
+        if sim_matrix is not None:
+            for j in remaining_added:
+                add = added_objects[j]
+                ax, ay, aw, ah = add["bbox"]
+                # Skip if bboxes overlap (different objects at same position)
+                if rx < ax + aw and rx + rw > ax and ry < ay + ah and ry + rh > ay:
+                    continue
+                sim = float(sim_matrix[ri, j])
+                if sim > best_sim:
+                    best_sim, best_j = sim, j
+
+        if best_j is not None and best_sim >= MOVED_SIM_THRESHOLD:
+            add = added_objects[best_j]
+            moved_objects.append({"from": rem, "to": add, "similarity": best_sim})
+            remaining_added.remove(best_j)
+            print(f"  MOVED: from {rem['centroid']} -> to {add['centroid']}  sim={best_sim:.3f}")
+        else:
+            surviving_removed.append(rem)
+            if best_j is not None:
+                print(f"  REMOVED (unmatched) @{rem['centroid']}: best sim={best_sim:.3f} < {MOVED_SIM_THRESHOLD}")
+            else:
+                print(f"  REMOVED (unmatched) @{rem['centroid']}: no non-overlapping ADDED found")
+
+    removed_objects = surviving_removed
+    added_objects = [added_objects[j] for j in remaining_added]
+
+    n_moved   = len(moved_objects)
+    n_added   = len(added_objects)
+    n_removed = len(removed_objects)
+    n_total   = n_added + n_removed + n_moved
+    severity  = "HIGH" if n_total > 0 else "NONE"
+
+    # 5) Render: dark-tint background + highlight changed objects
+    DARK_FACTOR = 0.3
+    result_img = cv2.multiply(aligned, np.array([DARK_FACTOR] * 3, dtype=np.float64))
+    result_img = np.clip(result_img, 0, 255).astype(np.uint8)
+
+    for a in added_objects:
+        _composite_object(result_img, aligned, a["mask"], a["bbox"], COLOR_ADDED)
+    for r in removed_objects:
+        _composite_object(result_img, r["source"], r["mask"], r["bbox"], COLOR_REMOVED)
+    for m in moved_objects:
+        _composite_object(result_img, m["from"]["source"], m["from"]["mask"], m["from"]["bbox"], COLOR_MOVED)
+        _composite_object(result_img, aligned,              m["to"]["mask"],   m["to"]["bbox"],   COLOR_MOVED)
+
+    # Draw severity label in top-left
+    label = f"{severity}  +{n_added}  -{n_removed}  ~{n_moved}"
+    cv2.putText(result_img, label, (20, 42),
+                cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 255, 255), 5)
+    cv2.putText(result_img, label, (20, 42),
+                cv2.FONT_HERSHEY_SIMPLEX, 1.0, (0, 0, 0), 2)
+
+    cv2.imwrite(output_path, result_img)
+    cv2.imwrite(heatmap_path, heatmap)
+
+    return {
+        "pipeline": "ai",
+        "added": n_added,
+        "removed": n_removed,
+        "moved": n_moved,
+        "object_changes": n_total,
+        "severity": severity,
+        "alignment_failed": alignment_failed,
+    }

requirements-render.txt

@@ -0,0 +1,9 @@
+flask
+flask-cors
+gunicorn
+bcrypt
+opencv-python-headless
+numpy<2
+scikit-image
+scipy
+python-dotenv

requirements.txt

@@ -0,0 +1,16 @@
+flask
+flask-cors
+gunicorn
+bcrypt
+opencv-python==4.9.0.80
+numpy<2
+scikit-image
+scipy
+torch
+torchvision
+lpips
+kornia
+ultralytics
+transformers
+python-dotenv
+gradio

sam2_t.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94375f988270836169320bd901960c67b5770e8bef3867d70102f01a8b5ca501
+size 78064050

test_describe.py

@@ -0,0 +1,56 @@
+import os
+import sys
+from dotenv import load_dotenv
+
+# Load variables from .env file
+load_dotenv()
+
+# Import the describe_change function
+from describe import describe_change
+
+def main():
+    # Allow specifying filenames as command line arguments or use defaults
+    img1 = sys.argv[1] if len(sys.argv) > 1 else "../uploads/img1.jpg"
+    img2 = sys.argv[2] if len(sys.argv) > 2 else "../uploads/img2.jpg"
+
+    print(f"Testing describe_change with:")
+    print(f"  Before image: {img1}")
+    print(f"  After image:  {img2}")
+
+    # Check if files exist
+    if not os.path.exists(img1):
+        print(f"Error: Before image '{img1}' does not exist.")
+        print("Please place test images or pass paths: python test_describe.py path/to/before.jpg path/to/after.jpg")
+        return
+    if not os.path.exists(img2):
+        print(f"Error: After image '{img2}' does not exist.")
+        print("Please place test images or pass paths: python test_describe.py path/to/before.jpg path/to/after.jpg")
+        return
+
+    # Check for API key or local setup
+    groq_api_key = os.environ.get("GROQ_API_KEY")
+    if groq_api_key:
+        print(f"Using Groq API for generation (Key starts with: {groq_api_key[:10]}...)")
+    else:
+        print("Using local Ollama fallback (GROQ_API_KEY not set).")
+
+    # Sample metrics detector hint
+    metrics = {
+        "added": 1,
+        "removed": 0
+    }
+    
+    print("\nGenerating description...")
+    try:
+        desc = describe_change(img1, img2, metrics)
+        print("\n--- Result ---")
+        if desc:
+            print(desc)
+        else:
+            print("Failed to generate description (returned None).")
+        print("--------------")
+    except Exception as e:
+        print(f"Error running description: {e}")
+
+if __name__ == "__main__":
+    main()

users.json

@@ -0,0 +1,5 @@
+{
+  "users": {
+    "username": "$2b$12$BFqw0a4oqR.Iky53Udd.TuA1TfBHWaIDNM7flIj8E5tJ6vnN4H5ny"
+  }
+}