scripts/evaluate_finetuned_vs_baseline.py

"""
Day 7 (Uday): compare fine-tuned AdaptFormer vs pretrained hub weights on
held-out Delhi pairs (docs/delhi_eval/test_split.json) and data/delhi_cd/test.

Usage:
    python scripts/evaluate_finetuned_vs_baseline.py
    python scripts/evaluate_finetuned_vs_baseline.py --methods AI
"""
from __future__ import annotations

import argparse
import json
import os
import sys
import time
from pathlib import Path

import numpy as np
from PIL import Image

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))

from app.detection_engine import run_detection  # noqa: E402
from app.evaluation.delhi_eval import iter_delhi_pairs  # noqa: E402
from app.evaluation.metrics import binary_metrics  # noqa: E402


def _load_held_out_ids() -> list[str]:
    path = ROOT / "docs" / "delhi_eval" / "test_split.json"
    if path.is_file():
        data = json.loads(path.read_text(encoding="utf-8"))
        return list(data.get("held_out_test_set", {}).get("pair_ids", []))
    # fallback: data/delhi_cd/test
    man = ROOT / "data" / "delhi_cd" / "test" / "manifest.json"
    if man.is_file():
        return [p["pair_id"] for p in json.loads(man.read_text())["pairs"]]
    return []


def _eval_weights(label: str, weights: str | None, pairs: list, method: str) -> dict:
    # Clear cached model so weights switch takes effect
    import app.model_inference as mi
    mi._MODEL = None
    mi._PROCESSOR = None
    mi._AVAILABLE = None
    mi._LOAD_FAILED = False
    mi._LOAD_ERROR = None
    mi._LOADED_FROM = None

    if weights:
        os.environ["ADAPTFORMER_WEIGHTS"] = weights
    else:
        os.environ.pop("ADAPTFORMER_WEIGHTS", None)
        # Force hub by pointing at a non-existent local so auto-dir is skipped
        # unless user has models/adaptformer_delhi — temporarily rename env.
        os.environ["ADAPTFORMER_WEIGHTS"] = "deepang/adaptformer-LEVIR-CD"

    f1s, ious = [], []
    per = {}
    t0 = time.time()
    for before, after, gt, pair_id, bp, ap in pairs:
        mask, _img, stats, _regions = run_detection(
            Image.fromarray(before), Image.fromarray(after),
            method=method,
            enable_registration=True, enable_normalization=True,
            detection_sensitivity=0.5,
            before_path=bp, after_path=ap,
        )
        if gt is None:
            # empty-GT FP test: IoU=1 if pred empty else 0
            pred_pos = int(np.sum(mask > 127))
            m = {
                "f1": 1.0 if pred_pos == 0 else 0.0,
                "iou": 1.0 if pred_pos == 0 else 0.0,
                "precision": 1.0 if pred_pos == 0 else 0.0,
                "recall": 1.0 if pred_pos == 0 else 0.0,
            }
        else:
            if mask.shape != gt.shape:
                import cv2
                mask = cv2.resize(mask, (gt.shape[1], gt.shape[0]),
                                  interpolation=cv2.INTER_NEAREST)
            m = binary_metrics(mask, gt)
        f1s.append(m["f1"])
        ious.append(m["iou"])
        per[pair_id] = {"f1": m["f1"], "iou": m["iou"],
                        "changePct": round(stats.get("change_percentage", 0), 3)}
        print(f"  [{label}] {pair_id}: F1={m['f1']:.3f} IoU={m['iou']:.3f}")

    return {
        "label": label,
        "weights": weights or "hub:deepang/adaptformer-LEVIR-CD",
        "n": len(pairs),
        "mean_f1": round(float(np.mean(f1s)), 4) if f1s else 0.0,
        "mean_iou": round(float(np.mean(ious)), 4) if ious else 0.0,
        "seconds": round(time.time() - t0, 1),
        "pairs": per,
    }


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--manifest", default="docs/delhi_eval/manifest.json")
    parser.add_argument("--finetuned", default="models/adaptformer_delhi/best")
    parser.add_argument("--method", default="AI-Based Deep Learning")
    parser.add_argument("--out", default="runs/day7_heldout_comparison.json")
    parser.add_argument("--also-feature", action="store_true",
                        help="also report Feature-Based on the same held-out set")
    args = parser.parse_args()

    held_ids = set(_load_held_out_ids())
    if not held_ids:
        raise SystemExit("No held-out pair IDs found in test_split.json")

    all_pairs = list(iter_delhi_pairs(args.manifest, require_gt=False))
    pairs = [p for p in all_pairs if p[3] in held_ids]
    # Prefer labeled if available; empty GT still included for FP test
    if not pairs:
        raise SystemExit(f"No held-out pairs loaded from {args.manifest}")

    print(f"Held-out pairs: {len(pairs)} -> {[p[3] for p in pairs]}")

    ft_path = ROOT / args.finetuned
    rows = []
    rows.append(_eval_weights("pretrained_hub", None, pairs, args.method))
    if ft_path.is_dir():
        rows.append(_eval_weights("finetuned_delhi", str(ft_path), pairs, args.method))
    else:
        print(f"WARNING: {ft_path} missing — run scripts/export_adaptformer_delhi.py first")

    if args.also_feature:
        # Feature-Based doesn't use AdaptFormer weights
        import app.model_inference as mi
        mi._MODEL = None
        f1s, ious, per = [], [], {}
        for before, after, gt, pair_id, bp, ap in pairs:
            mask, _img, stats, _ = run_detection(
                Image.fromarray(before), Image.fromarray(after),
                method="Feature-Based",
                enable_registration=True, enable_normalization=True,
                detection_sensitivity=0.5, before_path=bp, after_path=ap,
            )
            if gt is None:
                pred_pos = int(np.sum(mask > 127))
                m = {"f1": 1.0 if pred_pos == 0 else 0.0,
                     "iou": 1.0 if pred_pos == 0 else 0.0}
            else:
                if mask.shape != gt.shape:
                    import cv2
                    mask = cv2.resize(mask, (gt.shape[1], gt.shape[0]),
                                      interpolation=cv2.INTER_NEAREST)
                m = binary_metrics(mask, gt)
            f1s.append(m["f1"]); ious.append(m["iou"])
            per[pair_id] = m
        rows.append({
            "label": "feature_based",
            "weights": "n/a",
            "n": len(pairs),
            "mean_f1": round(float(np.mean(f1s)), 4),
            "mean_iou": round(float(np.mean(ious)), 4),
            "pairs": per,
        })

    report = {
        "held_out_ids": sorted(held_ids),
        "method": args.method,
        "comparison": rows,
        "winner": max(rows, key=lambda r: r["mean_f1"])["label"] if rows else None,
        "rca_notes": [
            "Domain mismatch (LEVIR→Delhi) is Critical #2 — fine-tune may still underperform "
            "on empty-GT FP pairs; Feature-Based remains strong for precision gates.",
            "GeoTIFF fullres_tiled auto + veg/registration fixes applied from RCA PDF.",
        ],
    }
    out = ROOT / args.out
    out.parent.mkdir(parents=True, exist_ok=True)
    out.write_text(json.dumps(report, indent=2), encoding="utf-8")
    print("\n=== Day 7 comparison ===")
    for r in rows:
        print(f"  {r['label']:20s} mean_F1={r['mean_f1']:.4f} mean_IoU={r['mean_iou']:.4f} n={r['n']}")
    print(f"Wrote {out}")


if __name__ == "__main__":
    main()