app.py

"""
app.py
------
Flask backend for the Pixel Art Sprite Dataset Builder.

Serves:
  - the static HTML/CSS/JS frontend (static/index.html, app.js, style.css)
  - a JSON REST API under /api/* used by that frontend

Why Flask + custom HTML/CSS/JS instead of Gradio: the spec explicitly asks
for a custom interface unconstrained by Gradio's component model (manual
bounding-box editing, merge/split, a numbered sprite-sheet overlay, batch
review controls) which is much more natural to build directly.

This file intentionally contains NO business logic itself -- every route
is a thin wrapper that calls into analyzer / segmentation / scoring /
duplicate_detector / dataset_manager / database and serializes the result.
That keeps the (harder to unit-test, since it talks HTTP) web layer as
small as possible.
"""

from __future__ import annotations

import io
import json
import logging
import os

from flask import Flask, jsonify, request, send_file, send_from_directory
from PIL import Image

import database as db
from analyzer import process_sheet
from config import (
    BASE_DIR,
    MAX_FILE_SIZE_MB,
    MAX_IMAGE_DIMENSION,
    MAX_ZIP_SIZE_MB,
    QUALITY_AUTO_ACCEPT_THRESHOLD,
    SPLIT_BY_SHEET,
    SPLIT_TRAIN_RATIO,
    SPLIT_VALIDATION_RATIO,
    SUPPORTED_EXTENSIONS,
)
from dataset_manager import (
    accepted_path_for,
    assign_split,
    build_manifest,
    move_to_accepted,
    move_to_rejected,
    push_accepted_batch,
    rejected_path_for,
    staging_path_for,
)
from dataset_report import balance_report, quality_calibration
from image_utils import BBox, UnsupportedImageError, load_image_rgba, nearest_preview, to_numpy, to_pil
from zip_processor import ZipSecurityError, process_zip

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("sprite_dataset_builder")

STATIC_DIR = BASE_DIR / "static"

app = Flask(__name__, static_folder=None)
app.config["MAX_CONTENT_LENGTH"] = max(MAX_ZIP_SIZE_MB, MAX_FILE_SIZE_MB) * 1024 * 1024 + 5 * 1024 * 1024

db.init_db()


# ---------------------------------------------------------------------------
# Static frontend
# ---------------------------------------------------------------------------

@app.route("/")
def index():
    return send_from_directory(STATIC_DIR, "index.html")


@app.route("/static/<path:path>")
def static_files(path):
    return send_from_directory(STATIC_DIR, path)


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

def _row_get(row, key: str, default=None):
    """
    Read a column that may not exist yet on a very old DB row. `sqlite3.Row`
    raises IndexError for unknown keys rather than returning None, and the
    schema gains columns over time (see database._add_missing_columns).
    """
    try:
        value = row[key]
    except (IndexError, KeyError):
        return default
    return default if value is None else value


def _sprite_to_json(row) -> dict:
    quality = json.loads(row["quality_json"]) if row["quality_json"] else {}
    reasons = json.loads(row["confidence_reasons"]) if row["confidence_reasons"] else []
    tags = json.loads(row["tags"]) if row["tags"] else []
    return {
        "id": row["id"],
        "sheet_id": row["sheet_id"],
        "source_file": row["source_file"],
        "sprite_index": row["sprite_index"],
        "bbox": {"x": row["x"], "y": row["y"], "width": row["width"], "height": row["height"]},
        "sha256": row["sha256"],
        "phash": row["phash"],
        "quality": quality,
        "confidence": row["confidence"],
        "confidence_reasons": reasons,
        "component_count": row["component_count"],
        "category": row["category"],
        "style": row["style"],
        "tags": tags,
        "status": row["status"],
        "is_duplicate_of": row["is_duplicate_of"],
        "duplicate_similarity": row["duplicate_similarity"],
        "segmentation_source": _row_get(row, "segmentation_source", "gap"),
        "auto_tagged": bool(_row_get(row, "auto_tagged", 0)),
        "auto_tag_confidence": _row_get(row, "auto_tag_confidence", None),
        "split": _row_get(row, "split", None),
        "preview_url": f"/api/sprite/{row['id']}/image?variant=preview",
        "png_url": f"/api/sprite/{row['id']}/image?variant=kept",
        "png_transparent_url": f"/api/sprite/{row['id']}/image?variant=transparent",
    }


def _sheet_to_json(row) -> dict:
    return {
        "id": row["id"],
        "filename": row["filename"],
        "width": row["width"],
        "height": row["height"],
        "sprite_count": row["sprite_count"],
        "zip_batch_id": row["zip_batch_id"],
    }


def _error(message: str, status: int = 400):
    return jsonify({"error": message}), status


# ---------------------------------------------------------------------------
# Upload
# ---------------------------------------------------------------------------

@app.route("/api/upload/images", methods=["POST"])
def upload_images():
    """Accept one or more raw image files (sprite sheets or single sprites)."""
    files = request.files.getlist("files")
    if not files:
        return _error("No files uploaded (expected form field 'files').")

    padding = int(request.form.get("padding", 0))
    results = []

    for f in files:
        filename = f.filename or "upload.png"
        ext = os.path.splitext(filename)[1].lower()
        if ext not in SUPPORTED_EXTENSIONS:
            results.append({"filename": filename, "error": f"Unsupported extension {ext}"})
            continue

        data = f.read()
        if len(data) > MAX_FILE_SIZE_MB * 1024 * 1024:
            results.append({"filename": filename, "error": f"File exceeds MAX_FILE_SIZE_MB={MAX_FILE_SIZE_MB}"})
            continue

        # Quick dimension guard before full processing.
        try:
            probe = Image.open(io.BytesIO(data))
            probe.verify()
            probe = Image.open(io.BytesIO(data))  # re-open after verify()
            if max(probe.size) > MAX_IMAGE_DIMENSION:
                results.append(
                    {"filename": filename, "error": f"Image dimension exceeds MAX_IMAGE_DIMENSION={MAX_IMAGE_DIMENSION}"}
                )
                continue
        except Exception as exc:  # noqa: BLE001
            results.append({"filename": filename, "error": f"Could not read image: {exc}"})
            continue

        result = process_sheet(data, filename, padding=padding)
        results.append(
            {
                "filename": filename,
                "sheet_id": result.sheet_id,
                "sprite_count": result.sprite_count,
                "low_confidence_count": result.low_confidence_count,
                "background_color": result.background_color,
                "error": result.error,
            }
        )

    total_sprites = sum(r.get("sprite_count", 0) for r in results)
    return jsonify({"sheets": results, "total_sheets": len(results), "total_sprites": total_sprites})


@app.route("/api/upload/zip", methods=["POST"])
def upload_zip():
    """Accept a ZIP containing multiple sprite sheets."""
    if "file" not in request.files:
        return _error("No file uploaded (expected form field 'file').")

    padding = int(request.form.get("padding", 0))
    upload = request.files["file"]
    data = upload.read()

    try:
        report = process_zip(data, filename_hint=upload.filename or "upload.zip")
    except ZipSecurityError as exc:
        return _error(str(exc), status=400)

    results = []
    for entry in report.entries:
        if len(entry.data) > MAX_FILE_SIZE_MB * 1024 * 1024:
            results.append({"filename": entry.filename, "error": f"File exceeds MAX_FILE_SIZE_MB={MAX_FILE_SIZE_MB}"})
            continue
        try:
            probe = Image.open(io.BytesIO(entry.data))
            probe.verify()
            probe = Image.open(io.BytesIO(entry.data))
            if max(probe.size) > MAX_IMAGE_DIMENSION:
                results.append(
                    {"filename": entry.filename, "error": f"Image dimension exceeds MAX_IMAGE_DIMENSION={MAX_IMAGE_DIMENSION}"}
                )
                continue
        except Exception as exc:  # noqa: BLE001
            results.append({"filename": entry.filename, "error": f"Could not read image: {exc}"})
            continue

        result = process_sheet(entry.data, entry.filename, padding=padding)
        results.append(
            {
                "filename": entry.filename,
                "sheet_id": result.sheet_id,
                "sprite_count": result.sprite_count,
                "low_confidence_count": result.low_confidence_count,
                "background_color": result.background_color,
                "error": result.error,
            }
        )

    total_sprites = sum(r.get("sprite_count", 0) for r in results)
    return jsonify(
        {
            "sheets": results,
            "total_sheets": len(results),
            "total_sprites": total_sprites,
            "skipped": report.skipped,
            "rejected_nested_zips": report.rejected_nested_zips,
            "total_files_in_archive": report.total_files_in_archive,
        }
    )


# ---------------------------------------------------------------------------
# Sheets & sprites listing
# ---------------------------------------------------------------------------

@app.route("/api/sheets", methods=["GET"])
def list_sheets():
    rows = db.list_sheets()
    return jsonify({"sheets": [_sheet_to_json(r) for r in rows]})


@app.route("/api/sheet/<sheet_id>/image", methods=["GET"])
def get_sheet_image(sheet_id: str):
    """
    Serve the ORIGINAL uploaded sheet image, so the frontend can draw the
    numbered bounding-box overlay directly on top of the real sheet (per
    spec: "orijinal sheet'in üzerinde bounding box'ları göster") rather
    than reconstructing an approximation from individual sprite crops.
    Uses the same source cache that manual edit/merge/split re-crop from.
    """
    from config import STAGING_DIR

    path = STAGING_DIR / f"{sheet_id}_source.png"
    if not path.exists():
        return _error("Original sheet image is no longer cached", 404)
    return send_file(path, mimetype="image/png")


@app.route("/api/sprites", methods=["GET"])
def list_sprites():
    status = request.args.get("status")
    sheet_id = request.args.get("sheet_id")
    rows = db.list_sprites(status=status, sheet_id=sheet_id)
    return jsonify({"sprites": [_sprite_to_json(r) for r in rows]})


@app.route("/api/sprite/<sprite_id>", methods=["GET"])
def get_sprite(sprite_id: str):
    row = db.get_sprite(sprite_id)
    if row is None:
        return _error("Sprite not found", 404)
    return jsonify(_sprite_to_json(row))


@app.route("/api/sprite/<sprite_id>/image", methods=["GET"])
def get_sprite_image(sprite_id: str):
    row = db.get_sprite(sprite_id)
    if row is None:
        return _error("Sprite not found", 404)

    variant = request.args.get("variant", "kept")
    status = row["status"]

    if status == "pending":
        base = staging_path_for(sprite_id, transparent=(variant == "transparent"))
    elif status in ("accepted", "uploaded"):
        base = accepted_path_for(sprite_id, transparent=(variant == "transparent"))
    else:
        base = rejected_path_for(sprite_id, transparent=(variant == "transparent"))

    if not base.exists():
        return _error("Sprite image file not found on disk", 404)

    if variant == "preview":
        img = Image.open(base).convert("RGBA")
        preview = nearest_preview(img, target_long_side=512)
        buf = io.BytesIO()
        preview.save(buf, format="PNG")
        buf.seek(0)
        return send_file(buf, mimetype="image/png")

    return send_file(base, mimetype="image/png")


# ---------------------------------------------------------------------------
# Review actions: accept / reject / edit
# ---------------------------------------------------------------------------

@app.route("/api/sprite/<sprite_id>/accept", methods=["POST"])
def accept_sprite(sprite_id: str):
    row = db.get_sprite(sprite_id)
    if row is None:
        return _error("Sprite not found", 404)
    move_to_accepted(sprite_id)
    return jsonify({"id": sprite_id, "status": "accepted"})


@app.route("/api/sprite/<sprite_id>/reject", methods=["POST"])
def reject_sprite(sprite_id: str):
    row = db.get_sprite(sprite_id)
    if row is None:
        return _error("Sprite not found", 404)
    move_to_rejected(sprite_id)
    return jsonify({"id": sprite_id, "status": "rejected"})


@app.route("/api/sprite/<sprite_id>/metadata", methods=["PATCH"])
def update_sprite_metadata(sprite_id: str):
    row = db.get_sprite(sprite_id)
    if row is None:
        return _error("Sprite not found", 404)
    payload = request.get_json(force=True, silent=True) or {}
    fields = {}
    if "category" in payload:
        fields["category"] = payload["category"]
        db.add_custom_category(payload["category"], kind="category")
    if "style" in payload:
        fields["style"] = payload["style"]
        db.add_custom_category(payload["style"], kind="style")
    if "tags" in payload and isinstance(payload["tags"], list):
        fields["tags"] = payload["tags"]
    if fields:
        # A human just made a decision here, so the row stops counting as
        # an unreviewed machine suggestion.
        fields["auto_tagged"] = 0
    db.update_sprite_fields(sprite_id, **fields)
    return jsonify(_sprite_to_json(db.get_sprite(sprite_id)))


@app.route("/api/sprite/<sprite_id>/edit-bbox", methods=["POST"])
def edit_bbox(sprite_id: str):
    """
    Manual bounding-box move/resize. Re-crops the sprite pixels from the
    ORIGINAL sheet using the new bbox (never resamples -- this is a new
    exact crop, exactly like the automatic extraction path), then re-runs
    quality scoring + hashing on the new crop so metadata stays accurate.
    """
    row = db.get_sprite(sprite_id)
    if row is None:
        return _error("Sprite not found", 404)
    if row["status"] != "pending":
        return _error("Only pending sprites can be edited. Reject/accept state must be undone first.", 400)

    payload = request.get_json(force=True, silent=True) or {}
    try:
        new_bbox = BBox(x=int(payload["x"]), y=int(payload["y"]), width=int(payload["width"]), height=int(payload["height"]))
    except (KeyError, ValueError, TypeError):
        return _error("Expected integer x, y, width, height in request body")
    if new_bbox.width <= 0 or new_bbox.height <= 0:
        return _error("width and height must be positive")

    _recrop_sprite_from_original(sprite_id, row, new_bbox)
    return jsonify(_sprite_to_json(db.get_sprite(sprite_id)))


def _find_original_sheet_bytes(sheet_id: str) -> bytes | None:
    """
    We don't keep a separate copy of every original uploaded sheet on disk
    long-term (memory/disk footprint) -- instead, edits re-derive the exact
    crop from any ALREADY-EXTRACTED sibling sprite's neighborhood is not
    reliable, so for edit-bbox support we reconstruct from the widest
    currently-known crop of that sheet. To guarantee pixel-perfect manual
    edits, the sheet's original bytes are cached under data/staging as
    `<sheet_id>_source.png` at upload time (see process_sheet caller
    below). If missing (e.g. very old sheet), edit-bbox degrades to
    resizing the existing crop's own canvas without new source pixels.
    """
    from config import STAGING_DIR

    candidate = STAGING_DIR / f"{sheet_id}_source.png"
    if candidate.exists():
        return candidate.read_bytes()
    return None


def _recrop_sprite_from_original(sprite_id: str, row, new_bbox: BBox) -> None:
    from background import detect_background
    from duplicate_detector import compute_dhash, compute_sha256
    from image_utils import crop_exact, image_to_png_bytes
    from scoring import analyze_sprite
    from sprite_extractor import extract_sprite
    from segmentation import DetectedSprite

    source_bytes = _find_original_sheet_bytes(row["sheet_id"])
    if source_bytes is None:
        logger.warning("No cached original sheet for %s; edit-bbox cannot re-crop pixels precisely.", row["sheet_id"])
        return

    img = load_image_rgba(source_bytes)
    sheet_rgba = to_numpy(img)
    bg_info = detect_background(sheet_rgba)

    fake_detection = DetectedSprite(
        sprite_index=row["sprite_index"],
        bbox=new_bbox,
        confidence=100.0,
        confidence_reasons=["Manually edited bounding box"],
        component_count=1,
        pixel_count=new_bbox.area,
    )
    extracted = extract_sprite(sheet_rgba, bg_info, fake_detection, source_file=row["source_file"])

    kept_path = staging_path_for(sprite_id, transparent=False)
    transparent_path = staging_path_for(sprite_id, transparent=True)
    to_pil(extracted.rgba).save(kept_path, format="PNG")
    to_pil(extracted.rgba_transparent_bg).save(transparent_path, format="PNG")

    quality = analyze_sprite(extracted.rgba)
    sha256 = compute_sha256(image_to_png_bytes(to_pil(extracted.rgba)))
    phash = compute_dhash(extracted.rgba)

    db.update_sprite_fields(
        sprite_id,
        x=new_bbox.x, y=new_bbox.y, width=new_bbox.width, height=new_bbox.height,
        png_path=str(kept_path), png_path_transparent=str(transparent_path),
    )
    with db.get_conn() as conn:
        conn.execute(
            "UPDATE sprites SET quality_json=?, sha256=?, phash=?, confidence=?, confidence_reasons=? WHERE id=?",
            (json.dumps(quality.to_dict()), sha256, phash, 100.0, json.dumps(["Manually edited bounding box"]), sprite_id),
        )


@app.route("/api/sprites/merge", methods=["POST"])
def merge_sprites():
    """
    Merge two currently-pending sprites (from the SAME sheet) into one.
    The merged sprite's bbox is the union of both; pixels are re-cropped
    from the original sheet (never resampled). One of the two sprite rows
    becomes the merged result; the other is deleted.
    """
    payload = request.get_json(force=True, silent=True) or {}
    ids = payload.get("sprite_ids", [])
    if not isinstance(ids, list) or len(ids) != 2:
        return _error("Expected {'sprite_ids': [id1, id2]}")

    rows = [db.get_sprite(i) for i in ids]
    if any(r is None for r in rows):
        return _error("One or both sprites not found", 404)
    if rows[0]["sheet_id"] != rows[1]["sheet_id"]:
        return _error("Can only merge sprites from the same sheet")
    if rows[0]["status"] != "pending" or rows[1]["status"] != "pending":
        return _error("Both sprites must be in pending status to merge")

    box_a = BBox(rows[0]["x"], rows[0]["y"], rows[0]["width"], rows[0]["height"])
    box_b = BBox(rows[1]["x"], rows[1]["y"], rows[1]["width"], rows[1]["height"])
    union_box = box_a.union(box_b)

    keep_id, drop_id = ids[0], ids[1]
    _recrop_sprite_from_original(keep_id, rows[0], union_box)

    with db.get_conn() as conn:
        conn.execute("DELETE FROM sprites WHERE id = ?", (drop_id,))

    return jsonify(_sprite_to_json(db.get_sprite(keep_id)))


@app.route("/api/sprites/split", methods=["POST"])
def split_sprite():
    """
    Split one pending sprite into two, given two new bounding boxes (both
    must be within/near the original sprite's region on the sheet). The
    original sprite row is repurposed as the first new sprite; a new row
    is created for the second.
    """
    payload = request.get_json(force=True, silent=True) or {}
    sprite_id = payload.get("sprite_id")
    boxes = payload.get("boxes", [])
    if not sprite_id or not isinstance(boxes, list) or len(boxes) != 2:
        return _error("Expected {'sprite_id': id, 'boxes': [{x,y,width,height}, {...}]}")

    row = db.get_sprite(sprite_id)
    if row is None:
        return _error("Sprite not found", 404)
    if row["status"] != "pending":
        return _error("Only pending sprites can be split")

    try:
        box_objs = [BBox(int(b["x"]), int(b["y"]), int(b["width"]), int(b["height"])) for b in boxes]
    except (KeyError, ValueError, TypeError):
        return _error("Each box needs integer x, y, width, height")

    _recrop_sprite_from_original(sprite_id, row, box_objs[0])

    new_id = db.new_id("sprite")
    new_row_source = db.get_sprite(sprite_id)  # refreshed row after recrop
    rec = db.SpriteRecord(
        id=new_id, sheet_id=row["sheet_id"], source_file=row["source_file"],
        sprite_index=row["sprite_index"] + 1000,  # keep ordering distinct; cosmetic only
        x=box_objs[1].x, y=box_objs[1].y, width=box_objs[1].width, height=box_objs[1].height,
        png_path=str(staging_path_for(new_id)), png_path_transparent=str(staging_path_for(new_id, transparent=True)),
        sha256="", phash="", quality={}, confidence=100.0,
        confidence_reasons=["Manually split from another sprite"], component_count=1,
        category=row["category"], style=row["style"], tags=json.loads(row["tags"]) if row["tags"] else [],
        status="pending",
    )
    db.insert_sprite(rec)
    _recrop_sprite_from_original(new_id, dict(row) | {"sprite_index": rec.sprite_index}, box_objs[1])

    return jsonify(
        {
            "sprite_a": _sprite_to_json(db.get_sprite(sprite_id)),
            "sprite_b": _sprite_to_json(db.get_sprite(new_id)),
        }
    )


# ---------------------------------------------------------------------------
# Batch controls
# ---------------------------------------------------------------------------

@app.route("/api/batch/accept-above", methods=["POST"])
def batch_accept_above():
    payload = request.get_json(force=True, silent=True) or {}
    threshold = float(payload.get("threshold", QUALITY_AUTO_ACCEPT_THRESHOLD))
    rows = db.list_sprites(status="pending")
    accepted_ids = []
    for row in rows:
        quality = json.loads(row["quality_json"]) if row["quality_json"] else {}
        score = quality.get("overall_quality_score", 0)
        is_low_conf = row["confidence"] is not None and row["confidence"] < 70
        is_dup = row["is_duplicate_of"] is not None
        if score >= threshold and not is_low_conf and not is_dup:
            move_to_accepted(row["id"])
            accepted_ids.append(row["id"])
    return jsonify({"accepted_count": len(accepted_ids), "accepted_ids": accepted_ids})


@app.route("/api/batch/reject-duplicates", methods=["POST"])
def batch_reject_duplicates():
    rows = db.list_sprites(status="pending")
    rejected_ids = []
    for row in rows:
        if row["is_duplicate_of"] is not None:
            move_to_rejected(row["id"])
            rejected_ids.append(row["id"])
    return jsonify({"rejected_count": len(rejected_ids), "rejected_ids": rejected_ids})


@app.route("/api/batch/assign", methods=["POST"])
def batch_assign_metadata():
    """
    Apply one category / style / tag change across a whole scope at once.

    This is the difference between labelling a sheet of 300 tiles and not
    labelling it at all. The common case is a sheet that is entirely one
    kind of thing -- a tree sheet, a UI icon sheet -- where the correct
    label is known before the sheet is even uploaded.

    Body: {
      "scope": "sheet" | "status" | "all" | "ids",
      "sheet_id": ...,          # for scope=sheet
      "status": "pending",      # optional extra filter
      "sprite_ids": [...],      # for scope=ids
      "category": "tree",
      "style": "pixel_art",
      "add_tags": ["forest"],   # additive by default
      "remove_tags": [...],
      "replace_tags": [...]     # only when the existing tags are wrong
    }
    """
    payload = request.get_json(force=True, silent=True) or {}
    scope = payload.get("scope", "sheet")
    if scope not in ("sheet", "status", "all", "ids"):
        return _error("scope must be one of: sheet, status, all, ids")

    sprite_ids = db.select_sprite_ids(
        scope=scope,
        sheet_id=payload.get("sheet_id"),
        status=payload.get("status"),
        sprite_ids=payload.get("sprite_ids"),
    )
    if not sprite_ids:
        return jsonify({"updated_count": 0, "sprite_ids": []})

    category = (payload.get("category") or "").strip() or None
    style = (payload.get("style") or "").strip() or None
    if category:
        db.add_custom_category(category, kind="category")
        category = category.strip().lower().replace(" ", "_")
    if style:
        db.add_custom_category(style, kind="style")
        style = style.strip().lower().replace(" ", "_")

    replace_tags = payload.get("replace_tags")
    updated = db.bulk_update_metadata(
        sprite_ids,
        category=category,
        style=style,
        add_tags=payload.get("add_tags") or [],
        remove_tags=payload.get("remove_tags") or [],
        replace_tags=replace_tags if isinstance(replace_tags, list) else None,
    )
    return jsonify({"updated_count": updated, "sprite_ids": sprite_ids})


def _sprite_rank(row) -> tuple:
    """
    How good a sprite is as the representative of its duplicate cluster:
    best quality first, then best segmentation confidence, then largest.
    """
    quality = json.loads(row["quality_json"]) if row["quality_json"] else {}
    return (
        quality.get("overall_quality_score") or 0,
        row["confidence"] or 0,
        (row["width"] or 0) * (row["height"] or 0),
    )


@app.route("/api/batch/dedupe-keep-one", methods=["POST"])
def batch_dedupe_keep_one():
    """
    Collapse every cluster of duplicates down to ONE representative.

    This is the difference between "Reject Duplicates" and dataset hygiene:
    rejecting every flagged sprite is fine when duplicates are accidental,
    but a tileset legitimately repeats the same floor tile dozens of times
    and each copy is flagged against a different earlier one. Training on
    forty identical tiles just teaches the model that one texture matters
    forty times as much as anything else, so the redundant copies go and
    the best-scoring member of each cluster stays.
    """
    clusters = db.duplicate_clusters(statuses=("pending",))
    rejected_ids: list[str] = []
    kept_ids: list[str] = []
    for members in clusters:
        ordered = sorted(members, key=_sprite_rank, reverse=True)
        kept_ids.append(ordered[0]["id"])
        # The survivor is no longer "a duplicate of" anything -- it is the
        # single copy now, and leaving the flag on would keep it out of
        # batch auto-accept for no reason.
        db.clear_duplicate_flag(ordered[0]["id"])
        for row in ordered[1:]:
            move_to_rejected(row["id"])
            rejected_ids.append(row["id"])
    return jsonify(
        {
            "cluster_count": len(clusters),
            "kept_count": len(kept_ids),
            "rejected_count": len(rejected_ids),
            "kept_ids": kept_ids,
            "rejected_ids": rejected_ids,
        }
    )


@app.route("/api/batch/low-confidence", methods=["GET"])
def batch_low_confidence():
    from config import LOW_CONFIDENCE_THRESHOLD

    rows = db.list_sprites(status="pending")
    low = [r for r in rows if r["confidence"] is not None and r["confidence"] < LOW_CONFIDENCE_THRESHOLD]
    return jsonify({"sprites": [_sprite_to_json(r) for r in low]})


# ---------------------------------------------------------------------------
# Dataset summary + HF upload
# ---------------------------------------------------------------------------

@app.route("/api/dataset/summary", methods=["GET"])
def dataset_summary():
    counts = db.counts_by_status()
    dup_count = db.count_possible_duplicates()
    return jsonify({**counts, "duplicates": dup_count})


@app.route("/api/animation/frames", methods=["GET"])
def animation_frames():
    """
    Sprites of one sheet, ordered and laid out for playback.

    The tricky part is alignment. Every sprite is cropped tight to its own
    content, so playing the crops back-to-back makes the animation jump
    around: a frame whose flames happen to reach one pixel further left
    starts one pixel further left. The frames are therefore returned with
    their ORIGINAL sheet coordinates plus a common canvas covering all of
    them, so the player can place each frame where it actually sat on the
    sheet. That is the difference between an animation preview and a
    slideshow of wobbling crops.
    """
    sheet_id = request.args.get("sheet_id")
    status = request.args.get("status")  # optional: pending / accepted / ...
    if not sheet_id:
        return _error("sheet_id is required")

    rows = db.list_sprites(sheet_id=sheet_id, status=status)
    rows = [r for r in rows if r["status"] != "rejected"]
    if not rows:
        return jsonify({"frames": [], "canvas": {"width": 0, "height": 0}})

    min_x = min(r["x"] for r in rows)
    min_y = min(r["y"] for r in rows)
    max_x = max(r["x"] + r["width"] for r in rows)
    max_y = max(r["y"] + r["height"] for r in rows)

    frames = [
        {
            "id": r["id"],
            "sprite_index": r["sprite_index"],
            "url": f"/api/sprite/{r['id']}/image?variant=transparent",
            "width": r["width"],
            "height": r["height"],
            # Position within the shared canvas, so frames line up exactly
            # as they did on the sheet.
            "offset_x": r["x"] - min_x,
            "offset_y": r["y"] - min_y,
            "status": r["status"],
        }
        for r in sorted(rows, key=lambda r: r["sprite_index"])
    ]
    return jsonify(
        {
            "frames": frames,
            "canvas": {"width": max_x - min_x, "height": max_y - min_y},
            "sheet_id": sheet_id,
        }
    )


@app.route("/api/dataset/balance", methods=["GET"])
def dataset_balance():
    """Composition of the sprites that will actually be trained on."""
    return jsonify(balance_report())


@app.route("/api/dataset/calibration", methods=["GET"])
def dataset_calibration():
    """Does the quality score agree with the operator's own decisions?"""
    return jsonify(quality_calibration())


@app.route("/api/dataset/split-preview", methods=["GET"])
def dataset_split_preview():
    """
    What the train/validation/test split would look like for everything
    currently accepted -- shown before uploading, since the assignment is
    deterministic and is what a later evaluation depends on.
    """
    rows = [r for status in ("accepted", "uploaded") for r in db.list_sprites(status=status)]
    counts: dict[str, int] = {}
    for row in rows:
        split = _row_get(row, "split", None) or assign_split(row["id"], row["sheet_id"])
        counts[split] = counts.get(split, 0) + 1
    warnings = []
    missing = [name for name in ("train", "validation", "test") if not counts.get(name)]
    if rows and missing and SPLIT_BY_SHEET:
        warnings.append(
            f"No sprites landed in: {', '.join(missing)}. Splitting is grouped by sheet to keep "
            f"near-identical frames out of both train and test, so with few source sheets a whole "
            f"split can come out empty. Upload material from more sheets."
        )

    return jsonify(
        {
            "counts": counts,
            "total": len(rows),
            "grouped_by_sheet": SPLIT_BY_SHEET,
            "train_ratio": SPLIT_TRAIN_RATIO,
            "validation_ratio": SPLIT_VALIDATION_RATIO,
            "warnings": warnings,
        }
    )


@app.route("/api/dataset/upload", methods=["POST"])
def dataset_upload():
    result = push_accepted_batch()
    return jsonify(
        {
            "success": result.success,
            "uploaded_count": result.uploaded_count,
            "commit_url": result.commit_url,
            "error": result.error,
            "split_counts": result.split_counts,
        }
    )


@app.route("/api/categories", methods=["GET"])
def get_categories():
    return jsonify(
        {
            "categories": db.list_categories("category"),
            "styles": db.list_categories("style"),
        }
    )


@app.route("/api/categories", methods=["POST"])
def add_category():
    payload = request.get_json(force=True, silent=True) or {}
    name = payload.get("name", "")
    kind = payload.get("kind", "category")
    if not name:
        return _error("name is required")
    db.add_custom_category(name, kind=kind)
    return jsonify({"categories": db.list_categories("category"), "styles": db.list_categories("style")})


@app.errorhandler(413)
def too_large(_e):
    return _error(f"Upload too large (limit ~{MAX_ZIP_SIZE_MB}MB for zip / {MAX_FILE_SIZE_MB}MB per file)", 413)


if __name__ == "__main__":
    port = int(os.environ.get("PORT", 7860))
    app.run(host="0.0.0.0", port=port, debug=False)