File size: 9,713 Bytes
ca179f0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 | """Run resumable paired/unpaired and ablation evaluation on a local dataset."""
from __future__ import annotations
import argparse
import json
import os
import re
import sys
import time
from pathlib import Path
from typing import Any
import numpy as np
from PIL import Image
REPO = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(REPO / "core"))
sys.path.insert(0, str(REPO))
from lumenx_core import create_engine # noqa: E402
from lumenx_core.contracts import EnhanceRequest, EnhanceResult, StageEvent # noqa: E402
from lumenx_core.contract_check import validate_run # noqa: E402
from common import align_reference, as_rgb_u8, decode_path, discover_records # noqa: E402
def _read_rows(path: Path) -> dict[str, dict[str, Any]]:
rows: dict[str, dict[str, Any]] = {}
if not path.is_file():
return rows
with path.open("r", encoding="utf-8") as stream:
for line in stream:
try:
row = json.loads(line)
except json.JSONDecodeError:
continue
if isinstance(row, dict) and row.get("key"):
rows[str(row["key"])] = row
return rows
def _atomic_json(path: Path, value: dict[str, Any]) -> None:
temporary = path.with_suffix(path.suffix + ".tmp")
temporary.write_text(json.dumps(value, indent=2, allow_nan=False), encoding="utf-8")
os.replace(temporary, path)
def _variants(modes: list[str], ablations: list[str]) -> list[tuple[str, frozenset[str]]]:
variants = [(mode, frozenset()) for mode in modes]
variants.extend(("hybrid", frozenset({flag})) for flag in ablations)
return variants
def _run_one(
engine: Any,
record: Any,
mode: str,
ablate: frozenset[str],
preview_path: Path | None = None,
) -> dict[str, Any]:
decoded = decode_path(record.input_path)
image = decoded.image
reference = None
if record.reference_path:
reference = as_rgb_u8(decode_path(record.reference_path))
reference = align_reference(reference, image.shape[:2])
request = EnhanceRequest(
image=image,
original_size=decoded.original_size,
alpha=decoded.alpha,
mode=mode,
ablate=ablate,
seed=0,
)
started = time.perf_counter()
items = list(engine.run(request))
elapsed_ms = (time.perf_counter() - started) * 1000.0
violations = validate_run(items, request)
if violations:
raise RuntimeError("Core stream contract violations: " + "; ".join(violations))
result = items[-1]
if not isinstance(result, EnhanceResult):
raise RuntimeError("Engine stream ended without an EnhanceResult")
final_visual = next((visual for visual in result.visuals if visual.key == "final"), None)
if final_visual is None:
raise RuntimeError("Engine did not return a final image")
output = final_visual.image[..., :3]
if preview_path is not None:
preview_path.parent.mkdir(parents=True, exist_ok=True)
Image.fromarray(output).save(preview_path, format="PNG", optimize=False)
suite = engine.metric_suite()
input_u8 = as_rgb_u8(decoded)
custom = suite.custom(input_u8, output)
unpaired = suite.unpaired(output)
paired = suite.paired(output, reference) if reference is not None else {}
confidence = suite.confidence_errors(input_u8, output, reference) if reference is not None else {}
return {
"status": "complete",
"relative_path": record.relative_path,
"mode": mode,
"ablate": sorted(ablate),
"actual_method": result.provenance.get("actual_method"),
"fallback_chain": result.provenance.get("fallback_chain", []),
"paired": reference is not None,
"metrics": {**paired, **unpaired, **custom, **confidence},
"stage_timings_ms": result.timings_ms,
"latency_ms": elapsed_ms,
"guardrail_failures": [check.check for check in result.guardrails if check.passed is False],
"warnings": [warning.code for warning in result.warnings],
"metric_unavailable": suite.unavailable(),
"provenance": result.provenance,
}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--dataset", required=True, help="Dataset label used in output rows")
parser.add_argument("--root", required=True, type=Path, help="Dataset root")
parser.add_argument("--input-dir", type=Path, help="Optional low-light image directory")
parser.add_argument("--reference-dir", type=Path, help="Optional ground-truth directory")
parser.add_argument("--paired", action="store_true", help="Require matched reference images")
parser.add_argument("--modes", default="classical,dl,hybrid", help="Comma-separated processing modes")
parser.add_argument("--ablations", default="", help="Comma-separated hybrid ablation flags")
parser.add_argument("--device", default="auto")
parser.add_argument("--config", type=Path)
parser.add_argument("--output", type=Path, default=REPO / "server" / "results")
parser.add_argument("--limit", type=int, default=0, help="Optional maximum number of source images")
parser.add_argument("--save-images", action="store_true", help="Save final images under results/previews for manual failure review")
args = parser.parse_args()
modes = [mode.strip() for mode in args.modes.split(",") if mode.strip()]
if not modes or any(mode not in {"classical", "dl", "hybrid", "zerodce"} for mode in modes):
parser.error("--modes must contain classical, dl, hybrid and/or zerodce")
allowed_flags = {"no_tuner", "no_fusion", "no_classical", "no_dl", "no_gain_denoise", "no_recoverability", "no_guardrails", "no_color_correction"}
ablations = [flag.strip() for flag in args.ablations.split(",") if flag.strip()]
invalid = set(ablations) - allowed_flags
if invalid:
parser.error("unknown ablation flag(s): " + ", ".join(sorted(invalid)))
records = discover_records(
args.root,
input_dir=args.input_dir,
reference_dir=args.reference_dir,
paired=args.paired,
)
if args.limit > 0:
records = records[: args.limit]
variants = _variants(modes, ablations)
args.output.mkdir(parents=True, exist_ok=True)
safe_dataset = re.sub(r"[^A-Za-z0-9_-]+", "_", args.dataset).strip("_") or "dataset"
output_file = args.output / f"{safe_dataset}_evaluation.jsonl"
progress_file = args.output / "progress.json"
existing = _read_rows(output_file)
total = len(records) * len(variants)
engine = create_engine(
config_path=str(args.config) if args.config else None,
device=None if args.device == "auto" else args.device,
)
engine.warmup()
completed = 0
errors = sum(row.get("status") == "error" for row in existing.values())
with output_file.open("a", encoding="utf-8", newline="\n") as stream:
for record in records:
for mode, ablate in variants:
key = "|".join((args.dataset, record.relative_path, mode, ",".join(sorted(ablate))))
completed += 1
if key in existing:
continue
started = time.perf_counter()
row: dict[str, Any] = {
"key": key,
"dataset": args.dataset,
"relative_path": record.relative_path,
"mode": mode,
"ablate": sorted(ablate),
"created_at": time.time(),
}
try:
preview_path = None
if args.save_images:
ablation_label = ",".join(sorted(ablate)) or "default"
preview_path = args.output / "previews" / safe_dataset / mode / ablation_label / Path(record.relative_path).with_suffix(".png")
row.update(_run_one(engine, record, mode, ablate, preview_path))
except Exception as exc:
errors += 1
row.update(status="error", error_type=type(exc).__name__, error=str(exc))
row["preview"] = (
str(preview_path.relative_to(args.output))
if preview_path is not None and preview_path.is_file()
else None
)
row["wall_ms"] = (time.perf_counter() - started) * 1000.0
stream.write(json.dumps(row, ensure_ascii=False, allow_nan=False) + "\n")
stream.flush()
os.fsync(stream.fileno())
_atomic_json(
progress_file,
{
"dataset": args.dataset,
"completed": completed,
"total": total,
"errors": errors,
"output": output_file.name,
},
)
print(f"[{completed}/{total}] {record.relative_path} · {mode} · {row['status']}")
_atomic_json(
progress_file,
{
"dataset": args.dataset,
"completed": total,
"total": total,
"errors": errors,
"output": output_file.name,
},
)
print(f"Evaluation rows: {output_file}")
print("Run `python server/eval/aggregate.py` to refresh tables and charts.")
return 0
if __name__ == "__main__":
raise SystemExit(main())
|