Download server/eval/evaluate.py from MohithD05/LumenX: direct link, hf CLI and curl.
- Browser
- Download file 9.49 kB
-
https://huggingface.co/spaces/MohithD05/LumenX/resolve/main/server/eval/evaluate.py
- Command line
-
hf download hf://spaces/MohithD05/LumenX/server/eval/evaluate.py
-
curl -L -o evaluate.py https://huggingface.co/spaces/MohithD05/LumenX/resolve/main/server/eval/evaluate.py
9.49 kB
| """Run resumable paired/unpaired and ablation evaluation on a local dataset.""" | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import re | |
| import sys | |
| import time | |
| from pathlib import Path | |
| from typing import Any | |
| import numpy as np | |
| from PIL import Image | |
| REPO = Path(__file__).resolve().parents[2] | |
| sys.path.insert(0, str(REPO / "core")) | |
| sys.path.insert(0, str(REPO)) | |
| from lumenx_core import create_engine # noqa: E402 | |
| from lumenx_core.contracts import EnhanceRequest, EnhanceResult, StageEvent # noqa: E402 | |
| from lumenx_core.contract_check import validate_run # noqa: E402 | |
| from common import align_reference, as_rgb_u8, decode_path, discover_records # noqa: E402 | |
| def _read_rows(path: Path) -> dict[str, dict[str, Any]]: | |
| rows: dict[str, dict[str, Any]] = {} | |
| if not path.is_file(): | |
| return rows | |
| with path.open("r", encoding="utf-8") as stream: | |
| for line in stream: | |
| try: | |
| row = json.loads(line) | |
| except json.JSONDecodeError: | |
| continue | |
| if isinstance(row, dict) and row.get("key"): | |
| rows[str(row["key"])] = row | |
| return rows | |
| def _atomic_json(path: Path, value: dict[str, Any]) -> None: | |
| temporary = path.with_suffix(path.suffix + ".tmp") | |
| temporary.write_text(json.dumps(value, indent=2, allow_nan=False), encoding="utf-8") | |
| os.replace(temporary, path) | |
| def _variants(modes: list[str], ablations: list[str]) -> list[tuple[str, frozenset[str]]]: | |
| variants = [(mode, frozenset()) for mode in modes] | |
| variants.extend(("hybrid", frozenset({flag})) for flag in ablations) | |
| return variants | |
| def _run_one( | |
| engine: Any, | |
| record: Any, | |
| mode: str, | |
| ablate: frozenset[str], | |
| preview_path: Path | None = None, | |
| ) -> dict[str, Any]: | |
| decoded = decode_path(record.input_path) | |
| image = decoded.image | |
| reference = None | |
| if record.reference_path: | |
| reference = as_rgb_u8(decode_path(record.reference_path)) | |
| reference = align_reference(reference, image.shape[:2]) | |
| request = EnhanceRequest( | |
| image=image, | |
| original_size=decoded.original_size, | |
| alpha=decoded.alpha, | |
| mode=mode, | |
| ablate=ablate, | |
| seed=0, | |
| ) | |
| started = time.perf_counter() | |
| items = list(engine.run(request)) | |
| elapsed_ms = (time.perf_counter() - started) * 1000.0 | |
| violations = validate_run(items, request) | |
| if violations: | |
| raise RuntimeError("Core stream contract violations: " + "; ".join(violations)) | |
| result = items[-1] | |
| if not isinstance(result, EnhanceResult): | |
| raise RuntimeError("Engine stream ended without an EnhanceResult") | |
| final_visual = next((visual for visual in result.visuals if visual.key == "final"), None) | |
| if final_visual is None: | |
| raise RuntimeError("Engine did not return a final image") | |
| output = final_visual.image[..., :3] | |
| if preview_path is not None: | |
| preview_path.parent.mkdir(parents=True, exist_ok=True) | |
| Image.fromarray(output).save(preview_path, format="PNG", optimize=False) | |
| suite = engine.metric_suite() | |
| input_u8 = as_rgb_u8(decoded) | |
| custom = suite.custom(input_u8, output) | |
| unpaired = suite.unpaired(output) | |
| paired = suite.paired(output, reference) if reference is not None else {} | |
| confidence = suite.confidence_errors(input_u8, output, reference) if reference is not None else {} | |
| return { | |
| "status": "complete", | |
| "relative_path": record.relative_path, | |
| "mode": mode, | |
| "ablate": sorted(ablate), | |
| "actual_method": result.provenance.get("actual_method"), | |
| "fallback_chain": result.provenance.get("fallback_chain", []), | |
| "paired": reference is not None, | |
| "metrics": {**paired, **unpaired, **custom, **confidence}, | |
| "stage_timings_ms": result.timings_ms, | |
| "latency_ms": elapsed_ms, | |
| "guardrail_failures": [check.check for check in result.guardrails if check.passed is False], | |
| "warnings": [warning.code for warning in result.warnings], | |
| "metric_unavailable": suite.unavailable(), | |
| "provenance": result.provenance, | |
| } | |
| def main() -> int: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--dataset", required=True, help="Dataset label used in output rows") | |
| parser.add_argument("--root", required=True, type=Path, help="Dataset root") | |
| parser.add_argument("--input-dir", type=Path, help="Optional low-light image directory") | |
| parser.add_argument("--reference-dir", type=Path, help="Optional ground-truth directory") | |
| parser.add_argument("--paired", action="store_true", help="Require matched reference images") | |
| parser.add_argument("--modes", default="classical,dl,hybrid", help="Comma-separated processing modes") | |
| parser.add_argument("--ablations", default="", help="Comma-separated hybrid ablation flags") | |
| parser.add_argument("--device", default="auto") | |
| parser.add_argument("--config", type=Path) | |
| parser.add_argument("--output", type=Path, default=REPO / "server" / "results") | |
| parser.add_argument("--limit", type=int, default=0, help="Optional maximum number of source images") | |
| parser.add_argument("--save-images", action="store_true", help="Save final images under results/previews for manual failure review") | |
| args = parser.parse_args() | |
| modes = [mode.strip() for mode in args.modes.split(",") if mode.strip()] | |
| if not modes or any(mode not in {"classical", "dl", "hybrid", "zerodce"} for mode in modes): | |
| parser.error("--modes must contain classical, dl, hybrid and/or zerodce") | |
| allowed_flags = {"no_tuner", "no_fusion", "no_classical", "no_dl", "no_gain_denoise", "no_recoverability", "no_guardrails", "no_color_correction"} | |
| ablations = [flag.strip() for flag in args.ablations.split(",") if flag.strip()] | |
| invalid = set(ablations) - allowed_flags | |
| if invalid: | |
| parser.error("unknown ablation flag(s): " + ", ".join(sorted(invalid))) | |
| records = discover_records( | |
| args.root, | |
| input_dir=args.input_dir, | |
| reference_dir=args.reference_dir, | |
| paired=args.paired, | |
| ) | |
| if args.limit > 0: | |
| records = records[: args.limit] | |
| variants = _variants(modes, ablations) | |
| args.output.mkdir(parents=True, exist_ok=True) | |
| safe_dataset = re.sub(r"[^A-Za-z0-9_-]+", "_", args.dataset).strip("_") or "dataset" | |
| output_file = args.output / f"{safe_dataset}_evaluation.jsonl" | |
| progress_file = args.output / "progress.json" | |
| existing = _read_rows(output_file) | |
| total = len(records) * len(variants) | |
| engine = create_engine( | |
| config_path=str(args.config) if args.config else None, | |
| device=None if args.device == "auto" else args.device, | |
| ) | |
| engine.warmup() | |
| completed = 0 | |
| errors = sum(row.get("status") == "error" for row in existing.values()) | |
| with output_file.open("a", encoding="utf-8", newline="\n") as stream: | |
| for record in records: | |
| for mode, ablate in variants: | |
| key = "|".join((args.dataset, record.relative_path, mode, ",".join(sorted(ablate)))) | |
| completed += 1 | |
| if key in existing: | |
| continue | |
| started = time.perf_counter() | |
| row: dict[str, Any] = { | |
| "key": key, | |
| "dataset": args.dataset, | |
| "relative_path": record.relative_path, | |
| "mode": mode, | |
| "ablate": sorted(ablate), | |
| "created_at": time.time(), | |
| } | |
| try: | |
| preview_path = None | |
| if args.save_images: | |
| ablation_label = ",".join(sorted(ablate)) or "default" | |
| preview_path = args.output / "previews" / safe_dataset / mode / ablation_label / Path(record.relative_path).with_suffix(".png") | |
| row.update(_run_one(engine, record, mode, ablate, preview_path)) | |
| except Exception as exc: | |
| errors += 1 | |
| row.update(status="error", error_type=type(exc).__name__, error=str(exc)) | |
| row["preview"] = ( | |
| str(preview_path.relative_to(args.output)) | |
| if preview_path is not None and preview_path.is_file() | |
| else None | |
| ) | |
| row["wall_ms"] = (time.perf_counter() - started) * 1000.0 | |
| stream.write(json.dumps(row, ensure_ascii=False, allow_nan=False) + "\n") | |
| stream.flush() | |
| os.fsync(stream.fileno()) | |
| _atomic_json( | |
| progress_file, | |
| { | |
| "dataset": args.dataset, | |
| "completed": completed, | |
| "total": total, | |
| "errors": errors, | |
| "output": output_file.name, | |
| }, | |
| ) | |
| print(f"[{completed}/{total}] {record.relative_path} · {mode} · {row['status']}") | |
| _atomic_json( | |
| progress_file, | |
| { | |
| "dataset": args.dataset, | |
| "completed": total, | |
| "total": total, | |
| "errors": errors, | |
| "output": output_file.name, | |
| }, | |
| ) | |
| print(f"Evaluation rows: {output_file}") | |
| print("Run `python server/eval/aggregate.py` to refresh tables and charts.") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |