| """The page audit on pages collected after everything was fixed: 12 sites, three pages each, none seen before.
|
|
|
| Nothing is calibrated here. The tolerance (tolerances.json), the per-line scale (multi_line.json), the card-size
|
| and comparable-size limits (group_checks.py) and the brief are the ones set on the first 24 home pages. This
|
| script only collects and counts.
|
|
|
| Contrast is compared with the page rendered twice, with and without its text (page_notext.png), not with the
|
| declared colours, so the reference holds over photos and gradients too.
|
|
|
| Usage: evaluate_fresh.py collect|report [fresh|dev|third] [clean] ("clean" skips the injected faults)
|
| """
|
|
|
| import json
|
| import sys
|
| from pathlib import Path
|
|
|
| sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
| PACKAGE = Path(__file__).resolve().parent
|
| sys.path.insert(0, str(PACKAGE))
|
|
|
| import numpy
|
| from PIL import Image
|
| import evaluate_generalisation as evaluation
|
|
|
| Image.MAX_IMAGE_PIXELS = None
|
| PAGES, OUTPUT = evaluation.SETS[sys.argv[2] if len(sys.argv) > 2 else "fresh"]
|
| VARIANTS = ("clean",) if "clean" in sys.argv[3:] else ("clean", *evaluation.FAULTS)
|
|
|
|
|
| def read_picture(path):
|
| with Image.open(path) as image:
|
| return numpy.asarray(image.convert("RGB"))
|
|
|
|
|
| def collect():
|
| evaluation.SITES = PAGES
|
| evaluation.OUTPUT = OUTPUT
|
| evaluation.collect(VARIANTS)
|
|
|
|
|
| def report():
|
| import page_audit
|
|
|
| read = page_audit.load_read()
|
| tolerances = page_audit.load_tolerances()
|
| audits = {}
|
| layouts = {}
|
| rows = []
|
| for name in PAGES:
|
| for variant in ("clean", *evaluation.FAULTS):
|
| directory = OUTPUT / name / variant
|
| if not (directory / "layout.json").exists():
|
| continue
|
| layouts[name, variant] = json.loads((directory / "layout.json").read_text(encoding="utf-8"))
|
| audits[name, variant] = page_audit.judge(page_audit.measure(directory), page_audit.DEFAULT_BRIEF, read, tolerances)
|
| if (name, "clean") in audits:
|
| pictures = (read_picture(OUTPUT / name / "clean" / "page.png"), read_picture(OUTPUT / name / "clean" / "page_notext.png"))
|
| rows.extend(evaluation.pixel_rows(name, audits[name, "clean"], layouts[name, "clean"], pictures))
|
| pages = [name for name in PAGES if (name, "clean") in audits]
|
|
|
| dom_rows = []
|
| for name in pages:
|
| directory = OUTPUT / name / "clean"
|
| if "computed" in layouts[name, "clean"]["texts"][0]:
|
| results = page_audit.judge(page_audit.measure(directory, "dom"), page_audit.DEFAULT_BRIEF, read)
|
| dom_rows.extend(row for row in evaluation.pixel_rows(name, results, layouts[name, "clean"]) if row["check"] != "text_contrast")
|
|
|
| injection = {}
|
| for fault, kind in evaluation.FAULTS.items():
|
| entry = injection.setdefault(fault, {"check": kind, "pages_with_a_target": 0, "caught": 0, "missed_on": []})
|
| for name in pages:
|
| if not layouts.get((name, fault), {}).get("fault_applied"):
|
| continue
|
| entry["pages_with_a_target"] += 1
|
| if evaluation.flagged(audits[name, fault], kind) > evaluation.flagged(audits[name, "clean"], kind):
|
| entry["caught"] += 1
|
| else:
|
| entry["missed_on"].append(name)
|
| untouched = {}
|
| for kind in evaluation.GEOMETRY_CHECKS:
|
| results = [result for name in pages for result in audits[name, "clean"] if result["check"] == kind]
|
| if results:
|
| flagged = [result for result in results if result["advice"] != "keep"]
|
| untouched[kind] = {"checks": len(results), "flagged": len(flagged), "flag_rate": len(flagged) / len(results)}
|
| relative_error = {kind: {"checks": len(values), "median": float(numpy.median(values)), "percentile_80": float(numpy.percentile(values, 80))}
|
| for kind in evaluation.PIXEL_CHECKS
|
| for values in [[abs(row["ratio"] / row["truth"] - 1) for row in rows if row["check"] == kind]] if values}
|
| summary = {"pages_collected": pages, "pages_failed": [name for name in PAGES if name not in pages],
|
| "repeated_groups": {name: len(layouts[name, "clean"]["repeats"]) for name in pages},
|
| "tolerance_used": tolerances, "pixel_relative_error": relative_error,
|
| "pixel_verdicts": {"without_tolerance": evaluation.agreement(rows, {}), "with_tolerance": evaluation.agreement(rows, tolerances),
|
| "dom_mode": evaluation.agreement(dom_rows, {})},
|
| "fault_injection": injection, "geometry_flags_on_untouched_pages": untouched}
|
| (OUTPUT / "report.json").write_text(json.dumps(summary, indent=1), encoding="utf-8")
|
| print("pages", len(pages), "failed", summary["pages_failed"])
|
| print("pages with a repeated group", sum(count > 0 for count in summary["repeated_groups"].values()))
|
| print("relative error", {kind: (entry["checks"], round(entry["median"], 3), round(entry["percentile_80"], 3)) for kind, entry in relative_error.items()})
|
| for label, table in summary["pixel_verdicts"].items():
|
| print(label)
|
| for kind, entry in table.items():
|
| print(" ", kind, {key: (round(value, 3) if isinstance(value, float) else value) for key, value in entry.items()})
|
| for fault, entry in injection.items():
|
| print("fault", fault, entry)
|
| for kind, entry in untouched.items():
|
| print("untouched", kind, entry["checks"], entry["flagged"], round(entry["flag_rate"], 3))
|
|
|
|
|
| if __name__ == "__main__":
|
| collect() if sys.argv[1] == "collect" else report()
|
|
|
