""" core/layoutmesh.py — the DocBank annotator as one stage of a longer pipeline. The repository this addresses does one hard thing very well: it produces layout ground truth from the source rather than recovering it afterwards. It compiles a LaTeX template twice — once clean, once with every semantic element recoloured — and reads glyph boxes out of the coloured twin. Colour is a zero-dimension graphics-state change, so the two renders are layout-identical, and it *proves* that rather than assuming it: if a glyph moved, the run fails the fidelity gate. That is a generator of perfect labels. It is not a dataset platform, and it should not become one. So nothing here modifies it. It becomes `layout.docbank` — one coordinate, one stage — and the mesh supplies the stages around it: layout.template.find → layout.docbank → layout.validate → layout.augment → layout.export → layout.train → layout.evaluate → layout.publish Two things this module insists on. The fidelity gate is a capability boundary, not a warning. A template that fails it produced labels that do not describe the page, and those labels must never reach a dataset. A silent 3% of misaligned boxes is worse than a failed run, because it trains a model to be confidently wrong. And a profile is a claim about a document class. The repository ships three; anything else is `unprofiled` — which is a real, reportable state and the thing the mesh can go and satisfy, rather than a crash or a guess. """ from __future__ import annotations import re from pathlib import Path from typing import Any, Dict, List, Optional, Sequence SCHEMA = "signalmesh.layoutmesh/v1" # The pipeline this stage sits inside. Declaring the whole shape means a request # can be routed even while most of it is unbuilt — each stage reports its own # rung instead of the pipeline pretending to be all-or-nothing. STAGES: List[Dict[str, str]] = [ {"coord": "layout.template.find", "does": "locate or generate a LaTeX template"}, {"coord": "layout.generate", "does": "write the document, so the labels are " "known before the page exists"}, {"coord": "layout.docbank", "does": "compile twice and read exact ground truth"}, {"coord": "layout.validate", "does": "reject impossible boxes, empty classes, drift"}, {"coord": "layout.augment", "does": "scanner streaks, folds, skew, fax, phone camera"}, {"coord": "layout.export", "does": "COCO, YOLO, Pascal VOC, HF dataset"}, {"coord": "layout.train", "does": "RT-DETR, DocLayout-YOLO, DiT, LayoutLM"}, {"coord": "layout.evaluate", "does": "AP50, AP75, IoU, per-class weakness"}, {"coord": "layout.publish", "does": "dataset and weights, with provenance"}, ] # Implemented by the repository itself. Everything else in STAGES is a # coordinate the mesh must satisfy from somewhere, and saying which is which is # the difference between a pipeline and a wish. IMPLEMENTED = {"layout.docbank"} # Satisfied by the mesh, hosted, needing nothing installed. `layout.generate` # is the inverse of docbank rather than a copy of it: docbank recovers labels # from a document it did not write, the generator writes the document, so the # labels are decided before the page exists. Same fidelity gate either way. HOSTED = {"layout.generate": "core.layout_generate", "layout.export": "core.layout_export"} _DOCUMENTCLASS = re.compile(r"\\documentclass(?:\[[^\]]*\])?\s*\{([^}]+)\}") def profiles(root: Path) -> List[str]: """Document classes this checkout can instrument, from what it ships.""" d = Path(root) / "docbank" / "profiles" return sorted(p.stem for p in d.glob("*.sty")) if d.is_dir() else [] def templates(root: Path) -> List[str]: """Template directories available to annotate.""" d = Path(root) / "templates" return sorted(p.name for p in d.iterdir() if p.is_dir()) if d.is_dir() else [] def document_class(template_dir: Path) -> Optional[str]: """What class a template declares, read from its own .tex.""" d = Path(template_dir) if not d.is_dir(): return None for tex in sorted(d.rglob("*.tex")): try: m = _DOCUMENTCLASS.search(tex.read_text(encoding="utf-8", errors="ignore")) except OSError: continue if m: return m.group(1).strip() return None def match_profile(doc_class: Optional[str], available: Sequence[str]) -> Dict[str, Any]: """Can this template be instrumented, and by what. An unmatched class is `unprofiled`, not an error. It is the exact request the mesh can broadcast — "this class needs a profile" — and the thing a new contributed profile satisfies. """ if not doc_class: return {"profile": None, "state": "unknown", "why": "no \\documentclass found — cannot tell what this is"} if doc_class in available: return {"profile": doc_class, "state": "profiled", "why": f"ships a profile for {doc_class}"} return {"profile": None, "state": "unprofiled", "document_class": doc_class, "why": f"no instrumentation profile for {doc_class!r}", "satisfied_by": "layout.profile.synthesize", "note": ("a draft profile can be generated and accepted only if it " "passes the fidelity gate — an unproven profile is worse " "than none, because it yields boxes that look fine")} def plan(root: Path, want: Sequence[str] = ()) -> Dict[str, Any]: """What this checkout can do now, and what the mesh has to supply.""" root = Path(root) avail = profiles(root) tmpl = templates(root) surveyed = [] for name in tmpl: cls = document_class(root / "templates" / name) surveyed.append({"template": name, "document_class": cls, **match_profile(cls, avail)}) ready = [t for t in surveyed if t["state"] == "profiled"] return { "schema": SCHEMA, "root": str(root), "profiles": avail, "templates": surveyed, "annotatable_now": [t["template"] for t in ready], "needs_profile": [t["template"] for t in surveyed if t["state"] == "unprofiled"], "stages": [{**s, "implemented": s["coord"] in IMPLEMENTED or s["coord"] in HOSTED, "by": "this repository" if s["coord"] in IMPLEMENTED else HOSTED.get(s["coord"], "another coordinate")} for s in STAGES], "requires": ["latexmk", "a LaTeX distribution"], "note": ("the fidelity gate decides whether a run counts — a template " "that fails it produced labels that do not describe the page"), } def command(root: Path, template: str = "", *, all_templates: bool = False, profile: str = "", overlay: bool = True) -> List[str]: """The exact invocation, matching the CLI the repository actually exposes. Checked against its argparse: positional templates, --all, --profile, --no-overlay. Nothing invented; a command nobody can run is worse than no command at all. """ cmd = ["python", str(Path(root) / "docbank" / "annotate.py")] if all_templates: cmd.append("--all") elif template: cmd.append(template) else: raise ValueError("give a template, or ask for all of them — the CLI " "requires one or the other") if profile: cmd += ["--profile", profile] if not overlay: cmd.append("--no-overlay") return cmd if __name__ == "__main__": import tempfile with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) (root / "docbank" / "profiles").mkdir(parents=True) for p in ("article", "beamer", "news"): (root / "docbank" / "profiles" / f"{p}.sty").write_text("%") (root / "templates" / "a_paper").mkdir(parents=True) (root / "templates" / "a_paper" / "main.tex").write_text( "\\documentclass[11pt]{article}\n\\begin{document}x\\end{document}") (root / "templates" / "a_poster").mkdir(parents=True) (root / "templates" / "a_poster" / "p.tex").write_text( "\\documentclass{tikzposter}\n") (root / "templates" / "empty").mkdir(parents=True) assert profiles(root) == ["article", "beamer", "news"] assert templates(root) == ["a_paper", "a_poster", "empty"] assert document_class(root / "templates" / "a_paper") == "article" assert document_class(root / "templates" / "a_poster") == "tikzposter" assert document_class(root / "templates" / "empty") is None p = plan(root) assert p["annotatable_now"] == ["a_paper"] assert p["needs_profile"] == ["a_poster"], p["needs_profile"] # an unknown class is a request, not a crash poster = next(t for t in p["templates"] if t["template"] == "a_poster") assert poster["state"] == "unprofiled" assert poster["satisfied_by"] == "layout.profile.synthesize" assert "worse than none" in poster["note"] blank = next(t for t in p["templates"] if t["template"] == "empty") assert blank["state"] == "unknown" # the pipeline is declared whole, and honest about what exists impl = [s for s in p["stages"] if s["implemented"]] assert [s["coord"] for s in impl] == ["layout.generate", "layout.docbank", "layout.export"], impl assert len(p["stages"]) == 9 assert all(s["by"] == "another coordinate" for s in p["stages"] if not s["implemented"]) gen = next(s for s in p["stages"] if s["coord"] == "layout.generate") assert gen["by"] == "core.layout_generate", "a hosted stage names its module" # commands match the CLI the repository exposes, not a nicer one assert command(root, "a_paper")[-1] == "a_paper" assert "--all" in command(root, all_templates=True) assert command(root, "a_paper", profile="news")[-2:] == ["--profile", "news"] assert command(root, "a_paper", overlay=False)[-1] == "--no-overlay" try: command(root) raise AssertionError("the CLI requires a template or --all") except ValueError: pass # ── against the real checkout, if it is here ───────────────────────────── real = Path.home() / "Downloads" / "fixMesh" / "layout-detection-synthetic-data-main" if (real / "docbank" / "annotate.py").exists(): rp = plan(real) assert rp["profiles"], "the checkout ships no profiles?" assert rp["templates"], "the checkout ships no templates?" print(f"real repo: profiles={rp['profiles']} " f"ready={rp['annotatable_now']} needs_profile={rp['needs_profile']}") # the CLI really does accept what command() emits cli = (real / "docbank" / "annotate.py").read_text(errors="ignore") for flag in ("--all", "--profile", "--no-overlay"): assert flag in cli, f"{flag} is not a real flag of this CLI" print("layoutmesh self-check OK")