Spaces:
Running
Running
| """ | |
| core/layoutmesh.py — the DocBank annotator as one stage of a longer pipeline. | |
| The repository this addresses does one hard thing very well: it produces layout | |
| ground truth from the source rather than recovering it afterwards. It compiles a | |
| LaTeX template twice — once clean, once with every semantic element recoloured — | |
| and reads glyph boxes out of the coloured twin. Colour is a zero-dimension | |
| graphics-state change, so the two renders are layout-identical, and it *proves* | |
| that rather than assuming it: if a glyph moved, the run fails the fidelity gate. | |
| That is a generator of perfect labels. It is not a dataset platform, and it | |
| should not become one. So nothing here modifies it. It becomes `layout.docbank` | |
| — one coordinate, one stage — and the mesh supplies the stages around it: | |
| layout.template.find → layout.docbank → layout.validate → layout.augment | |
| → layout.export → layout.train → layout.evaluate → layout.publish | |
| Two things this module insists on. | |
| The fidelity gate is a capability boundary, not a warning. A template that fails | |
| it produced labels that do not describe the page, and those labels must never | |
| reach a dataset. A silent 3% of misaligned boxes is worse than a failed run, | |
| because it trains a model to be confidently wrong. | |
| And a profile is a claim about a document class. The repository ships three; | |
| anything else is `unprofiled` — which is a real, reportable state and the thing | |
| the mesh can go and satisfy, rather than a crash or a guess. | |
| """ | |
| from __future__ import annotations | |
| import re | |
| from pathlib import Path | |
| from typing import Any, Dict, List, Optional, Sequence | |
| SCHEMA = "signalmesh.layoutmesh/v1" | |
| # The pipeline this stage sits inside. Declaring the whole shape means a request | |
| # can be routed even while most of it is unbuilt — each stage reports its own | |
| # rung instead of the pipeline pretending to be all-or-nothing. | |
| STAGES: List[Dict[str, str]] = [ | |
| {"coord": "layout.template.find", "does": "locate or generate a LaTeX template"}, | |
| {"coord": "layout.generate", "does": "write the document, so the labels are " | |
| "known before the page exists"}, | |
| {"coord": "layout.docbank", "does": "compile twice and read exact ground truth"}, | |
| {"coord": "layout.validate", "does": "reject impossible boxes, empty classes, drift"}, | |
| {"coord": "layout.augment", "does": "scanner streaks, folds, skew, fax, phone camera"}, | |
| {"coord": "layout.export", "does": "COCO, YOLO, Pascal VOC, HF dataset"}, | |
| {"coord": "layout.train", "does": "RT-DETR, DocLayout-YOLO, DiT, LayoutLM"}, | |
| {"coord": "layout.evaluate", "does": "AP50, AP75, IoU, per-class weakness"}, | |
| {"coord": "layout.publish", "does": "dataset and weights, with provenance"}, | |
| ] | |
| # Implemented by the repository itself. Everything else in STAGES is a | |
| # coordinate the mesh must satisfy from somewhere, and saying which is which is | |
| # the difference between a pipeline and a wish. | |
| IMPLEMENTED = {"layout.docbank"} | |
| # Satisfied by the mesh, hosted, needing nothing installed. `layout.generate` | |
| # is the inverse of docbank rather than a copy of it: docbank recovers labels | |
| # from a document it did not write, the generator writes the document, so the | |
| # labels are decided before the page exists. Same fidelity gate either way. | |
| HOSTED = {"layout.generate": "core.layout_generate", | |
| "layout.export": "core.layout_export"} | |
| _DOCUMENTCLASS = re.compile(r"\\documentclass(?:\[[^\]]*\])?\s*\{([^}]+)\}") | |
| def profiles(root: Path) -> List[str]: | |
| """Document classes this checkout can instrument, from what it ships.""" | |
| d = Path(root) / "docbank" / "profiles" | |
| return sorted(p.stem for p in d.glob("*.sty")) if d.is_dir() else [] | |
| def templates(root: Path) -> List[str]: | |
| """Template directories available to annotate.""" | |
| d = Path(root) / "templates" | |
| return sorted(p.name for p in d.iterdir() if p.is_dir()) if d.is_dir() else [] | |
| def document_class(template_dir: Path) -> Optional[str]: | |
| """What class a template declares, read from its own .tex.""" | |
| d = Path(template_dir) | |
| if not d.is_dir(): | |
| return None | |
| for tex in sorted(d.rglob("*.tex")): | |
| try: | |
| m = _DOCUMENTCLASS.search(tex.read_text(encoding="utf-8", errors="ignore")) | |
| except OSError: | |
| continue | |
| if m: | |
| return m.group(1).strip() | |
| return None | |
| def match_profile(doc_class: Optional[str], available: Sequence[str]) -> Dict[str, Any]: | |
| """Can this template be instrumented, and by what. | |
| An unmatched class is `unprofiled`, not an error. It is the exact request | |
| the mesh can broadcast — "this class needs a profile" — and the thing a new | |
| contributed profile satisfies. | |
| """ | |
| if not doc_class: | |
| return {"profile": None, "state": "unknown", | |
| "why": "no \\documentclass found — cannot tell what this is"} | |
| if doc_class in available: | |
| return {"profile": doc_class, "state": "profiled", | |
| "why": f"ships a profile for {doc_class}"} | |
| return {"profile": None, "state": "unprofiled", "document_class": doc_class, | |
| "why": f"no instrumentation profile for {doc_class!r}", | |
| "satisfied_by": "layout.profile.synthesize", | |
| "note": ("a draft profile can be generated and accepted only if it " | |
| "passes the fidelity gate — an unproven profile is worse " | |
| "than none, because it yields boxes that look fine")} | |
| def plan(root: Path, want: Sequence[str] = ()) -> Dict[str, Any]: | |
| """What this checkout can do now, and what the mesh has to supply.""" | |
| root = Path(root) | |
| avail = profiles(root) | |
| tmpl = templates(root) | |
| surveyed = [] | |
| for name in tmpl: | |
| cls = document_class(root / "templates" / name) | |
| surveyed.append({"template": name, "document_class": cls, | |
| **match_profile(cls, avail)}) | |
| ready = [t for t in surveyed if t["state"] == "profiled"] | |
| return { | |
| "schema": SCHEMA, | |
| "root": str(root), | |
| "profiles": avail, | |
| "templates": surveyed, | |
| "annotatable_now": [t["template"] for t in ready], | |
| "needs_profile": [t["template"] for t in surveyed if t["state"] == "unprofiled"], | |
| "stages": [{**s, "implemented": s["coord"] in IMPLEMENTED or s["coord"] in HOSTED, | |
| "by": "this repository" if s["coord"] in IMPLEMENTED | |
| else HOSTED.get(s["coord"], "another coordinate")} | |
| for s in STAGES], | |
| "requires": ["latexmk", "a LaTeX distribution"], | |
| "note": ("the fidelity gate decides whether a run counts — a template " | |
| "that fails it produced labels that do not describe the page"), | |
| } | |
| def command(root: Path, template: str = "", *, all_templates: bool = False, | |
| profile: str = "", overlay: bool = True) -> List[str]: | |
| """The exact invocation, matching the CLI the repository actually exposes. | |
| Checked against its argparse: positional templates, --all, --profile, | |
| --no-overlay. Nothing invented; a command nobody can run is worse than no | |
| command at all. | |
| """ | |
| cmd = ["python", str(Path(root) / "docbank" / "annotate.py")] | |
| if all_templates: | |
| cmd.append("--all") | |
| elif template: | |
| cmd.append(template) | |
| else: | |
| raise ValueError("give a template, or ask for all of them — the CLI " | |
| "requires one or the other") | |
| if profile: | |
| cmd += ["--profile", profile] | |
| if not overlay: | |
| cmd.append("--no-overlay") | |
| return cmd | |
| if __name__ == "__main__": | |
| import tempfile | |
| with tempfile.TemporaryDirectory() as tmp: | |
| root = Path(tmp) | |
| (root / "docbank" / "profiles").mkdir(parents=True) | |
| for p in ("article", "beamer", "news"): | |
| (root / "docbank" / "profiles" / f"{p}.sty").write_text("%") | |
| (root / "templates" / "a_paper").mkdir(parents=True) | |
| (root / "templates" / "a_paper" / "main.tex").write_text( | |
| "\\documentclass[11pt]{article}\n\\begin{document}x\\end{document}") | |
| (root / "templates" / "a_poster").mkdir(parents=True) | |
| (root / "templates" / "a_poster" / "p.tex").write_text( | |
| "\\documentclass{tikzposter}\n") | |
| (root / "templates" / "empty").mkdir(parents=True) | |
| assert profiles(root) == ["article", "beamer", "news"] | |
| assert templates(root) == ["a_paper", "a_poster", "empty"] | |
| assert document_class(root / "templates" / "a_paper") == "article" | |
| assert document_class(root / "templates" / "a_poster") == "tikzposter" | |
| assert document_class(root / "templates" / "empty") is None | |
| p = plan(root) | |
| assert p["annotatable_now"] == ["a_paper"] | |
| assert p["needs_profile"] == ["a_poster"], p["needs_profile"] | |
| # an unknown class is a request, not a crash | |
| poster = next(t for t in p["templates"] if t["template"] == "a_poster") | |
| assert poster["state"] == "unprofiled" | |
| assert poster["satisfied_by"] == "layout.profile.synthesize" | |
| assert "worse than none" in poster["note"] | |
| blank = next(t for t in p["templates"] if t["template"] == "empty") | |
| assert blank["state"] == "unknown" | |
| # the pipeline is declared whole, and honest about what exists | |
| impl = [s for s in p["stages"] if s["implemented"]] | |
| assert [s["coord"] for s in impl] == ["layout.generate", "layout.docbank", | |
| "layout.export"], impl | |
| assert len(p["stages"]) == 9 | |
| assert all(s["by"] == "another coordinate" | |
| for s in p["stages"] if not s["implemented"]) | |
| gen = next(s for s in p["stages"] if s["coord"] == "layout.generate") | |
| assert gen["by"] == "core.layout_generate", "a hosted stage names its module" | |
| # commands match the CLI the repository exposes, not a nicer one | |
| assert command(root, "a_paper")[-1] == "a_paper" | |
| assert "--all" in command(root, all_templates=True) | |
| assert command(root, "a_paper", profile="news")[-2:] == ["--profile", "news"] | |
| assert command(root, "a_paper", overlay=False)[-1] == "--no-overlay" | |
| try: | |
| command(root) | |
| raise AssertionError("the CLI requires a template or --all") | |
| except ValueError: | |
| pass | |
| # ── against the real checkout, if it is here ───────────────────────────── | |
| real = Path.home() / "Downloads" / "fixMesh" / "layout-detection-synthetic-data-main" | |
| if (real / "docbank" / "annotate.py").exists(): | |
| rp = plan(real) | |
| assert rp["profiles"], "the checkout ships no profiles?" | |
| assert rp["templates"], "the checkout ships no templates?" | |
| print(f"real repo: profiles={rp['profiles']} " | |
| f"ready={rp['annotatable_now']} needs_profile={rp['needs_profile']}") | |
| # the CLI really does accept what command() emits | |
| cli = (real / "docbank" / "annotate.py").read_text(errors="ignore") | |
| for flag in ("--all", "--profile", "--no-overlay"): | |
| assert flag in cli, f"{flag} is not a real flag of this CLI" | |
| print("layoutmesh self-check OK") | |