File size: 6,124 Bytes
1b4b6d2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
"""
QTO validation harness — references real villa projects IN PLACE (no copying,
so the repo stays lean and performance is unaffected).

A project qualifies for validation only if it has all three:
    • architectural drawing  (PDF, name ~ "arch")
    • structural drawing     (PDF, name ~ "str"/"struct")
    • ground-truth BOQ        (Excel, not a system output)

What this does now (cheap, no AI):
    1. Auto-discover qualifying projects under --root.
    2. Parse each ground-truth BOQ into normalized {item: qty}
       (reuses _extract_real_boqs.extract_from_workbook).
    3. Write validation_set/manifest.json (paths + parsed ground truth only).

The heavy drawing→BOQ accuracy run (which calls the AI extractor) is intentionally
opt-in via --run-engine so a full sweep never fires by accident. See README.md.

Usage:
    python validation_set/run_validation.py                 # default root = trainning
    python validation_set/run_validation.py --root "B:\\path\\to\\folder"
"""
import argparse
import json
import os
import re
import sys

_ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, _ROOT_DIR)

from _extract_real_boqs import extract_from_workbook  # reuse the existing BOQ parser

DEFAULT_ROOT = os.environ.get("VALIDATION_ROOT", r"B:\work\trainning")

# Excel files that are OUR system's output, not the human ground-truth BOQ
_SYS_OUTPUT_RX = re.compile(r"qto.?hybrid|hybrid.?test|_qto_test|project_\d+_qto|_qto\.xls", re.I)


def _find(files, *patterns):
    for pat in patterns:
        for f in files:
            if re.search(pat, os.path.basename(f), re.I):
                return f
    return None


def discover(root):
    """Return a list of project dicts with detected arch/str/boq paths."""
    projects = []
    if not os.path.isdir(root):
        return projects
    for name in sorted(os.listdir(root)):
        d = os.path.join(root, name)
        if not os.path.isdir(d):
            continue
        files = [os.path.join(d, f) for f in os.listdir(d) if os.path.isfile(os.path.join(d, f))]
        pdfs = [f for f in files if f.lower().endswith(".pdf")]
        xls = [f for f in files if f.lower().endswith((".xls", ".xlsx"))]

        # Structural abbreviations seen in the wild: str, struct, stc, STC.
        strc = _find(pdfs, r"st[rc]", r"struct")
        # Architectural: arch/architectural/architecture, or an "AR-/AR_/AR " prefix.
        arch = _find([f for f in pdfs if f != strc], r"arch", r"^ar[-_ ]", r"^ar\d")
        # A villa folder holds exactly the two drawings — infer the missing one
        # as "the other PDF" so odd naming never drops a real pair.
        if len(pdfs) == 2:
            if strc and not arch:
                arch = next((f for f in pdfs if f != strc), None)
            elif arch and not strc:
                strc = next((f for f in pdfs if f != arch), None)

        # ground-truth BOQ: drop system outputs; prefer a 'boq'-named file, else the largest
        gt_cands = [f for f in xls if not _SYS_OUTPUT_RX.search(os.path.basename(f))]
        boq = _find(gt_cands, r"boq", r"bill", r"quantit")
        if not boq and gt_cands:
            boq = max(gt_cands, key=lambda f: os.path.getsize(f))

        projects.append({
            "name": name,
            "dir": d,
            "arch_pdf": arch,
            "str_pdf": strc,
            "boq": boq,
            "complete": bool(arch and strc and boq),
        })
    return projects


def main():
    ap = argparse.ArgumentParser(description="QTO validation harness (references files in place).")
    ap.add_argument("--root", default=DEFAULT_ROOT, help="folder whose subfolders are projects")
    ap.add_argument("--out", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), "manifest.json"))
    ap.add_argument("--run-engine", action="store_true",
                    help="(heavy, calls AI) run drawings through the engine and score accuracy")
    args = ap.parse_args()

    projects = discover(args.root)
    complete = [p for p in projects if p["complete"]]
    incomplete = [p for p in projects if not p["complete"]]

    print(f"Root: {args.root}")
    print(f"Folders: {len(projects)} | qualifying (arch+str+BOQ): {len(complete)} | incomplete: {len(incomplete)}\n")

    report = []
    for p in complete:
        try:
            gt = extract_from_workbook(p["boq"])
        except Exception as e:
            print(f"[{p['name']}] BOQ parse error: {e}")
            gt = {}
        gt_simple = {k: round(v["qty"], 2) for k, v in gt.items()}
        report.append({
            "name": p["name"],
            "arch_pdf": p["arch_pdf"],
            "str_pdf": p["str_pdf"],
            "boq": p["boq"],
            "boq_item_count": len(gt_simple),
            "boq_items": gt_simple,
        })
        sample = ", ".join(sorted(gt_simple)[:6])
        print(f"[{p['name']:<3}] ground-truth BOQ items: {len(gt_simple):>2}  ({sample}{'...' if len(gt_simple) > 6 else ''})")

    for p in incomplete:
        missing = [k.replace('_pdf', '').replace('boq', 'BOQ') for k in ("arch_pdf", "str_pdf", "boq") if not p[k]]
        print(f"[{p['name']:<3}] SKIP — missing: {', '.join(missing)}")

    manifest = {
        "root": args.root,
        "generated_by": "validation_set/run_validation.py",
        "note": "Drawings/BOQ are referenced in place (never copied) to keep the repo lean.",
        "qualifying_count": len(complete),
        "complete_projects": report,
        "incomplete_projects": [
            {"name": p["name"], "missing": [k for k in ("arch_pdf", "str_pdf", "boq") if not p[k]]}
            for p in incomplete
        ],
    }
    with open(args.out, "w", encoding="utf-8") as f:
        json.dump(manifest, f, indent=2, ensure_ascii=False, default=str)
    print(f"\nManifest -> {args.out}  ({len(complete)} validation-ready projects)")

    if args.run_engine:
        print("\n--run-engine requested: the AI accuracy sweep is a separate step "
              "(needs AI_API_KEY and will incur Gemini cost). Not implemented in this cheap pass.")


if __name__ == "__main__":
    main()