File size: 4,817 Bytes
1b4b6d2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 | """
Batch-run the REAL QTO pipeline over several villa projects and score each output
against its ground-truth BOQ at the reference-item level (engine keys -> reference
keys via engine.result_validator._ALIAS, then aggregated).
Usage:
python validation_set/run_batch.py --root "B:\\work\\trainning" --projects 2 3 4 5 6
"""
import argparse
import json
import os
import re
import sys
from collections import defaultdict
_HERE = os.path.dirname(os.path.abspath(__file__))
_ROOT = os.path.dirname(_HERE)
sys.path.insert(0, _ROOT)
sys.path.insert(0, _HERE)
from run_engine import run, _detect
from _extract_real_boqs import extract_from_workbook
from engine.result_validator import _ALIAS
_SYS_RX = re.compile(r"qto.?hybrid|hybrid.?test|_qto_test|project_\d+_qto|_qto\.xls", re.I)
def find_boq(project_dir):
xls = [os.path.join(project_dir, f) for f in os.listdir(project_dir)
if f.lower().endswith((".xls", ".xlsx")) and not _SYS_RX.search(f)]
boq = next((x for x in xls if re.search(r"boq|bill|quantit|sheet", os.path.basename(x), re.I)), None)
return boq or (max(xls, key=os.path.getsize) if xls else None)
def engine_ref(qty_results):
out = defaultdict(float)
for ek, q in (qty_results or {}).items():
rk = _ALIAS.get(ek)
if rk and q:
out[rk] += float(q)
return dict(out)
def gt_ref(boq_path):
try:
return {k: v["qty"] for k, v in extract_from_workbook(boq_path).items()}
except Exception as e:
print(f" GT parse error: {e}")
return {}
# reference items the engine is supposed to compute (skip GT-only trades like MEP/marble)
ENGINE_KEYS = sorted(set(_ALIAS.values()))
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--root", default=r"B:\work\trainning")
ap.add_argument("--projects", nargs="+", default=["2", "3", "4", "5", "6"])
ap.add_argument("--out", default=os.path.join(_HERE, "batch_results.json"))
a = ap.parse_args()
report = {}
for pid in a.projects:
d = os.path.join(a.root, pid)
if not os.path.isdir(d):
print(f"[{pid}] missing"); continue
strp, arch = _detect(d)
boq = find_boq(d)
print(f"\n{'='*70}\n[{pid}] STR={os.path.basename(strp or '-')} ARCH={os.path.basename(arch or '-')} BOQ={os.path.basename(boq or '-')}\n{'='*70}")
try:
df, meta, c, qty = run(strp, arch, 2)
except Exception as e:
print(f" RUN ERROR: {e}")
report[pid] = {"error": str(e)}
continue
eng = engine_ref(qty)
gt = gt_ref(boq)
levels = len([k for k in (c.get("floors") or {}) if k in ("gf", "1f", "2f")])
nonzero = sum(1 for v in eng.values() if v > 0)
print(f" levels detected: {levels} | engine non-zero ref-items: {nonzero} | GT items: {len(gt)}")
print(f" {'ITEM':<22}{'ENGINE':>10}{'GT':>10}{'DEV%':>8}")
rows = {}
for k in ENGINE_KEYS:
e, g = eng.get(k, 0.0), gt.get(k)
if not g and not e:
continue
dev = (abs(e - g) / g * 100) if g else None
rows[k] = {"engine": round(e, 1), "gt": round(g, 1) if g else None, "dev": round(dev, 1) if dev is not None else None}
ds = f"{dev:6.1f}" if dev is not None else " - "
gs = f"{g:.1f}" if g else "-"
print(f" {k:<22}{e:>10.1f}{gs:>10}{ds:>8}")
report[pid] = {"levels": levels, "engine_nonzero": nonzero, "gt_items": len(gt),
"key_inputs": {k: c.get(k) for k in ("gf_area", "ext_perimeter", "plot_area", "longest_length", "longest_width") if c.get(k)},
"rows": rows}
with open(a.out, "w", encoding="utf-8") as f:
json.dump(report, f, indent=2, ensure_ascii=False, default=str)
# ── overall accuracy summary ──
print(f"\n{'#'*70}\nOVERALL ACCURACY (deviation of engine vs ground truth)\n{'#'*70}")
buckets = {"<=15%": 0, "<=30%": 0, ">30%": 0, "missing(engine=0,gt>0)": 0}
total = 0
for pid, r in report.items():
for k, v in r.get("rows", {}).items():
if v["gt"] is None:
continue
total += 1
if v["engine"] == 0:
buckets["missing(engine=0,gt>0)"] += 1
elif v["dev"] is not None and v["dev"] <= 15:
buckets["<=15%"] += 1
elif v["dev"] is not None and v["dev"] <= 30:
buckets["<=30%"] += 1
else:
buckets[">30%"] += 1
for k, n in buckets.items():
pct = (100 * n / total) if total else 0
print(f" {k:<28} {n:>3} ({pct:.0f}%)")
print(f" TOTAL comparable item-instances: {total}")
print(f"\nSaved -> {a.out}")
if __name__ == "__main__":
main()
|