File size: 8,698 Bytes
1b4b6d2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 | """
Run the REAL QTO pipeline end-to-end on one villa's two PDFs and print a BOQ
(Description + Quantity + Unit). Uses the SAME functions the API uses — AI-vision
classification & extraction (Gemini), OpenCV floor-plan CV, PyMuPDF/OCR text, the
healing logic, and the engine formulas. Nothing mocked, nothing modified.
Usage:
python validation_set/run_engine.py --project "B:\\work\\trainning\\1" --floors 2
python validation_set/run_engine.py --str STR.pdf --arch ARCH.pdf --floors 2 --out out.xlsx
"""
import argparse
import os
import re
import sys
import tempfile
import numpy as np
from PIL import Image
_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, _ROOT)
from dotenv import load_dotenv
load_dotenv(os.path.join(_ROOT, ".env"))
from pdf_engine.pdf_loader import load_pdf_pages, extract_page_text, page_to_pil
from pdf_engine.smart_classifier import PAGE_ITEMS_MAP
from workflow.step2_classify import _classify_single, _ai_classify
from workflow.step3_extract import extract_page
from engine.advanced_cv_extractor import process_floor_plan_image
from engine.project_boq_bridge import build_boq_dataframe_from_project
from api.workflow import reconstruct_project_inputs
FLOOR_TYPES = {"ground_floor_plan", "first_floor_plan", "second_floor_plan", "roof_floor_plan", "setting_out"}
def _save_pages(content, prefix, cache):
texts = extract_page_text(content)
pages = load_pdf_pages(content)
for idx, arr in enumerate(pages):
pil = page_to_pil(arr)
w, h = pil.size
if max(w, h) > 1600:
s = 1600 / max(w, h)
pil = pil.resize((int(w * s), int(h * s)))
pil.save(os.path.join(cache, f"{prefix}_page_{idx}.png"))
return texts
def _detect(project_dir):
files = [os.path.join(project_dir, f) for f in os.listdir(project_dir) if os.path.isfile(os.path.join(project_dir, f))]
pdfs = [f for f in files if f.lower().endswith(".pdf")]
strc = next((f for f in pdfs if re.search(r"st[rc]|struct", os.path.basename(f), re.I)), None)
arch = next((f for f in pdfs if f != strc and re.search(r"arch|^ar[-_ ]|^ar\d", os.path.basename(f), re.I)), None)
if len(pdfs) == 2:
if strc and not arch: arch = next((f for f in pdfs if f != strc), None)
elif arch and not strc: strc = next((f for f in pdfs if f != arch), None)
return strc, arch
def run(str_pdf, arch_pdf, floors):
cache = tempfile.mkdtemp(prefix="qto_run_")
str_texts = _save_pages(open(str_pdf, "rb").read(), "str", cache) if str_pdf else []
arch_texts = _save_pages(open(arch_pdf, "rb").read(), "arch", cache) if arch_pdf else []
print(f"Loaded pages: structural={len(str_texts)} architectural={len(arch_texts)}")
classified = []
for i, t in enumerate(str_texts):
classified.append({"pdf": "structural", "page_index": i, "page_num": i + 1, **_classify_single(t, "structural")})
for i, t in enumerate(arch_texts):
classified.append({"pdf": "architectural", "page_index": i, "page_num": i + 1, **_classify_single(t, "architectural")})
from concurrent.futures import ThreadPoolExecutor
def cls(p):
# AI-verify EVERY page (matches the API + Streamlit flow)
prefix = "str" if p["pdf"] == "structural" else "arch"
ip = os.path.join(cache, f"{prefix}_page_{p['page_index']}.png")
if os.path.exists(ip):
try:
arr = np.array(Image.open(ip))[:, :, ::-1].copy()
g = _ai_classify(arr, p["pdf"])
if g:
p["detected_type"] = g
p["confidence"] = "high(ai)"
p["items"] = PAGE_ITEMS_MAP.get(g, {}).get("extract_items", [])
except Exception as e:
print(f" [cls err p{p['page_num']}] {e}")
return p
with ThreadPoolExecutor(max_workers=6) as ex:
classified = list(ex.map(cls, classified))
ready = [p for p in classified if p.get("detected_type") and p["detected_type"] != "unknown"]
print(f"\nPages identified ({len(ready)}):")
for p in ready:
print(f" {p['pdf'][:4]:4} p{p['page_num']:<2} -> {p['detected_type']}")
results = {}
def ext(p):
dt = p["detected_type"]
prefix = "str" if p["pdf"] == "structural" else "arch"
ip = os.path.join(cache, f"{prefix}_page_{p['page_index']}.png")
if not os.path.exists(ip):
return None
texts = str_texts if p["pdf"] == "structural" else arch_texts
ptext = texts[p["page_index"]] if p["page_index"] < len(texts) else ""
try:
arr = np.array(Image.open(ip))[:, :, ::-1].copy()
e = extract_page(arr, dt, ptext, 1)
if dt in FLOOR_TYPES:
e.update(process_floor_plan_image(ip))
return f"{p['pdf']}_p{p['page_num']}_{dt}", {**e, **p}
except Exception as ee:
return f"{p['pdf']}_p{p['page_num']}_{dt}", {"_ok": False, "_error": str(ee), "drawing_type": dt, **p}
print("\nExtracting (AI vision + CV)...")
with ThreadPoolExecutor(max_workers=6) as ex:
for it in ex.map(ext, ready):
if it:
results[it[0]] = it[1]
state = {"extraction_results": results, "confirmed_auto_data": {}}
reconstruct_project_inputs(state)
c = state["confirmed_auto_data"]
payload = {
"longest_length": c.get("longest_length"), "longest_width": c.get("longest_width"),
"plot_area": c.get("plot_area"), "gf_area": c.get("gf_area"),
"external_perimeter": c.get("external_perimeter"), "ext_perimeter": c.get("ext_perimeter"),
"roof_perimeter": c.get("roof_perimeter"), "roof_slab_area": c.get("roof_slab_area"),
"compound_length": c.get("compound_length"), "total_villa_height": c.get("total_villa_height"),
"parapet_height": c.get("parapet_height", 1.0),
"excavation_depth": c.get("excavation_depth") or 1.25,
"neck_column_height": c.get("neck_column_height") or 1.0,
"solid_block_height": c.get("solid_block_height") or 1.0,
"staircase_volume_per_level": c.get("staircase_volume_per_level") or 5.2,
# auto-detect structural levels from the floor plans the AI actually found
# (works for G, G+1, G+2 ...) — fall back to the --floors hint
"structural_levels": (len([k for k in (c.get("floors") or {}) if k in ("gf", "1f", "2f")]) or floors),
"road_base_included": True,
"gf_height": 4.0, "f1_height": 4.0, "f2_height": 4.0,
"schedules": c.get("schedules") or {}, "floors": c.get("floors") or {},
"walls": c.get("walls") or {}, "openings": c.get("openings") or {},
}
from engine.project_boq_bridge import compute_all_quantities
qty_results, _ = compute_all_quantities(payload)
df, meta = build_boq_dataframe_from_project(payload)
return df, meta, c, qty_results
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--str", dest="strp")
ap.add_argument("--arch")
ap.add_argument("--project")
ap.add_argument("--floors", type=int, default=2)
ap.add_argument("--out")
a = ap.parse_args()
strp, arch = a.strp, a.arch
if a.project:
strp, arch = _detect(a.project)
print(f"STR : {strp}\nARCH: {arch}\n")
df, meta, confirmed, _qty = run(strp, arch, a.floors)
print("\n" + "=" * 64)
print("BOQ OUTPUT (Description | Unit | Quantity)")
print("=" * 64)
nonzero = 0
for _, r in df.iterrows():
if r.get("_is_header"):
print(f"\n# {r['Description (English)']}")
continue
q = r.get("Quantity") or 0
flag = "" if (isinstance(q, (int, float)) and q > 0) else " (0)"
if isinstance(q, (int, float)) and q > 0:
nonzero += 1
print(f" {str(r['Description (English)'])[:42]:<42} {str(r.get('Unit') or ''):<4} {q}{flag}")
items = df[df["_is_header"] == False] if "_is_header" in df.columns else df
print("\n" + "=" * 64)
print(f"Items total: {len(items)} | with a non-zero quantity: {nonzero}")
if meta.get("estimates"):
print(f"Estimated (flagged): {len(meta['estimates'])}")
if meta.get("needs_input"):
print(f"Needs input: {meta['needs_input']}")
print("Key extracted inputs:",
{k: confirmed.get(k) for k in ("gf_area", "ext_perimeter", "plot_area", "longest_length", "longest_width") if confirmed.get(k)})
if a.out:
from utils.exporter import export_boq_to_excel
with open(a.out, "wb") as f:
f.write(export_boq_to_excel(df, "Villa"))
print(f"\nSaved BOQ -> {a.out}")
if __name__ == "__main__":
main()
|