qshub / _master_extractor.py
basel00omar's picture
THE QS HUB v2.0 - Complete React + FastAPI Platform (clean deploy)
1b4b6d2
Raw History Blame Contribute Delete
16.3 kB
"""
End-to-end villa-QTO extraction pipeline.
Inputs:
--str path/to/STR.pdf
--arch path/to/ARCH.pdf
--apikey <AI API key> (optional; falls back to GOOGLE_API_KEY env)
Pipeline:
1. Classify every page of both PDFs.
2. Extract STR schedules (footings, tie beams, columns, slab beams).
3. Extract ARCH base data (plot/areas/perimeters/heights).
4. Extract opening schedules (doors + windows).
5. Estimate wall lengths (AI; user can refine in Streamlit).
6. Run sanity checks; print warnings.
7. Build a single _project_data.json that the BOQ engine consumes.
Output:
_project_data.json β€” single source of truth, replaces hardcoded `base` dict.
Usage:
py _master_extractor.py --str "B:/path/STR.pdf" --arch "B:/path/ARCH.pdf"
"""
import sys, os, json, argparse, subprocess
from typing import Optional
sys.stdout.reconfigure(encoding='utf-8')
sys.path.insert(0, os.path.dirname(__file__))
from _pdf_utils import pdf_page_count
from _page_classifier_ai import classify_and_save, OUT_JSON as CLASS_JSON
from _read_arch_ai import extract_arch, OUT_JSON as ARCH_JSON
from _read_openings_ai import extract_openings, OUT_JSON as OPEN_JSON
from _wall_length_extractor import extract_walls_all_floors, OUT_JSON as WALL_JSON
from _qto_config import (
SCHEDULE_JSON, PROJECT_JSON, SANITY, derive_villa_type, get_logger,
)
log = get_logger("master_extractor")
# ── Sanity checks [W11] ───────────────────────────────────────────────────────
def _in_range(val, key: str) -> bool:
"""Check val against SANITY[key]=(lo,hi)."""
if val is None:
return True
lo, hi = SANITY[key]
return lo <= val <= hi
def sanity_check(data: dict) -> list[str]:
"""Return list of warnings; empty list = all OK."""
warns = []
# ── Base dimensions ───────────────────────────────────────────────────────
for k in ("plot_area", "gf_area", "external_perimeter", "total_villa_height"):
v = data.get(k)
if v is not None and not _in_range(v, k):
lo, hi = SANITY[k]
warns.append(f"{k}={v} outside typical range ({lo}-{hi})")
# ── Bbox vs gf_area cross-check ───────────────────────────────────────────
ll = data.get("longest_length")
lw = data.get("longest_width")
ga = data.get("gf_area")
if ll and lw and ga:
ratio = ga / (ll * lw)
if ratio < 0.5 or ratio > 1.05:
warns.append(
f"gf_area/bbox ratio = {ratio:.2f} (expected 0.5-1.05) β€” "
f"likely a mis-read of one of: gf_area, longest_length, longest_width"
)
# ── Foundations ──────────────────────────────────────────────────────────
footings = data.get("schedules", {}).get("foundation", {}).get("footings", [])
if not footings:
warns.append("No footings extracted β€” check foundation schedule")
else:
total_v = sum(
(f.get("long_mm", 0) or 0) / 1000
* (f.get("short_mm", 0) or 0) / 1000
* (f.get("depth_mm", 0) or 0) / 1000
* (f.get("count", 0) or 0)
for f in footings
)
if not _in_range(total_v, "foundation_total_m3"):
lo, hi = SANITY["foundation_total_m3"]
warns.append(f"foundation total volume {total_v:.1f} mΒ³ outside {lo}-{hi}")
for f in footings:
d = f.get("depth_mm")
l = f.get("long_mm")
if d and not _in_range(d, "footing_depth_mm"):
lo, hi = SANITY["footing_depth_mm"]
warns.append(f"Footing {f.get('type')}: depth={d}mm outside {lo}-{hi}")
if l and not _in_range(l, "footing_long_mm"):
lo, hi = SANITY["footing_long_mm"]
warns.append(f"Footing {f.get('type')}: long={l}mm outside {lo}-{hi}")
if not f.get("count"):
warns.append(f"Footing {f.get('type')}: count missing or zero")
# ── Tie beams ─────────────────────────────────────────────────────────────
tbs = data.get("schedules", {}).get("tie_beam", {}).get("tie_beams", [])
for t in tbs:
if t.get("width_mm") and not _in_range(t["width_mm"], "tie_beam_width_mm"):
lo, hi = SANITY["tie_beam_width_mm"]
warns.append(f"TB {t.get('type')}: width={t['width_mm']}mm outside {lo}-{hi}")
if t.get("depth_mm") and not _in_range(t["depth_mm"], "tie_beam_depth_mm"):
lo, hi = SANITY["tie_beam_depth_mm"]
warns.append(f"TB {t.get('type')}: depth={t['depth_mm']}mm outside {lo}-{hi}")
# ── Columns ───────────────────────────────────────────────────────────────
cols = data.get("schedules", {}).get("column_schedule", {}).get("columns", [])
for c in cols:
if c.get("width_mm") and not _in_range(c["width_mm"], "column_width_mm"):
lo, hi = SANITY["column_width_mm"]
warns.append(f"Column {c.get('mark')}: width={c['width_mm']}mm outside {lo}-{hi}")
if c.get("depth_mm") and not _in_range(c["depth_mm"], "column_depth_mm"):
lo, hi = SANITY["column_depth_mm"]
warns.append(f"Column {c.get('mark')}: depth={c['depth_mm']}mm outside {lo}-{hi}")
# ── Slab beams (all floors) ───────────────────────────────────────────────
for slab_key in ("slab_1f", "slab_2f", "slab_roof"):
slab = data.get("schedules", {}).get(slab_key, {})
st = slab.get("slab_thickness_mm")
if st and not _in_range(st, "slab_thickness_mm"):
lo, hi = SANITY["slab_thickness_mm"]
warns.append(f"{slab_key}: slab_thickness={st}mm outside {lo}-{hi}")
for b in slab.get("beams", []):
if b.get("width_mm") and not _in_range(b["width_mm"], "beam_width_mm"):
lo, hi = SANITY["beam_width_mm"]
warns.append(f"{slab_key} {b.get('mark')}: width={b['width_mm']}mm outside {lo}-{hi}")
if b.get("depth_mm") and not _in_range(b["depth_mm"], "beam_depth_mm"):
lo, hi = SANITY["beam_depth_mm"]
warns.append(f"{slab_key} {b.get('mark')}: depth={b['depth_mm']}mm outside {lo}-{hi}")
# ── Openings ──────────────────────────────────────────────────────────────
op_tot = data.get("openings", {}).get("totals", {})
if op_tot.get("door_count", 0) == 0:
warns.append("No doors extracted")
if op_tot.get("window_count", 0) == 0:
warns.append("No windows extracted")
# ── Walls vs floor areas ──────────────────────────────────────────────────
walls = data.get("walls", {})
floors = data.get("floors", {})
for fk, w in walls.items():
intl = w.get("internal_total_m")
fa = (floors.get(fk) or {}).get("area")
if intl and fa:
ratio = intl / fa
lo, hi = SANITY["wall_to_area_ratio"]
if not (lo <= ratio <= hi):
warns.append(
f"{fk}: internal_walls/area = {ratio:.2f} m/mΒ² outside {lo}-{hi}"
)
return warns
# ── Stage runners ─────────────────────────────────────────────────────────────
def run_str_schedules(str_pdf: str, api_key: str | None):
"""Run the (already-existing) STR schedule extractor as a subprocess."""
env = os.environ.copy()
env["STR_PDF_PATH"] = str_pdf
if api_key:
env["GOOGLE_API_KEY"] = api_key
cmd = [sys.executable,
os.path.join(os.path.dirname(__file__), "_read_schedules_ai.py")]
if api_key:
cmd.append(api_key)
print(f"\n β†’ invoking _read_schedules_ai.py")
subprocess.run(cmd, env=env, check=False)
# ── Main pipeline ─────────────────────────────────────────────────────────────
def run_pipeline(str_pdf: str, arch_pdf: str, api_key: str | None = None,
skip_schedule: bool = False, skip_walls: bool = False,
skip_classify: bool = False) -> dict:
print("=" * 78)
print(" THE QS HUB MASTER EXTRACTOR")
print("=" * 78)
print(f" STR : {str_pdf} ({pdf_page_count(str_pdf)} pages)")
print(f" ARCH : {arch_pdf} ({pdf_page_count(arch_pdf)} pages)")
# ── Step 1: classify pages ────────────────────────────────────────────────
print("\n── STEP 1: Page Classification ──────────────────────────────────")
if skip_classify and os.path.exists(CLASS_JSON):
# Use the classification the user already reviewed/corrected in the UI.
with open(CLASS_JSON, encoding="utf-8") as f:
classification = json.load(f)
print(" (using existing / user-confirmed page classification)")
else:
classification = classify_and_save([str_pdf, arch_pdf], api_key=api_key)
# ── Step 2: STR schedules ─────────────────────────────────────────────────
print("\n── STEP 2: STR Schedule Extraction ──────────────────────────────")
if not skip_schedule:
run_str_schedules(str_pdf, api_key)
schedule_data = {}
if os.path.exists(SCHEDULE_JSON):
with open(SCHEDULE_JSON, encoding='utf-8') as f:
schedule_data = json.load(f)
# ── Step 3: ARCH base data ────────────────────────────────────────────────
print("\n── STEP 3: ARCH Base Data ───────────────────────────────────────")
arch_data = extract_arch(arch_pdf, classification, api_key=api_key)
with open(ARCH_JSON, 'w', encoding='utf-8') as f:
json.dump(arch_data, f, indent=2, ensure_ascii=False)
# ── Step 4: Openings ──────────────────────────────────────────────────────
print("\n── STEP 4: Door & Window Schedules ──────────────────────────────")
open_data = extract_openings(arch_pdf, classification, api_key=api_key)
with open(OPEN_JSON, 'w', encoding='utf-8') as f:
json.dump(open_data, f, indent=2, ensure_ascii=False)
# ── Step 5: Walls ─────────────────────────────────────────────────────────
wall_data = {}
if not skip_walls:
print("\n── STEP 5: Wall Length Estimation ───────────────────────────")
# Pass floor areas so wall extractor can sanity-check its output [W7]
floor_areas = {
fk: f.get("area") for fk, f in arch_data.get("floors", {}).items()
if f.get("area")
}
wall_data = extract_walls_all_floors(
arch_pdf, classification, api_key=api_key, floor_areas=floor_areas
)
with open(WALL_JSON, 'w', encoding='utf-8') as f:
json.dump(wall_data, f, indent=2, ensure_ascii=False)
# ── Step 6: Build _project_data.json ──────────────────────────────────────
print("\n── STEP 6: Build _project_data.json ─────────────────────────────")
villa_type = derive_villa_type(arch_data.get("structural_levels"))
project = {
# core dimensions
"villa_type": villa_type,
"plot_area": arch_data.get("plot_area"),
"gf_area": arch_data.get("gf_area"),
"external_perimeter": arch_data.get("external_perimeter"),
"ext_perimeter": arch_data.get("external_perimeter"),
"roof_slab_area": arch_data.get("roof_slab_area"),
"roof_perimeter": arch_data.get("roof_perimeter"),
"longest_length": arch_data.get("longest_length"),
"longest_width": arch_data.get("longest_width"),
"total_villa_height": arch_data.get("total_villa_height"),
"structural_levels": arch_data.get("structural_levels"),
"compound_length": arch_data.get("compound_length"),
"parapet_height": arch_data.get("parapet_height") or 1.0,
# default substructure (overrideable in UI)
"excavation_depth": 1.25,
# per-floor block
"floors": arch_data.get("floors", {}),
# source schedules
"schedules": schedule_data,
# openings
"openings": open_data,
# walls
"walls": wall_data,
# metadata
"_meta": {
"str_pdf": str_pdf,
"arch_pdf": arch_pdf,
},
}
# Merge wall data into per-floor block for engine consumption
for fk, w in wall_data.items():
if fk in project["floors"]:
project["floors"][fk]["wall_internal"] = w.get("internal_total_m")
project["floors"][fk]["wall_external"] = w.get("external_total_m")
# ── Step 7: Sanity check ──────────────────────────────────────────────────
print("\n── STEP 7: Sanity Check ─────────────────────────────────────────")
warns = sanity_check(project)
project["_sanity_warnings"] = warns
if not warns:
print(" βœ“ All values within expected ranges.")
else:
print(f" {len(warns)} warnings:")
for w in warns:
print(f" - {w}")
with open(PROJECT_JSON, 'w', encoding='utf-8') as f:
json.dump(project, f, indent=2, ensure_ascii=False)
print(f"\n βœ“ Saved β†’ {PROJECT_JSON}")
return project
def cli():
ap = argparse.ArgumentParser(description="THE QS HUB master extractor")
ap.add_argument("--str", required=True, help="path to STR PDF")
ap.add_argument("--arch", required=True, help="path to ARCH PDF")
ap.add_argument("--apikey", help="AI API key (optional)")
ap.add_argument("--skip-schedule", action="store_true",
help="skip the STR schedule pass (use existing _schedule_data.json)")
ap.add_argument("--skip-walls", action="store_true",
help="skip the wall-length AI pass")
args = ap.parse_args()
run_pipeline(args.str, args.arch,
api_key=args.apikey,
skip_schedule=args.skip_schedule,
skip_walls=args.skip_walls)
if __name__ == "__main__":
cli()