Download _master_extractor.py from Basel0/qshub: direct link, hf CLI and curl.
- Browser
- Download file 16.3 kB
-
https://huggingface.co/spaces/Basel0/qshub/resolve/main/_master_extractor.py
- Command line
-
hf download hf://spaces/Basel0/qshub/_master_extractor.py
-
curl -L -o _master_extractor.py https://huggingface.co/spaces/Basel0/qshub/resolve/main/_master_extractor.py
16.3 kB
| """ | |
| End-to-end villa-QTO extraction pipeline. | |
| Inputs: | |
| --str path/to/STR.pdf | |
| --arch path/to/ARCH.pdf | |
| --apikey <AI API key> (optional; falls back to GOOGLE_API_KEY env) | |
| Pipeline: | |
| 1. Classify every page of both PDFs. | |
| 2. Extract STR schedules (footings, tie beams, columns, slab beams). | |
| 3. Extract ARCH base data (plot/areas/perimeters/heights). | |
| 4. Extract opening schedules (doors + windows). | |
| 5. Estimate wall lengths (AI; user can refine in Streamlit). | |
| 6. Run sanity checks; print warnings. | |
| 7. Build a single _project_data.json that the BOQ engine consumes. | |
| Output: | |
| _project_data.json β single source of truth, replaces hardcoded `base` dict. | |
| Usage: | |
| py _master_extractor.py --str "B:/path/STR.pdf" --arch "B:/path/ARCH.pdf" | |
| """ | |
| import sys, os, json, argparse, subprocess | |
| from typing import Optional | |
| sys.stdout.reconfigure(encoding='utf-8') | |
| sys.path.insert(0, os.path.dirname(__file__)) | |
| from _pdf_utils import pdf_page_count | |
| from _page_classifier_ai import classify_and_save, OUT_JSON as CLASS_JSON | |
| from _read_arch_ai import extract_arch, OUT_JSON as ARCH_JSON | |
| from _read_openings_ai import extract_openings, OUT_JSON as OPEN_JSON | |
| from _wall_length_extractor import extract_walls_all_floors, OUT_JSON as WALL_JSON | |
| from _qto_config import ( | |
| SCHEDULE_JSON, PROJECT_JSON, SANITY, derive_villa_type, get_logger, | |
| ) | |
| log = get_logger("master_extractor") | |
| # ββ Sanity checks [W11] βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def _in_range(val, key: str) -> bool: | |
| """Check val against SANITY[key]=(lo,hi).""" | |
| if val is None: | |
| return True | |
| lo, hi = SANITY[key] | |
| return lo <= val <= hi | |
| def sanity_check(data: dict) -> list[str]: | |
| """Return list of warnings; empty list = all OK.""" | |
| warns = [] | |
| # ββ Base dimensions βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| for k in ("plot_area", "gf_area", "external_perimeter", "total_villa_height"): | |
| v = data.get(k) | |
| if v is not None and not _in_range(v, k): | |
| lo, hi = SANITY[k] | |
| warns.append(f"{k}={v} outside typical range ({lo}-{hi})") | |
| # ββ Bbox vs gf_area cross-check βββββββββββββββββββββββββββββββββββββββββββ | |
| ll = data.get("longest_length") | |
| lw = data.get("longest_width") | |
| ga = data.get("gf_area") | |
| if ll and lw and ga: | |
| ratio = ga / (ll * lw) | |
| if ratio < 0.5 or ratio > 1.05: | |
| warns.append( | |
| f"gf_area/bbox ratio = {ratio:.2f} (expected 0.5-1.05) β " | |
| f"likely a mis-read of one of: gf_area, longest_length, longest_width" | |
| ) | |
| # ββ Foundations ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| footings = data.get("schedules", {}).get("foundation", {}).get("footings", []) | |
| if not footings: | |
| warns.append("No footings extracted β check foundation schedule") | |
| else: | |
| total_v = sum( | |
| (f.get("long_mm", 0) or 0) / 1000 | |
| * (f.get("short_mm", 0) or 0) / 1000 | |
| * (f.get("depth_mm", 0) or 0) / 1000 | |
| * (f.get("count", 0) or 0) | |
| for f in footings | |
| ) | |
| if not _in_range(total_v, "foundation_total_m3"): | |
| lo, hi = SANITY["foundation_total_m3"] | |
| warns.append(f"foundation total volume {total_v:.1f} mΒ³ outside {lo}-{hi}") | |
| for f in footings: | |
| d = f.get("depth_mm") | |
| l = f.get("long_mm") | |
| if d and not _in_range(d, "footing_depth_mm"): | |
| lo, hi = SANITY["footing_depth_mm"] | |
| warns.append(f"Footing {f.get('type')}: depth={d}mm outside {lo}-{hi}") | |
| if l and not _in_range(l, "footing_long_mm"): | |
| lo, hi = SANITY["footing_long_mm"] | |
| warns.append(f"Footing {f.get('type')}: long={l}mm outside {lo}-{hi}") | |
| if not f.get("count"): | |
| warns.append(f"Footing {f.get('type')}: count missing or zero") | |
| # ββ Tie beams βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| tbs = data.get("schedules", {}).get("tie_beam", {}).get("tie_beams", []) | |
| for t in tbs: | |
| if t.get("width_mm") and not _in_range(t["width_mm"], "tie_beam_width_mm"): | |
| lo, hi = SANITY["tie_beam_width_mm"] | |
| warns.append(f"TB {t.get('type')}: width={t['width_mm']}mm outside {lo}-{hi}") | |
| if t.get("depth_mm") and not _in_range(t["depth_mm"], "tie_beam_depth_mm"): | |
| lo, hi = SANITY["tie_beam_depth_mm"] | |
| warns.append(f"TB {t.get('type')}: depth={t['depth_mm']}mm outside {lo}-{hi}") | |
| # ββ Columns βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| cols = data.get("schedules", {}).get("column_schedule", {}).get("columns", []) | |
| for c in cols: | |
| if c.get("width_mm") and not _in_range(c["width_mm"], "column_width_mm"): | |
| lo, hi = SANITY["column_width_mm"] | |
| warns.append(f"Column {c.get('mark')}: width={c['width_mm']}mm outside {lo}-{hi}") | |
| if c.get("depth_mm") and not _in_range(c["depth_mm"], "column_depth_mm"): | |
| lo, hi = SANITY["column_depth_mm"] | |
| warns.append(f"Column {c.get('mark')}: depth={c['depth_mm']}mm outside {lo}-{hi}") | |
| # ββ Slab beams (all floors) βββββββββββββββββββββββββββββββββββββββββββββββ | |
| for slab_key in ("slab_1f", "slab_2f", "slab_roof"): | |
| slab = data.get("schedules", {}).get(slab_key, {}) | |
| st = slab.get("slab_thickness_mm") | |
| if st and not _in_range(st, "slab_thickness_mm"): | |
| lo, hi = SANITY["slab_thickness_mm"] | |
| warns.append(f"{slab_key}: slab_thickness={st}mm outside {lo}-{hi}") | |
| for b in slab.get("beams", []): | |
| if b.get("width_mm") and not _in_range(b["width_mm"], "beam_width_mm"): | |
| lo, hi = SANITY["beam_width_mm"] | |
| warns.append(f"{slab_key} {b.get('mark')}: width={b['width_mm']}mm outside {lo}-{hi}") | |
| if b.get("depth_mm") and not _in_range(b["depth_mm"], "beam_depth_mm"): | |
| lo, hi = SANITY["beam_depth_mm"] | |
| warns.append(f"{slab_key} {b.get('mark')}: depth={b['depth_mm']}mm outside {lo}-{hi}") | |
| # ββ Openings ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| op_tot = data.get("openings", {}).get("totals", {}) | |
| if op_tot.get("door_count", 0) == 0: | |
| warns.append("No doors extracted") | |
| if op_tot.get("window_count", 0) == 0: | |
| warns.append("No windows extracted") | |
| # ββ Walls vs floor areas ββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| walls = data.get("walls", {}) | |
| floors = data.get("floors", {}) | |
| for fk, w in walls.items(): | |
| intl = w.get("internal_total_m") | |
| fa = (floors.get(fk) or {}).get("area") | |
| if intl and fa: | |
| ratio = intl / fa | |
| lo, hi = SANITY["wall_to_area_ratio"] | |
| if not (lo <= ratio <= hi): | |
| warns.append( | |
| f"{fk}: internal_walls/area = {ratio:.2f} m/mΒ² outside {lo}-{hi}" | |
| ) | |
| return warns | |
| # ββ Stage runners βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def run_str_schedules(str_pdf: str, api_key: str | None): | |
| """Run the (already-existing) STR schedule extractor as a subprocess.""" | |
| env = os.environ.copy() | |
| env["STR_PDF_PATH"] = str_pdf | |
| if api_key: | |
| env["GOOGLE_API_KEY"] = api_key | |
| cmd = [sys.executable, | |
| os.path.join(os.path.dirname(__file__), "_read_schedules_ai.py")] | |
| if api_key: | |
| cmd.append(api_key) | |
| print(f"\n β invoking _read_schedules_ai.py") | |
| subprocess.run(cmd, env=env, check=False) | |
| # ββ Main pipeline βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def run_pipeline(str_pdf: str, arch_pdf: str, api_key: str | None = None, | |
| skip_schedule: bool = False, skip_walls: bool = False, | |
| skip_classify: bool = False) -> dict: | |
| print("=" * 78) | |
| print(" THE QS HUB MASTER EXTRACTOR") | |
| print("=" * 78) | |
| print(f" STR : {str_pdf} ({pdf_page_count(str_pdf)} pages)") | |
| print(f" ARCH : {arch_pdf} ({pdf_page_count(arch_pdf)} pages)") | |
| # ββ Step 1: classify pages ββββββββββββββββββββββββββββββββββββββββββββββββ | |
| print("\nββ STEP 1: Page Classification ββββββββββββββββββββββββββββββββββ") | |
| if skip_classify and os.path.exists(CLASS_JSON): | |
| # Use the classification the user already reviewed/corrected in the UI. | |
| with open(CLASS_JSON, encoding="utf-8") as f: | |
| classification = json.load(f) | |
| print(" (using existing / user-confirmed page classification)") | |
| else: | |
| classification = classify_and_save([str_pdf, arch_pdf], api_key=api_key) | |
| # ββ Step 2: STR schedules βββββββββββββββββββββββββββββββββββββββββββββββββ | |
| print("\nββ STEP 2: STR Schedule Extraction ββββββββββββββββββββββββββββββ") | |
| if not skip_schedule: | |
| run_str_schedules(str_pdf, api_key) | |
| schedule_data = {} | |
| if os.path.exists(SCHEDULE_JSON): | |
| with open(SCHEDULE_JSON, encoding='utf-8') as f: | |
| schedule_data = json.load(f) | |
| # ββ Step 3: ARCH base data ββββββββββββββββββββββββββββββββββββββββββββββββ | |
| print("\nββ STEP 3: ARCH Base Data βββββββββββββββββββββββββββββββββββββββ") | |
| arch_data = extract_arch(arch_pdf, classification, api_key=api_key) | |
| with open(ARCH_JSON, 'w', encoding='utf-8') as f: | |
| json.dump(arch_data, f, indent=2, ensure_ascii=False) | |
| # ββ Step 4: Openings ββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| print("\nββ STEP 4: Door & Window Schedules ββββββββββββββββββββββββββββββ") | |
| open_data = extract_openings(arch_pdf, classification, api_key=api_key) | |
| with open(OPEN_JSON, 'w', encoding='utf-8') as f: | |
| json.dump(open_data, f, indent=2, ensure_ascii=False) | |
| # ββ Step 5: Walls βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| wall_data = {} | |
| if not skip_walls: | |
| print("\nββ STEP 5: Wall Length Estimation βββββββββββββββββββββββββββ") | |
| # Pass floor areas so wall extractor can sanity-check its output [W7] | |
| floor_areas = { | |
| fk: f.get("area") for fk, f in arch_data.get("floors", {}).items() | |
| if f.get("area") | |
| } | |
| wall_data = extract_walls_all_floors( | |
| arch_pdf, classification, api_key=api_key, floor_areas=floor_areas | |
| ) | |
| with open(WALL_JSON, 'w', encoding='utf-8') as f: | |
| json.dump(wall_data, f, indent=2, ensure_ascii=False) | |
| # ββ Step 6: Build _project_data.json ββββββββββββββββββββββββββββββββββββββ | |
| print("\nββ STEP 6: Build _project_data.json βββββββββββββββββββββββββββββ") | |
| villa_type = derive_villa_type(arch_data.get("structural_levels")) | |
| project = { | |
| # core dimensions | |
| "villa_type": villa_type, | |
| "plot_area": arch_data.get("plot_area"), | |
| "gf_area": arch_data.get("gf_area"), | |
| "external_perimeter": arch_data.get("external_perimeter"), | |
| "ext_perimeter": arch_data.get("external_perimeter"), | |
| "roof_slab_area": arch_data.get("roof_slab_area"), | |
| "roof_perimeter": arch_data.get("roof_perimeter"), | |
| "longest_length": arch_data.get("longest_length"), | |
| "longest_width": arch_data.get("longest_width"), | |
| "total_villa_height": arch_data.get("total_villa_height"), | |
| "structural_levels": arch_data.get("structural_levels"), | |
| "compound_length": arch_data.get("compound_length"), | |
| "parapet_height": arch_data.get("parapet_height") or 1.0, | |
| # default substructure (overrideable in UI) | |
| "excavation_depth": 1.25, | |
| # per-floor block | |
| "floors": arch_data.get("floors", {}), | |
| # source schedules | |
| "schedules": schedule_data, | |
| # openings | |
| "openings": open_data, | |
| # walls | |
| "walls": wall_data, | |
| # metadata | |
| "_meta": { | |
| "str_pdf": str_pdf, | |
| "arch_pdf": arch_pdf, | |
| }, | |
| } | |
| # Merge wall data into per-floor block for engine consumption | |
| for fk, w in wall_data.items(): | |
| if fk in project["floors"]: | |
| project["floors"][fk]["wall_internal"] = w.get("internal_total_m") | |
| project["floors"][fk]["wall_external"] = w.get("external_total_m") | |
| # ββ Step 7: Sanity check ββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| print("\nββ STEP 7: Sanity Check βββββββββββββββββββββββββββββββββββββββββ") | |
| warns = sanity_check(project) | |
| project["_sanity_warnings"] = warns | |
| if not warns: | |
| print(" β All values within expected ranges.") | |
| else: | |
| print(f" {len(warns)} warnings:") | |
| for w in warns: | |
| print(f" - {w}") | |
| with open(PROJECT_JSON, 'w', encoding='utf-8') as f: | |
| json.dump(project, f, indent=2, ensure_ascii=False) | |
| print(f"\n β Saved β {PROJECT_JSON}") | |
| return project | |
| def cli(): | |
| ap = argparse.ArgumentParser(description="THE QS HUB master extractor") | |
| ap.add_argument("--str", required=True, help="path to STR PDF") | |
| ap.add_argument("--arch", required=True, help="path to ARCH PDF") | |
| ap.add_argument("--apikey", help="AI API key (optional)") | |
| ap.add_argument("--skip-schedule", action="store_true", | |
| help="skip the STR schedule pass (use existing _schedule_data.json)") | |
| ap.add_argument("--skip-walls", action="store_true", | |
| help="skip the wall-length AI pass") | |
| args = ap.parse_args() | |
| run_pipeline(args.str, args.arch, | |
| api_key=args.apikey, | |
| skip_schedule=args.skip_schedule, | |
| skip_walls=args.skip_walls) | |
| if __name__ == "__main__": | |
| cli() | |