qshub / engine /result_validator.py
basel00omar's picture
THE QS HUB v2.0 - Complete React + FastAPI Platform (clean deploy)
1b4b6d2
Raw History Blame Contribute Delete
11.6 kB
"""
Validates AI-calculated quantities against reference ranges
built from real UAE villa BOQ files.
"""
import json
import os
import statistics
_DB_PATH = os.path.join(os.path.dirname(__file__), "reference_ranges.json")
_PROJECT_JSON = os.path.join(os.path.dirname(__file__), "..", "_project_data.json")
def _load_db() -> dict:
try:
with open(_DB_PATH, encoding="utf-8") as f:
return json.load(f)
except Exception:
return {}
def reload_db() -> dict:
"""Force reload reference DB from disk (used after _build_reference_db.py --add)."""
global _DB
_DB = _load_db()
return _DB
_DB: dict = _load_db()
def auto_villa_type() -> str | None:
"""
Auto-detect villa_type from _project_data.json [W8].
Returns 'G', 'G+1', 'G+2', etc., or None if project data unavailable.
"""
try:
with open(_PROJECT_JSON, encoding="utf-8") as f:
project = json.load(f)
except Exception:
return None
levels = project.get("structural_levels")
if levels is None:
return None
if levels <= 2:
return "G"
return f"G+{levels - 2}"
# ── Key aliases: our item keys β†’ reference DB keys ────────────────────────────
_ALIAS: dict[str, str] = {
# ── Sub-structure (exact matches) ────────────────────────────────────────
"excavation": "excavation",
"foundation_pcc": "foundation_pcc",
"foundation_concrete": "foundation_concrete",
"neck_column_concrete": "neck_column_concrete",
"road_base": "road_base",
"backfill": "backfill",
# polyethylene_found alone = ~134 mΒ² (just foundation level)
# reference "polyethylene" = both levels combined (~450 mΒ²) β†’ use poly_sheet_combined
# "polyethylene_found": [skip individual β€” use poly_sheet_combined for combined check]
"slab_on_grade": "slab_on_grade",
# ── Tie beams ────────────────────────────────────────────────────────────
"tie_beam_concrete": "tie_beam_concrete",
# tie_beam_pcc alone is ~5 mΒ³; reference "foundation_pcc" = combined footing+TB PCC (22 mΒ³)
# Individual comparison gives misleading 🟑 β€” validate via pcc_combined instead
# "tie_beam_pcc": [skip individual β€” use pcc_combined]
# virtual key (foundation_pcc + tie_beam_pcc summed) β€” correct combined comparison
"pcc_combined": "foundation_pcc",
# ── Bitumen items β€” no unit-matched reference available ──────────────────
# (BOQ references are in mΒ³ concrete; bitumen is in mΒ²; do NOT compare)
# "foundation_bitumen": [no ref β€” mΒ² vs mΒ³ would be meaningless]
# "neck_column_bitumen": [no ref]
# "tie_beam_bitumen": [no ref]
# ── Poly sheet / anti-termite ────────────────────────────────────────────
# poly_sheet_tb (414 mΒ²) + polyethylene_found (134 mΒ²) = ~548 mΒ² combined
# reference "polyethylene" = combined both levels (median 450 mΒ²)
# poly_sheet_tb alone comparing to full reference is misleading β€” use combined instead
# "poly_sheet_tb": [skip individual β€” too high alone vs combined reference]
"poly_sheet_combined": "polyethylene", # virtual key for combined poly sheet
# ── Columns ──────────────────────────────────────────────────────────────
"col_gf_to_1st": "column_concrete",
"col_1st_to_2nd": "column_concrete",
"col_roof": "column_concrete",
# ── Beams & slabs (now backed by 11 real villa BOQs) ─────────────────────
"beam_concrete_1st": "beam_concrete",
"beam_concrete_2nd": "beam_concrete",
"beam_concrete_roof": "beam_concrete",
"slab_concrete_1st": "slab_concrete",
"slab_concrete_2nd": "slab_concrete",
"slab_concrete_roof": "slab_concrete",
"staircase_1st": "staircase_concrete",
# ── Parapet ──────────────────────────────────────────────────────────────
# parapet_block_work is in mΒ²; reference "parapet" is parapet CONCRETE in mΒ³ β€” unit mismatch
# "parapet_block_work": [no ref]
"parapet_concrete": "parapet",
# ── Block work ───────────────────────────────────────────────────────────
# solid_block_work = perimeter tie beam formwork only (~73 mΒ²)
# reference "solid_block" = all solid block throughout building (~208 mΒ² median)
# Comparing partial to total gives false 🟑 β€” skip
# "solid_block_work": [skip β€” perimeter-only vs full-building reference]
"internal_block_gf": "internal_block",
"internal_block_1f": "internal_block",
"internal_block_2f": "internal_block",
# thermal_block now n=3 β€” still small but better than nothing
"thermal_block_gf": "thermal_block",
"thermal_block_1f": "thermal_block",
# ── Finishes (now backed by 7-13 real villa BOQs) ────────────────────────
"skirting_gf": "skirting",
"internal_plaster": "internal_plaster",
"internal_plaster_gf": "internal_plaster",
"internal_plaster_1f": "internal_plaster",
"external_plaster": "external_plaster",
"ceiling_plaster": "ceiling_plaster",
"internal_paint": "internal_paint",
"external_paint": "external_paint",
# ── Waterproofing (now have references) ──────────────────────────────────
"foundation_bitumen": "foundation_bitumen",
# neck_column_bitumen / tie_beam_bitumen are small-area items (tens of mΒ²);
# comparing them to foundation_bitumen's range (hundreds of mΒ²) produced
# false πŸ”΄ outliers. No magnitude-matched reference β†’ leave unmapped.
# "neck_column_bitumen": [no comparable reference]
# "tie_beam_bitumen": [no comparable reference]
"wet_waterproofing": "wet_waterproofing",
"wet_wp_gf": "wet_waterproofing",
"wet_wp_1f": "wet_waterproofing",
"wet_wp_2f": "wet_waterproofing",
# ── External ─────────────────────────────────────────────────────────────
"interlock_paving": "interlock_paving",
"compound_wall": "compound_wall",
"roof_waterproofing": "polyethylene",
# ── Openings ─────────────────────────────────────────────────────────────
"door_openings": "door_openings",
"window_openings": "window_openings",
}
def validate_quantity(item_key: str, quantity: float,
villa_type: str | None = None) -> dict:
"""
Returns a validation dict:
status: 'ok' | 'low' | 'high' | 'outlier' | 'no_data'
label: human-readable label with emoji
ref: reference dict (min/max/median/count) or None
pct_dev: % deviation from median
Args:
item_key: The item key in our engine's output.
quantity: The computed quantity to validate.
villa_type: Optional UAE villa type (e.g. "G+1", "G+2"). When provided,
the validator picks the narrower by_villa_type band if it has
>= 2 samples for that type β€” giving more accurate comparisons
for non-typical villas.
"""
ref_key = _ALIAS.get(item_key)
if not ref_key or ref_key not in _DB:
return {"status": "no_data", "label": "βšͺ No reference", "ref": None, "pct_dev": None}
# Auto-detect villa type if caller didn't specify [W8]
if villa_type is None:
villa_type = auto_villa_type()
ref = _DB[ref_key]
# Prefer same-type band if available
if villa_type and ref.get("by_villa_type"):
sub = ref["by_villa_type"].get(villa_type)
if sub and sub.get("count", 0) >= 2:
ref = {
**ref,
"min": sub["min"],
"max": sub["max"],
"median": sub["median"],
"count": sub["count"],
"_filtered_by": villa_type,
}
vmin = ref["min"]
vmax = ref["max"]
med = ref["median"]
std = ref.get("stdev", 0)
pct_dev = round(abs(quantity - med) / med * 100, 1) if med > 0 else None
# Define "normal" band: median Β± 1.5 Γ— stdev, but at minimum within [min, max]
low_ok = max(vmin, med - 1.5 * std) if std > 0 else vmin
high_ok = min(vmax, med + 1.5 * std) if std > 0 else vmax
if quantity < vmin * 0.5:
status = "outlier"
label = f"πŸ”΄ Very low (ref min {vmin:.0f})"
elif quantity > vmax * 1.5:
status = "outlier"
label = f"πŸ”΄ Very high (ref max {vmax:.0f})"
elif quantity < low_ok:
status = "low"
label = f"🟑 Below typical (median {med:.0f})"
elif quantity > high_ok:
status = "high"
label = f"🟑 Above typical (median {med:.0f})"
else:
status = "ok"
label = f"🟒 Normal (median {med:.0f})"
return {
"status": status,
"label": label,
"ref": ref,
"pct_dev": pct_dev,
}
def validate_all_results(results: dict[str, float],
villa_type: str | None = None) -> dict[str, dict]:
"""
Validates a dict of {item_key: quantity}.
Returns {item_key: validation_dict}.
"""
return {
k: validate_quantity(k, v, villa_type=villa_type)
for k, v in results.items() if v is not None
}
def get_reference_summary() -> list[dict]:
"""Returns a flat list of all reference ranges for display."""
rows = []
for key, data in _DB.items():
if key.startswith("_") or not isinstance(data, dict) or "min" not in data:
continue # skip metadata blocks like _projects
rows.append({
"Item": key.replace("_", " ").title(),
"Projects": data["count"],
"Min": data["min"],
"Median": data["median"],
"Max": data["max"],
"Unit": data["unit"].upper(),
"Std Dev": data.get("stdev", 0),
})
return sorted(rows, key=lambda r: r["Item"])
def list_known_villa_types() -> list[str]:
"""Return the distinct villa_type values present in the DB metadata."""
projs = _DB.get("_projects", {})
types = set()
for _, meta in projs.items():
vt = meta.get("villa_type")
if vt and vt != "unknown":
types.add(vt)
return sorted(types)