File size: 11,554 Bytes
1b4b6d2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 | """
Validates AI-calculated quantities against reference ranges
built from real UAE villa BOQ files.
"""
import json
import os
import statistics
_DB_PATH = os.path.join(os.path.dirname(__file__), "reference_ranges.json")
_PROJECT_JSON = os.path.join(os.path.dirname(__file__), "..", "_project_data.json")
def _load_db() -> dict:
try:
with open(_DB_PATH, encoding="utf-8") as f:
return json.load(f)
except Exception:
return {}
def reload_db() -> dict:
"""Force reload reference DB from disk (used after _build_reference_db.py --add)."""
global _DB
_DB = _load_db()
return _DB
_DB: dict = _load_db()
def auto_villa_type() -> str | None:
"""
Auto-detect villa_type from _project_data.json [W8].
Returns 'G', 'G+1', 'G+2', etc., or None if project data unavailable.
"""
try:
with open(_PROJECT_JSON, encoding="utf-8") as f:
project = json.load(f)
except Exception:
return None
levels = project.get("structural_levels")
if levels is None:
return None
if levels <= 2:
return "G"
return f"G+{levels - 2}"
# ββ Key aliases: our item keys β reference DB keys ββββββββββββββββββββββββββββ
_ALIAS: dict[str, str] = {
# ββ Sub-structure (exact matches) ββββββββββββββββββββββββββββββββββββββββ
"excavation": "excavation",
"foundation_pcc": "foundation_pcc",
"foundation_concrete": "foundation_concrete",
"neck_column_concrete": "neck_column_concrete",
"road_base": "road_base",
"backfill": "backfill",
# polyethylene_found alone = ~134 mΒ² (just foundation level)
# reference "polyethylene" = both levels combined (~450 mΒ²) β use poly_sheet_combined
# "polyethylene_found": [skip individual β use poly_sheet_combined for combined check]
"slab_on_grade": "slab_on_grade",
# ββ Tie beams ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
"tie_beam_concrete": "tie_beam_concrete",
# tie_beam_pcc alone is ~5 mΒ³; reference "foundation_pcc" = combined footing+TB PCC (22 mΒ³)
# Individual comparison gives misleading π‘ β validate via pcc_combined instead
# "tie_beam_pcc": [skip individual β use pcc_combined]
# virtual key (foundation_pcc + tie_beam_pcc summed) β correct combined comparison
"pcc_combined": "foundation_pcc",
# ββ Bitumen items β no unit-matched reference available ββββββββββββββββββ
# (BOQ references are in mΒ³ concrete; bitumen is in mΒ²; do NOT compare)
# "foundation_bitumen": [no ref β mΒ² vs mΒ³ would be meaningless]
# "neck_column_bitumen": [no ref]
# "tie_beam_bitumen": [no ref]
# ββ Poly sheet / anti-termite ββββββββββββββββββββββββββββββββββββββββββββ
# poly_sheet_tb (414 mΒ²) + polyethylene_found (134 mΒ²) = ~548 mΒ² combined
# reference "polyethylene" = combined both levels (median 450 mΒ²)
# poly_sheet_tb alone comparing to full reference is misleading β use combined instead
# "poly_sheet_tb": [skip individual β too high alone vs combined reference]
"poly_sheet_combined": "polyethylene", # virtual key for combined poly sheet
# ββ Columns ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
"col_gf_to_1st": "column_concrete",
"col_1st_to_2nd": "column_concrete",
"col_roof": "column_concrete",
# ββ Beams & slabs (now backed by 11 real villa BOQs) βββββββββββββββββββββ
"beam_concrete_1st": "beam_concrete",
"beam_concrete_2nd": "beam_concrete",
"beam_concrete_roof": "beam_concrete",
"slab_concrete_1st": "slab_concrete",
"slab_concrete_2nd": "slab_concrete",
"slab_concrete_roof": "slab_concrete",
"staircase_1st": "staircase_concrete",
# ββ Parapet ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
# parapet_block_work is in mΒ²; reference "parapet" is parapet CONCRETE in mΒ³ β unit mismatch
# "parapet_block_work": [no ref]
"parapet_concrete": "parapet",
# ββ Block work βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
# solid_block_work = perimeter tie beam formwork only (~73 mΒ²)
# reference "solid_block" = all solid block throughout building (~208 mΒ² median)
# Comparing partial to total gives false π‘ β skip
# "solid_block_work": [skip β perimeter-only vs full-building reference]
"internal_block_gf": "internal_block",
"internal_block_1f": "internal_block",
"internal_block_2f": "internal_block",
# thermal_block now n=3 β still small but better than nothing
"thermal_block_gf": "thermal_block",
"thermal_block_1f": "thermal_block",
# ββ Finishes (now backed by 7-13 real villa BOQs) ββββββββββββββββββββββββ
"skirting_gf": "skirting",
"internal_plaster": "internal_plaster",
"internal_plaster_gf": "internal_plaster",
"internal_plaster_1f": "internal_plaster",
"external_plaster": "external_plaster",
"ceiling_plaster": "ceiling_plaster",
"internal_paint": "internal_paint",
"external_paint": "external_paint",
# ββ Waterproofing (now have references) ββββββββββββββββββββββββββββββββββ
"foundation_bitumen": "foundation_bitumen",
# neck_column_bitumen / tie_beam_bitumen are small-area items (tens of mΒ²);
# comparing them to foundation_bitumen's range (hundreds of mΒ²) produced
# false π΄ outliers. No magnitude-matched reference β leave unmapped.
# "neck_column_bitumen": [no comparable reference]
# "tie_beam_bitumen": [no comparable reference]
"wet_waterproofing": "wet_waterproofing",
"wet_wp_gf": "wet_waterproofing",
"wet_wp_1f": "wet_waterproofing",
"wet_wp_2f": "wet_waterproofing",
# ββ External βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
"interlock_paving": "interlock_paving",
"compound_wall": "compound_wall",
"roof_waterproofing": "polyethylene",
# ββ Openings βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
"door_openings": "door_openings",
"window_openings": "window_openings",
}
def validate_quantity(item_key: str, quantity: float,
villa_type: str | None = None) -> dict:
"""
Returns a validation dict:
status: 'ok' | 'low' | 'high' | 'outlier' | 'no_data'
label: human-readable label with emoji
ref: reference dict (min/max/median/count) or None
pct_dev: % deviation from median
Args:
item_key: The item key in our engine's output.
quantity: The computed quantity to validate.
villa_type: Optional UAE villa type (e.g. "G+1", "G+2"). When provided,
the validator picks the narrower by_villa_type band if it has
>= 2 samples for that type β giving more accurate comparisons
for non-typical villas.
"""
ref_key = _ALIAS.get(item_key)
if not ref_key or ref_key not in _DB:
return {"status": "no_data", "label": "βͺ No reference", "ref": None, "pct_dev": None}
# Auto-detect villa type if caller didn't specify [W8]
if villa_type is None:
villa_type = auto_villa_type()
ref = _DB[ref_key]
# Prefer same-type band if available
if villa_type and ref.get("by_villa_type"):
sub = ref["by_villa_type"].get(villa_type)
if sub and sub.get("count", 0) >= 2:
ref = {
**ref,
"min": sub["min"],
"max": sub["max"],
"median": sub["median"],
"count": sub["count"],
"_filtered_by": villa_type,
}
vmin = ref["min"]
vmax = ref["max"]
med = ref["median"]
std = ref.get("stdev", 0)
pct_dev = round(abs(quantity - med) / med * 100, 1) if med > 0 else None
# Define "normal" band: median Β± 1.5 Γ stdev, but at minimum within [min, max]
low_ok = max(vmin, med - 1.5 * std) if std > 0 else vmin
high_ok = min(vmax, med + 1.5 * std) if std > 0 else vmax
if quantity < vmin * 0.5:
status = "outlier"
label = f"π΄ Very low (ref min {vmin:.0f})"
elif quantity > vmax * 1.5:
status = "outlier"
label = f"π΄ Very high (ref max {vmax:.0f})"
elif quantity < low_ok:
status = "low"
label = f"π‘ Below typical (median {med:.0f})"
elif quantity > high_ok:
status = "high"
label = f"π‘ Above typical (median {med:.0f})"
else:
status = "ok"
label = f"π’ Normal (median {med:.0f})"
return {
"status": status,
"label": label,
"ref": ref,
"pct_dev": pct_dev,
}
def validate_all_results(results: dict[str, float],
villa_type: str | None = None) -> dict[str, dict]:
"""
Validates a dict of {item_key: quantity}.
Returns {item_key: validation_dict}.
"""
return {
k: validate_quantity(k, v, villa_type=villa_type)
for k, v in results.items() if v is not None
}
def get_reference_summary() -> list[dict]:
"""Returns a flat list of all reference ranges for display."""
rows = []
for key, data in _DB.items():
if key.startswith("_") or not isinstance(data, dict) or "min" not in data:
continue # skip metadata blocks like _projects
rows.append({
"Item": key.replace("_", " ").title(),
"Projects": data["count"],
"Min": data["min"],
"Median": data["median"],
"Max": data["max"],
"Unit": data["unit"].upper(),
"Std Dev": data.get("stdev", 0),
})
return sorted(rows, key=lambda r: r["Item"])
def list_known_villa_types() -> list[str]:
"""Return the distinct villa_type values present in the DB metadata."""
projs = _DB.get("_projects", {})
types = set()
for _, meta in projs.items():
vt = meta.get("villa_type")
if vt and vt != "unknown":
types.add(vt)
return sorted(types)
|