File size: 11,554 Bytes
1b4b6d2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
"""
Validates AI-calculated quantities against reference ranges
built from real UAE villa BOQ files.
"""
import json
import os
import statistics

_DB_PATH      = os.path.join(os.path.dirname(__file__), "reference_ranges.json")
_PROJECT_JSON = os.path.join(os.path.dirname(__file__), "..", "_project_data.json")


def _load_db() -> dict:
    try:
        with open(_DB_PATH, encoding="utf-8") as f:
            return json.load(f)
    except Exception:
        return {}


def reload_db() -> dict:
    """Force reload reference DB from disk (used after _build_reference_db.py --add)."""
    global _DB
    _DB = _load_db()
    return _DB


_DB: dict = _load_db()


def auto_villa_type() -> str | None:
    """
    Auto-detect villa_type from _project_data.json [W8].
    Returns 'G', 'G+1', 'G+2', etc., or None if project data unavailable.
    """
    try:
        with open(_PROJECT_JSON, encoding="utf-8") as f:
            project = json.load(f)
    except Exception:
        return None
    levels = project.get("structural_levels")
    if levels is None:
        return None
    if levels <= 2:
        return "G"
    return f"G+{levels - 2}"


# ── Key aliases: our item keys β†’ reference DB keys ────────────────────────────
_ALIAS: dict[str, str] = {
    # ── Sub-structure (exact matches) ────────────────────────────────────────
    "excavation":            "excavation",
    "foundation_pcc":        "foundation_pcc",
    "foundation_concrete":   "foundation_concrete",
    "neck_column_concrete":  "neck_column_concrete",
    "road_base":             "road_base",
    "backfill":              "backfill",
    # polyethylene_found alone = ~134 mΒ² (just foundation level)
    # reference "polyethylene" = both levels combined (~450 mΒ²) β†’ use poly_sheet_combined
    # "polyethylene_found":  [skip individual β€” use poly_sheet_combined for combined check]
    "slab_on_grade":         "slab_on_grade",

    # ── Tie beams ────────────────────────────────────────────────────────────
    "tie_beam_concrete":     "tie_beam_concrete",
    # tie_beam_pcc alone is ~5 mΒ³; reference "foundation_pcc" = combined footing+TB PCC (22 mΒ³)
    # Individual comparison gives misleading 🟑 β€” validate via pcc_combined instead
    # "tie_beam_pcc":        [skip individual β€” use pcc_combined]
    # virtual key (foundation_pcc + tie_beam_pcc summed) β€” correct combined comparison
    "pcc_combined":          "foundation_pcc",

    # ── Bitumen items β€” no unit-matched reference available ──────────────────
    # (BOQ references are in mΒ³ concrete; bitumen is in mΒ²; do NOT compare)
    # "foundation_bitumen":  [no ref β€” mΒ² vs mΒ³ would be meaningless]
    # "neck_column_bitumen": [no ref]
    # "tie_beam_bitumen":    [no ref]

    # ── Poly sheet / anti-termite ────────────────────────────────────────────
    # poly_sheet_tb (414 mΒ²) + polyethylene_found (134 mΒ²) = ~548 mΒ² combined
    # reference "polyethylene" = combined both levels (median 450 mΒ²)
    # poly_sheet_tb alone comparing to full reference is misleading β€” use combined instead
    # "poly_sheet_tb":       [skip individual β€” too high alone vs combined reference]
    "poly_sheet_combined":   "polyethylene",   # virtual key for combined poly sheet

    # ── Columns ──────────────────────────────────────────────────────────────
    "col_gf_to_1st":         "column_concrete",
    "col_1st_to_2nd":        "column_concrete",
    "col_roof":              "column_concrete",

    # ── Beams & slabs (now backed by 11 real villa BOQs) ─────────────────────
    "beam_concrete_1st":     "beam_concrete",
    "beam_concrete_2nd":     "beam_concrete",
    "beam_concrete_roof":    "beam_concrete",
    "slab_concrete_1st":     "slab_concrete",
    "slab_concrete_2nd":     "slab_concrete",
    "slab_concrete_roof":    "slab_concrete",
    "staircase_1st":         "staircase_concrete",

    # ── Parapet ──────────────────────────────────────────────────────────────
    # parapet_block_work is in mΒ²; reference "parapet" is parapet CONCRETE in mΒ³ β€” unit mismatch
    # "parapet_block_work": [no ref]
    "parapet_concrete":      "parapet",

    # ── Block work ───────────────────────────────────────────────────────────
    # solid_block_work = perimeter tie beam formwork only (~73 mΒ²)
    # reference "solid_block" = all solid block throughout building (~208 mΒ² median)
    # Comparing partial to total gives false 🟑 β€” skip
    # "solid_block_work":    [skip β€” perimeter-only vs full-building reference]
    "internal_block_gf":     "internal_block",
    "internal_block_1f":     "internal_block",
    "internal_block_2f":     "internal_block",
    # thermal_block now n=3 β€” still small but better than nothing
    "thermal_block_gf":      "thermal_block",
    "thermal_block_1f":      "thermal_block",

    # ── Finishes (now backed by 7-13 real villa BOQs) ────────────────────────
    "skirting_gf":           "skirting",
    "internal_plaster":      "internal_plaster",
    "internal_plaster_gf":   "internal_plaster",
    "internal_plaster_1f":   "internal_plaster",
    "external_plaster":      "external_plaster",
    "ceiling_plaster":       "ceiling_plaster",
    "internal_paint":        "internal_paint",
    "external_paint":        "external_paint",

    # ── Waterproofing (now have references) ──────────────────────────────────
    "foundation_bitumen":    "foundation_bitumen",
    # neck_column_bitumen / tie_beam_bitumen are small-area items (tens of mΒ²);
    # comparing them to foundation_bitumen's range (hundreds of mΒ²) produced
    # false πŸ”΄ outliers. No magnitude-matched reference β†’ leave unmapped.
    # "neck_column_bitumen": [no comparable reference]
    # "tie_beam_bitumen":    [no comparable reference]
    "wet_waterproofing":     "wet_waterproofing",
    "wet_wp_gf":             "wet_waterproofing",
    "wet_wp_1f":             "wet_waterproofing",
    "wet_wp_2f":             "wet_waterproofing",

    # ── External ─────────────────────────────────────────────────────────────
    "interlock_paving":      "interlock_paving",
    "compound_wall":         "compound_wall",
    "roof_waterproofing":    "polyethylene",

    # ── Openings ─────────────────────────────────────────────────────────────
    "door_openings":         "door_openings",
    "window_openings":       "window_openings",
}


def validate_quantity(item_key: str, quantity: float,
                      villa_type: str | None = None) -> dict:
    """
    Returns a validation dict:
      status:  'ok' | 'low' | 'high' | 'outlier' | 'no_data'
      label:   human-readable label with emoji
      ref:     reference dict (min/max/median/count) or None
      pct_dev: % deviation from median

    Args:
        item_key:   The item key in our engine's output.
        quantity:   The computed quantity to validate.
        villa_type: Optional UAE villa type (e.g. "G+1", "G+2"). When provided,
                    the validator picks the narrower by_villa_type band if it has
                    >= 2 samples for that type β€” giving more accurate comparisons
                    for non-typical villas.
    """
    ref_key = _ALIAS.get(item_key)
    if not ref_key or ref_key not in _DB:
        return {"status": "no_data", "label": "βšͺ No reference", "ref": None, "pct_dev": None}

    # Auto-detect villa type if caller didn't specify [W8]
    if villa_type is None:
        villa_type = auto_villa_type()

    ref  = _DB[ref_key]
    # Prefer same-type band if available
    if villa_type and ref.get("by_villa_type"):
        sub = ref["by_villa_type"].get(villa_type)
        if sub and sub.get("count", 0) >= 2:
            ref = {
                **ref,
                "min":    sub["min"],
                "max":    sub["max"],
                "median": sub["median"],
                "count":  sub["count"],
                "_filtered_by": villa_type,
            }
    vmin = ref["min"]
    vmax = ref["max"]
    med  = ref["median"]
    std  = ref.get("stdev", 0)

    pct_dev = round(abs(quantity - med) / med * 100, 1) if med > 0 else None

    # Define "normal" band: median Β± 1.5 Γ— stdev, but at minimum within [min, max]
    low_ok  = max(vmin, med - 1.5 * std) if std > 0 else vmin
    high_ok = min(vmax, med + 1.5 * std) if std > 0 else vmax

    if quantity < vmin * 0.5:
        status = "outlier"
        label  = f"πŸ”΄ Very low (ref min {vmin:.0f})"
    elif quantity > vmax * 1.5:
        status = "outlier"
        label  = f"πŸ”΄ Very high (ref max {vmax:.0f})"
    elif quantity < low_ok:
        status = "low"
        label  = f"🟑 Below typical (median {med:.0f})"
    elif quantity > high_ok:
        status = "high"
        label  = f"🟑 Above typical (median {med:.0f})"
    else:
        status = "ok"
        label  = f"🟒 Normal (median {med:.0f})"

    return {
        "status":  status,
        "label":   label,
        "ref":     ref,
        "pct_dev": pct_dev,
    }


def validate_all_results(results: dict[str, float],
                          villa_type: str | None = None) -> dict[str, dict]:
    """
    Validates a dict of {item_key: quantity}.
    Returns {item_key: validation_dict}.
    """
    return {
        k: validate_quantity(k, v, villa_type=villa_type)
        for k, v in results.items() if v is not None
    }


def get_reference_summary() -> list[dict]:
    """Returns a flat list of all reference ranges for display."""
    rows = []
    for key, data in _DB.items():
        if key.startswith("_") or not isinstance(data, dict) or "min" not in data:
            continue   # skip metadata blocks like _projects
        rows.append({
            "Item":        key.replace("_", " ").title(),
            "Projects":    data["count"],
            "Min":         data["min"],
            "Median":      data["median"],
            "Max":         data["max"],
            "Unit":        data["unit"].upper(),
            "Std Dev":     data.get("stdev", 0),
        })
    return sorted(rows, key=lambda r: r["Item"])


def list_known_villa_types() -> list[str]:
    """Return the distinct villa_type values present in the DB metadata."""
    projs = _DB.get("_projects", {})
    types = set()
    for _, meta in projs.items():
        vt = meta.get("villa_type")
        if vt and vt != "unknown":
            types.add(vt)
    return sorted(types)