Update app.py

app.py

@@ -48,12 +48,19 @@ from huggingface_hub import hf_hub_download
 # alongside this file in the same Space for this import to work.
 from structural_patterns import scan_structural_patterns, structural_flags_to_dict
 
+# NEW: Phase 5 reconciliation layer — upload reconciliation.py alongside
+# this file too. Only scan_hard_negatives is used here; the structural
+# override itself is handled inline below because it needs to update
+# `pred` before Phase 2/3 enrichment run later in this same function.
+from reconciliation import scan_hard_negatives
+
 # ── App ────────────────────────────────────────────────────────
 app = FastAPI(
     title       = "MACI — Muslim AI Content Intelligence",
     description = "Shariah compliance classifier v5.5 — institutional grade. "
                   "Full Terry v0.1 Signal Packet + Phase 2 Maqasid enrichment "
-                  "+ Phase 3 jurisdiction overlay + Phase 4 structural patterns.",
+                  "+ Phase 3 jurisdiction overlay + Phase 4 structural patterns "
+                  "+ Phase 5 reconciliation.",
     version     = "5.5",
 )
 app.add_middleware(CORSMiddleware, allow_origins=["*"],
@@ -134,6 +141,23 @@ SEV = {
     (7,"HIGH"):"HIGH",    (7,"MEDIUM"):"MEDIUM",   (7,"LOW"):"LOW",
 }
 
+# NEW (Phase 5): maps each structural pattern to the closest existing
+# `pred` class id. This is what lets a structural override also correct
+# maqasid_enrichment (Phase 2) and jurisdiction_overlay (Phase 3) below —
+# both of those blocks key off `pred`, and both run AFTER this mapping is
+# applied, so getting this mapping right means the whole rest of the
+# packet (pillar, AAOIFI standard, jurisdiction exposure/penalty) becomes
+# consistent with the corrected verdict automatically, not just the
+# top-line violation_class field.
+STRUCTURAL_PATTERN_TO_PRED = {
+    "SP-01": 1,  # Bay' al-Inah buy-back           -> Riba (disguised loan)
+    "SP-02": 1,  # Disguised Riba late-payment-income -> Riba
+    "SP-03": 2,  # Unilateral price variation       -> Gharar
+    "SP-04": 2,  # Risk/ownership decoupling        -> Gharar
+    "SP-05": 2,  # Total liability waiver           -> Gharar (closest of the 8 classes)
+}
+STRUCTURAL_SEVERITY_RANK = {"LOW":1, "MEDIUM":2, "HIGH":3, "CRITICAL":4}
+
 # ── Phase 2: Maqasid + consequence ────────────────────────────
 CONSEQUENCE_TABLE = {
     0:{"maqasid_pillar":"Hifz al-Mal","maqasid_meaning":"Protection of Wealth",
@@ -614,6 +638,118 @@ def classify(text: str, context: str = "general",
     if structural_override_reason:
         review_reason = structural_override_reason
 
+    # ══════════════════════════════════════════════════════════
+    # NEW — PHASE 5: RECONCILIATION LAYER
+    #
+    # The Phase 4 block above only ever touched `recommendation` —
+    # the top-line `violation_class` / `severity` / `result` fields
+    # that the frontend actually displays still came straight from
+    # the ML layer untouched. That's what caused the demo to show
+    # "Authentic / NONE" on a real Arabic Bay'-al-Inah buy-back and
+    # a real Urdu risk/ownership-decoupling clause even though the
+    # structural layer had already matched them correctly — the
+    # match existed in structural_pattern_flags, but nothing
+    # upstream of it was told to care.
+    #
+    # This block does two things, and only these two things:
+    #
+    #   (a) STRUCTURAL OVERRIDE — if a structural pattern matched,
+    #       it now overrides pred/label/conf/tier/verdict/terry/
+    #       severity themselves, not just recommendation. Because
+    #       this runs BEFORE the Phase 2/3 enrichment sections below
+    #       (which key off `pred`), the pillar, AAOIFI standard, and
+    #       jurisdiction exposure/penalty all become consistent with
+    #       the corrected verdict automatically — no separate mapping
+    #       needed for those sections.
+    #
+    #   (b) HARD-NEGATIVE DOWNGRADE — only when NO structural pattern
+    #       matched, and the ML layer flagged a violation, checks a
+    #       narrow set of known-compliant patterns (currently one:
+    #       CHN-01, the AAOIFI-permitted actual-loss indemnity formula
+    #       that was mis-flagged as Gharar/HIGH during pre-meeting
+    #       testing). A match downgrades the verdict to
+    #       REVIEW_REQUIRED — never silently back to AUTHENTIC. A
+    #       human still confirms; the system just stops presenting a
+    #       compliant clause as a confirmed violation.
+    #
+    # Neither path touches the base model's weights. Both are
+    # deterministic, narrow, and fully logged in `reconciliation`
+    # below so the audit trail always shows which layer produced the
+    # final call.
+    # ══════════════════════════════════════════════════════════
+    reconciliation = {
+        "applied"                    : False,
+        "source"                     : None,
+        "reason"                     : None,
+        "original_ml_violation_class": label,
+        "original_ml_severity"       : severity,
+        "original_ml_confidence"     : round(conf, 4),
+    }
+
+    if structural_flags_list:
+        top = max(
+            structural_flags_list,
+            key=lambda f: STRUCTURAL_SEVERITY_RANK.get(f["severity"], 0),
+        )
+        mapped_pred = STRUCTURAL_PATTERN_TO_PRED.get(top["pattern_id"])
+
+        if mapped_pred is not None:
+            pred     = mapped_pred
+            label    = f"{LABEL_NAMES[mapped_pred]} — {top['name']}"
+            conf     = max(conf, 0.95)   # deterministic match — high confidence by construction
+            tier     = "HIGH"
+            verdict  = "VIOLATION"
+            terry    = "FLAGGED"
+            severity = top["severity"]
+
+            reconciliation.update({
+                "applied": True,
+                "source" : "structural_override",
+                "reason" : (
+                    f"Deterministic structural pattern {top['pattern_id']} "
+                    f"({top['name']}) matched. This overrides the base "
+                    f"classifier's verdict for this text regardless of "
+                    f"language — structural patterns are validated directly "
+                    f"against SRB findings and are trusted over a "
+                    f"probabilistic language-signal miss."
+                ),
+                "structural_matches": [f["pattern_id"] for f in structural_flags_list],
+            })
+
+    elif verdict == "VIOLATION":
+        hard_negative = scan_hard_negatives(text, label, severity)
+        if hard_negative is not None:
+            severity = "LOW"
+            verdict  = "REVIEW_REQUIRED"
+            terry    = "UNCERTAIN"
+            needs_review = True
+            recommendation = "REVIEW"
+            review_reason = f"hard_negative_pattern_match:{hard_negative.pattern_id}"
+
+            reconciliation.update({
+                "applied": True,
+                "source" : "hard_negative_downgrade",
+                "reason" : (
+                    f"Base classifier flagged '{reconciliation['original_ml_violation_class']}' "
+                    f"({reconciliation['original_ml_severity']}), but this text matches known "
+                    f"AAOIFI-compliant pattern {hard_negative.pattern_id} "
+                    f"({hard_negative.name}). Downgraded to REVIEW_REQUIRED — "
+                    f"a human confirms, the system does not self-clear to Authentic."
+                ),
+                "hard_negative_match": hard_negative.pattern_id,
+                "hard_negative_aaoifi_standard": hard_negative.aaoifi_standard,
+            })
+
+    # Recompute movement + recommendation now that pred/verdict/severity
+    # may have changed above, so downstream fields stay consistent.
+    mv = MOVE[pred]
+    if verdict in ("AUTHENTIC","PASS"): mv = "deliver_output_to_user"
+    if severity=="CRITICAL" and verdict=="VIOLATION": mv = "quarantine_output"
+
+    if reconciliation["source"] == "structural_override":
+        recommendation = "QUARANTINE" if severity == "CRITICAL" else "ESCALATE"
+        needs_review = True
+
     # Phase 2 enrichment
     conseq      = CONSEQUENCE_TABLE[pred]
     altmap      = ALTERNATIVE_MAP[pred]
@@ -624,6 +760,21 @@ def classify(text: str, context: str = "general",
     if partial and not disclaimer:
         disclaimer = "Farsi/Tajik input — partial coverage. Scholar review recommended."
 
+    # NEW (Phase 5): when a structural override fired, use that pattern's
+    # own AAOIFI citation and explanation instead of the generic per-class
+    # ALTERNATIVE_MAP entry — it's more specific to the actual clause than
+    # the class-level default.
+    compliant_alternative = altmap["primary"]
+    structural_fix         = altmap["structural_fix"]
+    aaoifi_standard         = altmap["aaoifi_standard"]
+    if reconciliation["source"] == "structural_override":
+        aaoifi_standard = top["aaoifi_standard"]
+        structural_fix  = top["explanation"]
+    elif reconciliation["source"] == "hard_negative_downgrade":
+        aaoifi_standard        = hard_negative.aaoifi_standard
+        compliant_alternative  = "No change required — matches a recognized AAOIFI-compliant pattern"
+        structural_fix          = hard_negative.explanation
+
     # Phase 3 jurisdiction
     juris_data = JURISDICTION_EXPOSURE.get(pred, {})
     jurisdiction_recognized = jurisdiction in juris_data
@@ -675,9 +826,9 @@ def classify(text: str, context: str = "general",
             "economic_signal"    : econ_signal,
             "action_required"    : (conseq["action_required"]
                                     if tier=="HIGH" else "human_review_first"),
-            "compliant_alternative": altmap["primary"],
-            "structural_fix"     : altmap["structural_fix"],
-            "aaoifi_standard"    : altmap["aaoifi_standard"],
+            "compliant_alternative": compliant_alternative,
+            "structural_fix"     : structural_fix,
+            "aaoifi_standard"    : aaoifi_standard,
         },
 
         # Phase 3: Jurisdiction
@@ -698,7 +849,7 @@ def classify(text: str, context: str = "general",
             } if pred != 0 else "compliant — no exposure",
         },
 
-        # NEW — Phase 4: Structural pattern layer
+        # Phase 4: Structural pattern layer
         "structural_pattern_flags": {
             "layer_type": "rule-based structural detection (Phase 4)",
             "note": (
@@ -711,12 +862,17 @@ def classify(text: str, context: str = "general",
                 "1500-char truncation applied to the ML layer below. Coverage is "
                 "limited to the patterns currently defined — absence of a flag "
                 "here is NOT confirmation of structural compliance, only that none "
-                "of the currently-modeled red-flag patterns matched."
+                "of the currently-modeled red-flag patterns matched. As of Phase 5, "
+                "a match here also overrides maci_evaluation directly — see "
+                "`reconciliation` below."
             ),
             "flags": structural_flags_list,
             "flag_count": len(structural_flags_list),
         },
 
+        # NEW — Phase 5: Reconciliation layer
+        "reconciliation": reconciliation,
+
         # Overclaiming controls
         "overclaiming_controls"  : {
             "severity_is_confidence_modulated": True,
@@ -728,6 +884,8 @@ def classify(text: str, context: str = "general",
             "original_char_count" : original_length,
             "structural_layer_overrode_ml_verdict": bool(structural_override_reason),
             "structural_override_reason": structural_override_reason,
+            "reconciliation_applied": reconciliation["applied"],
+            "reconciliation_source": reconciliation["source"],
         },
 
         "boundary_behavior"      : {
@@ -769,6 +927,7 @@ def classify(text: str, context: str = "general",
             "exposure"    : juris_info.get("exposure","unknown"),
             "truncated"   : was_truncated,
             "structural_flags": len(structural_flags_list),
+            "reconciliation_applied": reconciliation["applied"],
             "audit_hash"  : f"sha256:{phash[:16]}...",
         },
 
@@ -794,6 +953,10 @@ def root():
                                     "(Bay' al-Inah, disguised Riba late-fee, "
                                     "unilateral price variation, risk/ownership "
                                     "decoupling, total liability waiver)",
+        "reconciliation_layer": "Phase 5 — structural patterns override "
+                                "maci_evaluation directly; one hard-negative "
+                                "pattern (CHN-01, actual-loss indemnity) "
+                                "downgrades known false positives to REVIEW_REQUIRED",
         "endpoints" : {
             "single"    : "POST /api/v1/classify",
             "batch"     : "POST /api/v1/classify/batch",
@@ -817,6 +980,7 @@ def health():
         "ml_f1"        : 0.9026,
         "rulings"      : len(STATE.get("rulings",[])),
         "structural_pattern_layer": "active",
+        "reconciliation_layer": "active",
     }
 
 @app.post(
@@ -830,7 +994,7 @@ def health():
                 "Returns the full MACI-0.4 signal packet, including "
                 "maqasid_enrichment (AAOIFI standard, structural fix, "
                 "compliant alternative), jurisdiction_overlay, "
-                "structural_pattern_flags, and overclaiming_controls."
+                "structural_pattern_flags, reconciliation, and overclaiming_controls."
 )
 def classify_endpoint(req: ClassifyRequest):
     if "xlmr" not in STATE:
@@ -982,4 +1146,4 @@ def classify_audit(req: AuditRequest):
         "mechanism_results": results,
 
         "audit_hash"      : f"sha256:{audit_hash}",
-    }
+    }