runtime/cell8a.py

#@title 8A. COMPREHENSIVE MULTIDOMAIN REGRESSION + SAFETY v2
# Complete replacement for Cell 8A.
# Run after Cell 8 v4 and before Cell 9.
# Compatible with:
#   Cell 5 v4.1 -> Cell 5A v3.4 -> Cell 6 v5.1 -> Cell 7 v4
#   -> Cell 7A v2 -> Cell 7B v2 -> Cell 8 v4

import json
import re
from collections import Counter

import pandas as pd

EXPECTED_8A_RULE_COUNT = 30

_REQUIRED_8A = [
    "ENGINE", "Engine", "Analyzer", "Generator", "Validator", "Ranker",
    "PROFILES", "RULES", "CFG", "clean", "surface_key",
    "realization_issues", "expanded_regression", "safety_tests",
    "relation_regression",
]
_missing_8a = [name for name in _REQUIRED_8A if name not in globals()]
if _missing_8a:
    raise RuntimeError(
        "Cell 8A cannot run. Missing: " + ", ".join(_missing_8a)
        + ". Run Cells 1–8 v4 in order first."
    )
if len(RULES) != EXPECTED_8A_RULE_COUNT:
    raise RuntimeError(
        f"Cell 8A expected {EXPECTED_8A_RULE_COUNT} rules but found {len(RULES)}."
    )
if getattr(Engine.one, "__name__", "") != "_one_with_must_paraphrase_v2":
    raise RuntimeError("Cell 8A requires Cell 7B v2.")


# =====================================================================
# 1. MULTIDOMAIN CORPUS
# =====================================================================

REGRESSION_CORPUS = [
    # Academic
    {"id":"ACA-01","domain":"Academic","mode":"Balanced","text":"The next step is to examine how representation quality affects classifier robustness.","protected":"classifier robustness","expected_any":{"COPULAR_ACTION_FRAME"}},
    {"id":"ACA-02","domain":"Academic","mode":"Balanced","text":"These findings provide evidence that adversarial training improves robustness under the evaluated attack settings.","protected":"adversarial training, robustness","expected_any":{"EVIDENTIAL_COMPLEMENT"}},
    {"id":"ACA-03","domain":"Academic","mode":"Balanced","text":"The study requires consideration of whether the observed effect generalizes across datasets.","protected":"datasets","expected_any":{"WHETHER_COMPLEMENT"}},
    {"id":"ACA-04","domain":"Academic","mode":"Balanced","text":"Applying the same evaluation protocol across both platforms reduces methodological confounding.","protected":"evaluation protocol","expected_any":{"GERUND_SUBJECT"}},
    {"id":"ACA-05","domain":"Academic","mode":"Structural","text":"The framework preserves platform-specific constraints while applying a common evaluation procedure.","protected":"platform-specific constraints","expected_any":{"WHILE_PARTICIPIAL"}},
    {"id":"ACA-06","domain":"Academic","mode":"Balanced","text":"The purpose is not to identify a universally superior model but to compare robustness across feature spaces.","protected":"robustness, feature spaces","expected_any":{"NEGATED_PURPOSE_CONTRAST"}},

    # Technical
    {"id":"TEC-01","domain":"Technical","mode":"Balanced","text":"The next step is to inspect how the deployed handler checks for GPT configuration.","protected":"GPT configuration","expected_any":{"COPULAR_ACTION_FRAME"}},
    {"id":"TEC-02","domain":"Technical","mode":"Balanced","text":"We need to distinguish those possibilities using the existing source.","protected":"existing source","expected_any":{"NEED_TO_ACTION","USING_METHOD_RELATION","METHOD_RELATION"}},
    {"id":"TEC-03","domain":"Technical","mode":"Balanced","text":"The fault could be in the environment-variable lookup, the initialization routine, or the error-reporting logic.","protected":"environment-variable lookup, initialization routine, error-reporting logic","expected_any":{"MODAL_COPULAR_ENUMERATION","ENUMERATION"}},
    {"id":"TEC-04","domain":"Technical","mode":"Balanced","text":"The service is evaluated using latency, throughput, and failure-rate measurements.","protected":"latency, throughput, failure-rate","expected_any":{"PASSIVE_USING","USING_METHOD_RELATION"}},
    {"id":"TEC-05","domain":"Technical","mode":"Structural","text":"The deployment was tested by the engineering team.","protected":"deployment, engineering team","expected_any":{"PASSIVE"}},
    {"id":"TEC-06","domain":"Technical","mode":"Balanced","text":"The process reduces configuration errors by validating required environment variables before startup.","protected":"environment variables","expected_any":{"BY_GERUND_METHOD_RELATION","METHOD_RELATION"}},

    # Business
    {"id":"BUS-01","domain":"Business","mode":"Balanced","text":"The next step is to review the supplier's revised implementation schedule.","protected":"implementation schedule","expected_any":{"COPULAR_ACTION_FRAME"}},
    {"id":"BUS-02","domain":"Business","mode":"Balanced","text":"The delay could be in procurement, vendor mobilization, or material delivery.","protected":"procurement, vendor mobilization, material delivery","expected_any":{"MODAL_COPULAR_ENUMERATION","ENUMERATION"}},
    {"id":"BUS-03","domain":"Business","mode":"Balanced","text":"We need to verify the invoice totals using the approved cost ledger.","protected":"invoice totals, approved cost ledger","expected_any":{"NEED_TO_ACTION","USING_METHOD_RELATION","METHOD_RELATION"}},
    {"id":"BUS-04","domain":"Business","mode":"Balanced","text":"The team intends to submit the report this week rather than waiting until month-end.","protected":"report, month-end","expected_any":{"FINITE_PURPOSE_INTENT","RATHER_THAN_GERUND"}},
    {"id":"BUS-05","domain":"Business","mode":"Structural","text":"The manager reviewed the estimate and approved the revised budget.","protected":"estimate, revised budget","expected_any":{"COORDINATED_PREDICATES"}},
    {"id":"BUS-06","domain":"Business","mode":"Structural","text":"The schedule identifies mobilization, procurement, installation, testing, and closeout; each stage has a separate milestone.","protected":"mobilization, procurement, installation, testing, closeout","expected_any":{"LONG_MULTI_CLAUSE","ENUMERATION"}},

    # Contract / formal requirements
    {"id":"CON-01","domain":"Contract","mode":"Balanced","text":"For purposes of this plan, a Sub-RFR is an individual reimbursement request for eligible paid costs.","protected":"Sub-RFR, eligible paid costs","expected_any":{"FRONTED_ADJUNCT"},"must_remain_unchanged":True},
    {"id":"CON-02","domain":"Contract","mode":"Balanced","text":"To support reimbursement processing, the Contractor shall provide invoices, inspection reports, photographs, lien releases, and other supporting documentation.","protected":"Contractor, invoices, inspection reports, photographs, lien releases","expected_any":{"PURPOSE_INFINITIVE_FRONTING","ENUMERATION"}},
    {"id":"CON-03","domain":"Contract","mode":"Balanced","text":"Eligible invoices must be properly documented, reviewed, and approved by WSC.","protected":"Eligible invoices, WSC","expected_any":{"COORDINATED_PASSIVE_PARTICIPLES","COORDINATED_PREDICATES"}},
    {"id":"CON-04","domain":"Contract","mode":"Balanced","text":"WSC intends to submit reimbursement requests progressively rather than waiting until project completion.","protected":"WSC, reimbursement requests, project completion","expected_any":{"FINITE_PURPOSE_INTENT","RATHER_THAN_GERUND"}},
    {"id":"CON-05","domain":"Contract","mode":"Structural","text":"The work was inspected by the Owner's representative.","protected":"Owner's representative","expected_any":{"PASSIVE"}},
    {"id":"CON-06","domain":"Contract","mode":"Balanced","text":"Payments are based on approved work rather than on the passage of time.","protected":"approved work, passage of time","expected_any":{"PASSIVE","CONTRAST"}},

    # General / out-of-sample primary-clause structures
    {"id":"GEN-01","domain":"General","mode":"Balanced","text":"General-purpose machine-learning systems often train separate models without explicit rules for competency assessment, evidence-grounded correction, and graduation.","protected":"competency assessment, evidence-grounded correction, graduation","expected_any":{"FINITE_COMPLEMENT_LIST"}},
    {"id":"GEN-02","domain":"General","mode":"Balanced","text":"We propose Framework X (FX), a domain-independent orchestration protocol consisting of contracts, examinations, peer study, and auditable promotion.","protected":"Framework X, FX, contracts, examinations, peer study, auditable promotion","expected_any":{"PROPOSAL_FRAME","APPOSITIVE_DEFINITION"}},
    {"id":"GEN-03","domain":"General","mode":"Balanced","text":"The experimental testbed applies this protocol to small cybersecurity learners spanning classification, generative drafting, and structured diagram planning.","protected":"small cybersecurity learners, classification, generative drafting, structured diagram planning","expected_any":{"PARTICIPIAL_POSTMODIFIER"}},
    {"id":"GEN-04","domain":"General","mode":"Balanced","text":"We pre-register comparable questions, disjoint data, matched baselines, ablations, and independently held-out testing.","protected":"comparable questions, disjoint data, matched baselines, ablations, independently held-out testing","expected_any":{"FINITE_OBJECT_LIST_FRAME"}},
    {"id":"GEN-05","domain":"General","mode":"Balanced","text":"Research claims about improvement, cost, generality, and deployment are contingent on execution and empirical analysis.","protected":"improvement, cost, generality, deployment, execution, empirical analysis","expected_any":{"CONTINGENCY_RELATION"}},
]


# =====================================================================
# 2. HELPERS
# =====================================================================

def _terms(text):
    return {part.strip() for part in re.split(r"[,;\n]", text or "") if part.strip()}


def _contains_term(text, term):
    return re.search(re.escape(term), text, re.I) is not None


def _protected_missing(source, revised, protected_text):
    return [
        term for term in _terms(protected_text)
        if _contains_term(source, term) and not _contains_term(revised, term)
    ]


def _candidate_trace_count(trace):
    return len(trace.get("candidate_traces", []) or [])


def _realization_issues_safe(text):
    try:
        return list(realization_issues(text) or [])
    except Exception as exc:
        return [f"REALIZATION_CHECK_ERROR:{exc}"]


def _process_case(case):
    profile, routing = ENGINE.route(case["text"], "Auto Detect")
    result = ENGINE.one(
        case["text"],
        profile,
        _terms(case.get("protected", "")),
        case.get("mode", "Balanced"),
    )

    trace = dict(getattr(result, "diagnostics", {}) or {})
    must = trace.get("must_paraphrase", {}) or {}
    families = set(getattr(result, "families", []) or [])
    revised = getattr(result, "revised", case["text"])
    status = getattr(result, "status", "UNKNOWN")
    changed = surface_key(revised) != surface_key(case["text"])
    missing = _protected_missing(case["text"], revised, case.get("protected", ""))
    realization = _realization_issues_safe(revised)
    expected = set(case.get("expected_any", set()))
    family_hit = not expected or bool(families & expected)
    must_remain = bool(case.get("must_remain_unchanged", False))

    terminal_ok = status in {
        "SUCCESS", "NO_PATTERN_COVERAGE", "CANDIDATE_GENERATION_FAILURE",
        "TERMINOLOGY_REJECTION", "POLARITY_REJECTION", "MODALITY_REJECTION",
        "QUANTITY_REJECTION", "ENUMERATION_REJECTION", "GRAMMAR_REJECTION",
        "SEMANTIC_REJECTION", "INSUFFICIENT_REWRITE", "RANKING_EMPTY",
        "PARSE_FAILURE",
    }

    # Must-paraphrase contract:
    # - successful results must be changed;
    # - unchanged results must document exhaustive safe search;
    mp_outcome = must.get("outcome")
    mp_contract = (
        (status == "SUCCESS" and changed and mp_outcome == "SAFE_CHANGED_OUTPUT")
        or (
            status != "SUCCESS"
            and not changed
            and mp_outcome == "ORIGINAL_PRESERVED_AFTER_EXHAUSTIVE_SAFE_SEARCH"
        )
    )

    pass_core = (
        terminal_ok
        and not missing
        and not realization
        and (not must_remain or not changed)
        and (family_hit or status == "NO_PATTERN_COVERAGE")
        and mp_contract
    )

    return {
        "ID": case["id"],
        "Domain": case["domain"],
        "Mode": case.get("mode", "Balanced"),
        "Status": status,
        "Changed": changed,
        "Detected families": " | ".join(sorted(families)),
        "Expected family hit": family_hit,
        "Selected family": getattr(result, "family", None),
        "Requested mode": must.get("requested_mode"),
        "Accepted mode": must.get("accepted_mode"),
        "Attempt count": must.get("attempt_count"),
        "Must-paraphrase outcome": mp_outcome,
        "Must-paraphrase contract": mp_contract,
        "Generated Minimal": (trace.get("generated_by_lane") or {}).get("Minimal", 0),
        "Generated Balanced": (trace.get("generated_by_lane") or {}).get("Balanced", 0),
        "Generated Structural": (trace.get("generated_by_lane") or {}).get("Structural", 0),
        "Before final filter": trace.get("generated_before_final_filter"),
        "After final filter": trace.get("generated_after_final_filter"),
        "Candidate traces": _candidate_trace_count(trace),
        "Missing protected": missing,
        "Realization issues": realization,
        "Must remain unchanged": must_remain,
        "Core pass": pass_core,
        "Original": case["text"],
        "Revised": revised,
        "Reason": getattr(result, "reason", ""),
        "Rule errors": trace.get("rule_errors", []),
        "Rejections": trace.get("rejections", {}),
        "Routing evidence": routing,
    }


# =====================================================================
# 3. SAFETY SUITE
# =====================================================================

SAFETY_CASES = [
    {"id":"SAFE-01","name":"Subject/object reversal","source":"XGBoost outperformed DNN.","candidate":"DNN outperformed XGBoost.","expected":"SEMANTIC_REJECTION"},
    {"id":"SAFE-02","name":"Negation removal","source":"The model did not fail.","candidate":"The model failed.","expected":"POLARITY_REJECTION"},
    {"id":"SAFE-03","name":"Negation addition","source":"The model failed.","candidate":"The model did not fail.","expected":"POLARITY_REJECTION"},
    {"id":"SAFE-04","name":"Quantity change","source":"The study evaluated 26 attacks.","candidate":"The study evaluated 25 attacks.","expected":"QUANTITY_REJECTION"},
    {"id":"SAFE-05","name":"Modality strengthening","source":"The model may improve robustness.","candidate":"The model improves robustness.","expected":"MODALITY_REJECTION"},
    {"id":"SAFE-06","name":"Enumeration omission","source":"The system evaluated accuracy, precision, recall, and F1 score.","candidate":"The system evaluated accuracy, precision, and recall.","expected":"ENUMERATION_REJECTION"},
    {"id":"SAFE-07","name":"Protected terminology removal","source":"The XGBoost model was evaluated.","candidate":"The model was evaluated.","expected":"TERMINOLOGY_REJECTION","protected":{"XGBoost"}},
    {"id":"SAFE-08","name":"Citation removal","source":"Demetrio et al. (2021) evaluated PE-file manipulations.","candidate":"The study evaluated PE-file manipulations.","expected":"TERMINOLOGY_REJECTION"},
]


def _safety_case(case):
    profile = PROFILES.get("General Academic", next(iter(PROFILES.values())))
    protected = set(case.get("protected", set()))
    analysis = Analyzer().analyze(case["source"], profile, protected)
    candidate = Candidate(
        case["candidate"], "Balanced", "SAFETY_TEST", "Adversarial safety test", .5
    )
    accepted, failures, checks = Validator().validate(
        case["source"], candidate, analysis, profile, protected, "Balanced"
    )
    failures = list(failures or [])
    return {
        "ID": case["id"],
        "Test": case["name"],
        "Expected": case["expected"],
        "Accepted": bool(accepted),
        "Failures": failures,
        "Pass": case["expected"] in failures,
        "Checks": checks,
    }


# =====================================================================
# 4. RUNNER
# =====================================================================


def run_comprehensive_regression(show_all=True):
    # Keep optional model resources off so results test the deterministic stack.
    previous = {
        key: getattr(CFG, key, None)
        for key in ["sts", "nli", "mlm", "lt", "online"]
        if hasattr(CFG, key)
    }
    for key in previous:
        setattr(CFG, key, False)

    rows = []
    crashes = []
    try:
        for case in REGRESSION_CORPUS:
            try:
                rows.append(_process_case(case))
            except Exception as exc:
                crashes.append({"ID":case["id"],"Domain":case["domain"],"Error":repr(exc)})
                rows.append({
                    "ID":case["id"],"Domain":case["domain"],"Mode":case.get("mode","Balanced"),
                    "Status":"TEST_CRASH","Changed":False,"Detected families":"",
                    "Expected family hit":False,"Selected family":None,
                    "Requested mode":None,"Accepted mode":None,"Attempt count":None,
                    "Must-paraphrase outcome":None,"Must-paraphrase contract":False,
                    "Generated Minimal":0,"Generated Balanced":0,"Generated Structural":0,
                    "Before final filter":0,"After final filter":0,"Candidate traces":0,
                    "Missing protected":[],"Realization issues":[repr(exc)],
                    "Must remain unchanged":False,"Core pass":False,
                    "Original":case["text"],"Revised":case["text"],
                    "Reason":"Regression test crashed.","Rule errors":[],"Rejections":{},
                    "Routing evidence":[],
                })

        safety_rows = []
        for case in SAFETY_CASES:
            try:
                safety_rows.append(_safety_case(case))
            except Exception as exc:
                safety_rows.append({
                    "ID":case["id"],"Test":case["name"],"Expected":case["expected"],
                    "Accepted":None,"Failures":[repr(exc)],"Pass":False,"Checks":{},
                })
    finally:
        for key, value in previous.items():
            setattr(CFG, key, value)

    regression_df = pd.DataFrame(rows)
    safety_df = pd.DataFrame(safety_rows)

    domain_summary = regression_df.groupby("Domain", dropna=False).agg(
        Sentences=("ID","count"),
        Successes=("Status",lambda values:int((values=="SUCCESS").sum())),
        Changed=("Changed","sum"),
        Core_passes=("Core pass","sum"),
        Family_hits=("Expected family hit","sum"),
        MP_contracts=("Must-paraphrase contract","sum"),
        Crashes=("Status",lambda values:int((values=="TEST_CRASH").sum())),
    ).reset_index()

    status_summary = (
        regression_df.groupby(["Domain","Status"], dropna=False)
        .size().reset_index(name="Count")
    )

    quality_failures = regression_df[
        (~regression_df["Core pass"])
        | regression_df["Realization issues"].map(bool)
        | regression_df["Missing protected"].map(bool)
        | regression_df["Rule errors"].map(bool)
    ].copy()

    changed_without_trace = regression_df[
        regression_df["Changed"] & (regression_df["Candidate traces"] == 0)
    ].copy()

    summary = {
        "regression_sentences": int(len(regression_df)),
        "domains": int(regression_df["Domain"].nunique()),
        "successes": int((regression_df["Status"] == "SUCCESS").sum()),
        "changed": int(regression_df["Changed"].sum()),
        "core_passes": int(regression_df["Core pass"].sum()),
        "family_hits": int(regression_df["Expected family hit"].sum()),
        "must_paraphrase_contracts": int(regression_df["Must-paraphrase contract"].sum()),
        "crashes": int((regression_df["Status"] == "TEST_CRASH").sum()),
        "quality_failures": int(len(quality_failures)),
        "changed_without_trace": int(len(changed_without_trace)),
        "safety_tests": int(len(safety_df)),
        "safety_passes": int(safety_df["Pass"].sum()),
        "registered_rules": int(len(RULES)),
    }

    print("=" * 88)
    print("COMPREHENSIVE MULTIDOMAIN REGRESSION SUMMARY v2")
    print("=" * 88)
    print(json.dumps(summary, indent=2))
    print("=" * 88)

    try:
        display(domain_summary)
        display(status_summary)
        display(safety_df[["ID","Test","Expected","Accepted","Failures","Pass"]])
        if show_all:
            display(regression_df)
        if len(quality_failures):
            print("QUALITY FAILURES")
            display(quality_failures)
        if len(changed_without_trace):
            print("CHANGED OUTPUTS WITHOUT CANDIDATE TRACE")
            display(changed_without_trace)
    except NameError:
        print(domain_summary.to_string(index=False))
        print(status_summary.to_string(index=False))

    return {
        "summary":summary,
        "regression":regression_df,
        "domain_summary":domain_summary,
        "status_summary":status_summary,
        "safety":safety_df,
        "quality_failures":quality_failures,
        "changed_without_trace":changed_without_trace,
        "crashes":crashes,
    }


# =====================================================================
# 5. EXPORT
# =====================================================================


def export_comprehensive_regression(results, filename="nll_comprehensive_regression_v2.xlsx"):
    path = f"/content/{filename}"
    with pd.ExcelWriter(path, engine="openpyxl") as writer:
        results["regression"].to_excel(writer, sheet_name="Regression", index=False)
        results["domain_summary"].to_excel(writer, sheet_name="Domain Summary", index=False)
        results["status_summary"].to_excel(writer, sheet_name="Status Summary", index=False)
        results["safety"].to_excel(writer, sheet_name="Safety", index=False)
        results["quality_failures"].to_excel(writer, sheet_name="Quality Failures", index=False)
    print("Saved:", path)
    return path


print("=" * 88)
print("CELL 8A COMPREHENSIVE MULTIDOMAIN REGRESSION SUITE v2 LOADED")
print("=" * 88)
print("Aligned pipeline: 5 v4.1 -> 5A v3.4 -> 6 v5.1 -> 7 v4 -> 7A v2 -> 7B v2 -> 8 v4")
print("Regression sentences:", len(REGRESSION_CORPUS))
print("Domains:", sorted({case['domain'] for case in REGRESSION_CORPUS}))
print("Adversarial safety tests:", len(SAFETY_CASES))
print("Registered transformation rules:", len(RULES))
print("Must-paraphrase contract checks: active")
print("Out-of-sample primary-clause cases: active")
print("Run: RESULTS_8A = run_comprehensive_regression()")
print("Optional export: export_comprehensive_regression(RESULTS_8A)")
print("=" * 88)