#@title 8A. COMPREHENSIVE MULTIDOMAIN REGRESSION + SAFETY v2 # Complete replacement for Cell 8A. # Run after Cell 8 v4 and before Cell 9. # Compatible with: # Cell 5 v4.1 -> Cell 5A v3.4 -> Cell 6 v5.1 -> Cell 7 v4 # -> Cell 7A v2 -> Cell 7B v2 -> Cell 8 v4 import json import re from collections import Counter import pandas as pd EXPECTED_8A_RULE_COUNT = 30 _REQUIRED_8A = [ "ENGINE", "Engine", "Analyzer", "Generator", "Validator", "Ranker", "PROFILES", "RULES", "CFG", "clean", "surface_key", "realization_issues", "expanded_regression", "safety_tests", "relation_regression", ] _missing_8a = [name for name in _REQUIRED_8A if name not in globals()] if _missing_8a: raise RuntimeError( "Cell 8A cannot run. Missing: " + ", ".join(_missing_8a) + ". Run Cells 1–8 v4 in order first." ) if len(RULES) != EXPECTED_8A_RULE_COUNT: raise RuntimeError( f"Cell 8A expected {EXPECTED_8A_RULE_COUNT} rules but found {len(RULES)}." ) if getattr(Engine.one, "__name__", "") != "_one_with_must_paraphrase_v2": raise RuntimeError("Cell 8A requires Cell 7B v2.") # ===================================================================== # 1. MULTIDOMAIN CORPUS # ===================================================================== REGRESSION_CORPUS = [ # Academic {"id":"ACA-01","domain":"Academic","mode":"Balanced","text":"The next step is to examine how representation quality affects classifier robustness.","protected":"classifier robustness","expected_any":{"COPULAR_ACTION_FRAME"}}, {"id":"ACA-02","domain":"Academic","mode":"Balanced","text":"These findings provide evidence that adversarial training improves robustness under the evaluated attack settings.","protected":"adversarial training, robustness","expected_any":{"EVIDENTIAL_COMPLEMENT"}}, {"id":"ACA-03","domain":"Academic","mode":"Balanced","text":"The study requires consideration of whether the observed effect generalizes across datasets.","protected":"datasets","expected_any":{"WHETHER_COMPLEMENT"}}, {"id":"ACA-04","domain":"Academic","mode":"Balanced","text":"Applying the same evaluation protocol across both platforms reduces methodological confounding.","protected":"evaluation protocol","expected_any":{"GERUND_SUBJECT"}}, {"id":"ACA-05","domain":"Academic","mode":"Structural","text":"The framework preserves platform-specific constraints while applying a common evaluation procedure.","protected":"platform-specific constraints","expected_any":{"WHILE_PARTICIPIAL"}}, {"id":"ACA-06","domain":"Academic","mode":"Balanced","text":"The purpose is not to identify a universally superior model but to compare robustness across feature spaces.","protected":"robustness, feature spaces","expected_any":{"NEGATED_PURPOSE_CONTRAST"}}, # Technical {"id":"TEC-01","domain":"Technical","mode":"Balanced","text":"The next step is to inspect how the deployed handler checks for GPT configuration.","protected":"GPT configuration","expected_any":{"COPULAR_ACTION_FRAME"}}, {"id":"TEC-02","domain":"Technical","mode":"Balanced","text":"We need to distinguish those possibilities using the existing source.","protected":"existing source","expected_any":{"NEED_TO_ACTION","USING_METHOD_RELATION","METHOD_RELATION"}}, {"id":"TEC-03","domain":"Technical","mode":"Balanced","text":"The fault could be in the environment-variable lookup, the initialization routine, or the error-reporting logic.","protected":"environment-variable lookup, initialization routine, error-reporting logic","expected_any":{"MODAL_COPULAR_ENUMERATION","ENUMERATION"}}, {"id":"TEC-04","domain":"Technical","mode":"Balanced","text":"The service is evaluated using latency, throughput, and failure-rate measurements.","protected":"latency, throughput, failure-rate","expected_any":{"PASSIVE_USING","USING_METHOD_RELATION"}}, {"id":"TEC-05","domain":"Technical","mode":"Structural","text":"The deployment was tested by the engineering team.","protected":"deployment, engineering team","expected_any":{"PASSIVE"}}, {"id":"TEC-06","domain":"Technical","mode":"Balanced","text":"The process reduces configuration errors by validating required environment variables before startup.","protected":"environment variables","expected_any":{"BY_GERUND_METHOD_RELATION","METHOD_RELATION"}}, # Business {"id":"BUS-01","domain":"Business","mode":"Balanced","text":"The next step is to review the supplier's revised implementation schedule.","protected":"implementation schedule","expected_any":{"COPULAR_ACTION_FRAME"}}, {"id":"BUS-02","domain":"Business","mode":"Balanced","text":"The delay could be in procurement, vendor mobilization, or material delivery.","protected":"procurement, vendor mobilization, material delivery","expected_any":{"MODAL_COPULAR_ENUMERATION","ENUMERATION"}}, {"id":"BUS-03","domain":"Business","mode":"Balanced","text":"We need to verify the invoice totals using the approved cost ledger.","protected":"invoice totals, approved cost ledger","expected_any":{"NEED_TO_ACTION","USING_METHOD_RELATION","METHOD_RELATION"}}, {"id":"BUS-04","domain":"Business","mode":"Balanced","text":"The team intends to submit the report this week rather than waiting until month-end.","protected":"report, month-end","expected_any":{"FINITE_PURPOSE_INTENT","RATHER_THAN_GERUND"}}, {"id":"BUS-05","domain":"Business","mode":"Structural","text":"The manager reviewed the estimate and approved the revised budget.","protected":"estimate, revised budget","expected_any":{"COORDINATED_PREDICATES"}}, {"id":"BUS-06","domain":"Business","mode":"Structural","text":"The schedule identifies mobilization, procurement, installation, testing, and closeout; each stage has a separate milestone.","protected":"mobilization, procurement, installation, testing, closeout","expected_any":{"LONG_MULTI_CLAUSE","ENUMERATION"}}, # Contract / formal requirements {"id":"CON-01","domain":"Contract","mode":"Balanced","text":"For purposes of this plan, a Sub-RFR is an individual reimbursement request for eligible paid costs.","protected":"Sub-RFR, eligible paid costs","expected_any":{"FRONTED_ADJUNCT"},"must_remain_unchanged":True}, {"id":"CON-02","domain":"Contract","mode":"Balanced","text":"To support reimbursement processing, the Contractor shall provide invoices, inspection reports, photographs, lien releases, and other supporting documentation.","protected":"Contractor, invoices, inspection reports, photographs, lien releases","expected_any":{"PURPOSE_INFINITIVE_FRONTING","ENUMERATION"}}, {"id":"CON-03","domain":"Contract","mode":"Balanced","text":"Eligible invoices must be properly documented, reviewed, and approved by WSC.","protected":"Eligible invoices, WSC","expected_any":{"COORDINATED_PASSIVE_PARTICIPLES","COORDINATED_PREDICATES"}}, {"id":"CON-04","domain":"Contract","mode":"Balanced","text":"WSC intends to submit reimbursement requests progressively rather than waiting until project completion.","protected":"WSC, reimbursement requests, project completion","expected_any":{"FINITE_PURPOSE_INTENT","RATHER_THAN_GERUND"}}, {"id":"CON-05","domain":"Contract","mode":"Structural","text":"The work was inspected by the Owner's representative.","protected":"Owner's representative","expected_any":{"PASSIVE"}}, {"id":"CON-06","domain":"Contract","mode":"Balanced","text":"Payments are based on approved work rather than on the passage of time.","protected":"approved work, passage of time","expected_any":{"PASSIVE","CONTRAST"}}, # General / out-of-sample primary-clause structures {"id":"GEN-01","domain":"General","mode":"Balanced","text":"General-purpose machine-learning systems often train separate models without explicit rules for competency assessment, evidence-grounded correction, and graduation.","protected":"competency assessment, evidence-grounded correction, graduation","expected_any":{"FINITE_COMPLEMENT_LIST"}}, {"id":"GEN-02","domain":"General","mode":"Balanced","text":"We propose Framework X (FX), a domain-independent orchestration protocol consisting of contracts, examinations, peer study, and auditable promotion.","protected":"Framework X, FX, contracts, examinations, peer study, auditable promotion","expected_any":{"PROPOSAL_FRAME","APPOSITIVE_DEFINITION"}}, {"id":"GEN-03","domain":"General","mode":"Balanced","text":"The experimental testbed applies this protocol to small cybersecurity learners spanning classification, generative drafting, and structured diagram planning.","protected":"small cybersecurity learners, classification, generative drafting, structured diagram planning","expected_any":{"PARTICIPIAL_POSTMODIFIER"}}, {"id":"GEN-04","domain":"General","mode":"Balanced","text":"We pre-register comparable questions, disjoint data, matched baselines, ablations, and independently held-out testing.","protected":"comparable questions, disjoint data, matched baselines, ablations, independently held-out testing","expected_any":{"FINITE_OBJECT_LIST_FRAME"}}, {"id":"GEN-05","domain":"General","mode":"Balanced","text":"Research claims about improvement, cost, generality, and deployment are contingent on execution and empirical analysis.","protected":"improvement, cost, generality, deployment, execution, empirical analysis","expected_any":{"CONTINGENCY_RELATION"}}, ] # ===================================================================== # 2. HELPERS # ===================================================================== def _terms(text): return {part.strip() for part in re.split(r"[,;\n]", text or "") if part.strip()} def _contains_term(text, term): return re.search(re.escape(term), text, re.I) is not None def _protected_missing(source, revised, protected_text): return [ term for term in _terms(protected_text) if _contains_term(source, term) and not _contains_term(revised, term) ] def _candidate_trace_count(trace): return len(trace.get("candidate_traces", []) or []) def _realization_issues_safe(text): try: return list(realization_issues(text) or []) except Exception as exc: return [f"REALIZATION_CHECK_ERROR:{exc}"] def _process_case(case): profile, routing = ENGINE.route(case["text"], "Auto Detect") result = ENGINE.one( case["text"], profile, _terms(case.get("protected", "")), case.get("mode", "Balanced"), ) trace = dict(getattr(result, "diagnostics", {}) or {}) must = trace.get("must_paraphrase", {}) or {} families = set(getattr(result, "families", []) or []) revised = getattr(result, "revised", case["text"]) status = getattr(result, "status", "UNKNOWN") changed = surface_key(revised) != surface_key(case["text"]) missing = _protected_missing(case["text"], revised, case.get("protected", "")) realization = _realization_issues_safe(revised) expected = set(case.get("expected_any", set())) family_hit = not expected or bool(families & expected) must_remain = bool(case.get("must_remain_unchanged", False)) terminal_ok = status in { "SUCCESS", "NO_PATTERN_COVERAGE", "CANDIDATE_GENERATION_FAILURE", "TERMINOLOGY_REJECTION", "POLARITY_REJECTION", "MODALITY_REJECTION", "QUANTITY_REJECTION", "ENUMERATION_REJECTION", "GRAMMAR_REJECTION", "SEMANTIC_REJECTION", "INSUFFICIENT_REWRITE", "RANKING_EMPTY", "PARSE_FAILURE", } # Must-paraphrase contract: # - successful results must be changed; # - unchanged results must document exhaustive safe search; mp_outcome = must.get("outcome") mp_contract = ( (status == "SUCCESS" and changed and mp_outcome == "SAFE_CHANGED_OUTPUT") or ( status != "SUCCESS" and not changed and mp_outcome == "ORIGINAL_PRESERVED_AFTER_EXHAUSTIVE_SAFE_SEARCH" ) ) pass_core = ( terminal_ok and not missing and not realization and (not must_remain or not changed) and (family_hit or status == "NO_PATTERN_COVERAGE") and mp_contract ) return { "ID": case["id"], "Domain": case["domain"], "Mode": case.get("mode", "Balanced"), "Status": status, "Changed": changed, "Detected families": " | ".join(sorted(families)), "Expected family hit": family_hit, "Selected family": getattr(result, "family", None), "Requested mode": must.get("requested_mode"), "Accepted mode": must.get("accepted_mode"), "Attempt count": must.get("attempt_count"), "Must-paraphrase outcome": mp_outcome, "Must-paraphrase contract": mp_contract, "Generated Minimal": (trace.get("generated_by_lane") or {}).get("Minimal", 0), "Generated Balanced": (trace.get("generated_by_lane") or {}).get("Balanced", 0), "Generated Structural": (trace.get("generated_by_lane") or {}).get("Structural", 0), "Before final filter": trace.get("generated_before_final_filter"), "After final filter": trace.get("generated_after_final_filter"), "Candidate traces": _candidate_trace_count(trace), "Missing protected": missing, "Realization issues": realization, "Must remain unchanged": must_remain, "Core pass": pass_core, "Original": case["text"], "Revised": revised, "Reason": getattr(result, "reason", ""), "Rule errors": trace.get("rule_errors", []), "Rejections": trace.get("rejections", {}), "Routing evidence": routing, } # ===================================================================== # 3. SAFETY SUITE # ===================================================================== SAFETY_CASES = [ {"id":"SAFE-01","name":"Subject/object reversal","source":"XGBoost outperformed DNN.","candidate":"DNN outperformed XGBoost.","expected":"SEMANTIC_REJECTION"}, {"id":"SAFE-02","name":"Negation removal","source":"The model did not fail.","candidate":"The model failed.","expected":"POLARITY_REJECTION"}, {"id":"SAFE-03","name":"Negation addition","source":"The model failed.","candidate":"The model did not fail.","expected":"POLARITY_REJECTION"}, {"id":"SAFE-04","name":"Quantity change","source":"The study evaluated 26 attacks.","candidate":"The study evaluated 25 attacks.","expected":"QUANTITY_REJECTION"}, {"id":"SAFE-05","name":"Modality strengthening","source":"The model may improve robustness.","candidate":"The model improves robustness.","expected":"MODALITY_REJECTION"}, {"id":"SAFE-06","name":"Enumeration omission","source":"The system evaluated accuracy, precision, recall, and F1 score.","candidate":"The system evaluated accuracy, precision, and recall.","expected":"ENUMERATION_REJECTION"}, {"id":"SAFE-07","name":"Protected terminology removal","source":"The XGBoost model was evaluated.","candidate":"The model was evaluated.","expected":"TERMINOLOGY_REJECTION","protected":{"XGBoost"}}, {"id":"SAFE-08","name":"Citation removal","source":"Demetrio et al. (2021) evaluated PE-file manipulations.","candidate":"The study evaluated PE-file manipulations.","expected":"TERMINOLOGY_REJECTION"}, ] def _safety_case(case): profile = PROFILES.get("General Academic", next(iter(PROFILES.values()))) protected = set(case.get("protected", set())) analysis = Analyzer().analyze(case["source"], profile, protected) candidate = Candidate( case["candidate"], "Balanced", "SAFETY_TEST", "Adversarial safety test", .5 ) accepted, failures, checks = Validator().validate( case["source"], candidate, analysis, profile, protected, "Balanced" ) failures = list(failures or []) return { "ID": case["id"], "Test": case["name"], "Expected": case["expected"], "Accepted": bool(accepted), "Failures": failures, "Pass": case["expected"] in failures, "Checks": checks, } # ===================================================================== # 4. RUNNER # ===================================================================== def run_comprehensive_regression(show_all=True): # Keep optional model resources off so results test the deterministic stack. previous = { key: getattr(CFG, key, None) for key in ["sts", "nli", "mlm", "lt", "online"] if hasattr(CFG, key) } for key in previous: setattr(CFG, key, False) rows = [] crashes = [] try: for case in REGRESSION_CORPUS: try: rows.append(_process_case(case)) except Exception as exc: crashes.append({"ID":case["id"],"Domain":case["domain"],"Error":repr(exc)}) rows.append({ "ID":case["id"],"Domain":case["domain"],"Mode":case.get("mode","Balanced"), "Status":"TEST_CRASH","Changed":False,"Detected families":"", "Expected family hit":False,"Selected family":None, "Requested mode":None,"Accepted mode":None,"Attempt count":None, "Must-paraphrase outcome":None,"Must-paraphrase contract":False, "Generated Minimal":0,"Generated Balanced":0,"Generated Structural":0, "Before final filter":0,"After final filter":0,"Candidate traces":0, "Missing protected":[],"Realization issues":[repr(exc)], "Must remain unchanged":False,"Core pass":False, "Original":case["text"],"Revised":case["text"], "Reason":"Regression test crashed.","Rule errors":[],"Rejections":{}, "Routing evidence":[], }) safety_rows = [] for case in SAFETY_CASES: try: safety_rows.append(_safety_case(case)) except Exception as exc: safety_rows.append({ "ID":case["id"],"Test":case["name"],"Expected":case["expected"], "Accepted":None,"Failures":[repr(exc)],"Pass":False,"Checks":{}, }) finally: for key, value in previous.items(): setattr(CFG, key, value) regression_df = pd.DataFrame(rows) safety_df = pd.DataFrame(safety_rows) domain_summary = regression_df.groupby("Domain", dropna=False).agg( Sentences=("ID","count"), Successes=("Status",lambda values:int((values=="SUCCESS").sum())), Changed=("Changed","sum"), Core_passes=("Core pass","sum"), Family_hits=("Expected family hit","sum"), MP_contracts=("Must-paraphrase contract","sum"), Crashes=("Status",lambda values:int((values=="TEST_CRASH").sum())), ).reset_index() status_summary = ( regression_df.groupby(["Domain","Status"], dropna=False) .size().reset_index(name="Count") ) quality_failures = regression_df[ (~regression_df["Core pass"]) | regression_df["Realization issues"].map(bool) | regression_df["Missing protected"].map(bool) | regression_df["Rule errors"].map(bool) ].copy() changed_without_trace = regression_df[ regression_df["Changed"] & (regression_df["Candidate traces"] == 0) ].copy() summary = { "regression_sentences": int(len(regression_df)), "domains": int(regression_df["Domain"].nunique()), "successes": int((regression_df["Status"] == "SUCCESS").sum()), "changed": int(regression_df["Changed"].sum()), "core_passes": int(regression_df["Core pass"].sum()), "family_hits": int(regression_df["Expected family hit"].sum()), "must_paraphrase_contracts": int(regression_df["Must-paraphrase contract"].sum()), "crashes": int((regression_df["Status"] == "TEST_CRASH").sum()), "quality_failures": int(len(quality_failures)), "changed_without_trace": int(len(changed_without_trace)), "safety_tests": int(len(safety_df)), "safety_passes": int(safety_df["Pass"].sum()), "registered_rules": int(len(RULES)), } print("=" * 88) print("COMPREHENSIVE MULTIDOMAIN REGRESSION SUMMARY v2") print("=" * 88) print(json.dumps(summary, indent=2)) print("=" * 88) try: display(domain_summary) display(status_summary) display(safety_df[["ID","Test","Expected","Accepted","Failures","Pass"]]) if show_all: display(regression_df) if len(quality_failures): print("QUALITY FAILURES") display(quality_failures) if len(changed_without_trace): print("CHANGED OUTPUTS WITHOUT CANDIDATE TRACE") display(changed_without_trace) except NameError: print(domain_summary.to_string(index=False)) print(status_summary.to_string(index=False)) return { "summary":summary, "regression":regression_df, "domain_summary":domain_summary, "status_summary":status_summary, "safety":safety_df, "quality_failures":quality_failures, "changed_without_trace":changed_without_trace, "crashes":crashes, } # ===================================================================== # 5. EXPORT # ===================================================================== def export_comprehensive_regression(results, filename="nll_comprehensive_regression_v2.xlsx"): path = f"/content/{filename}" with pd.ExcelWriter(path, engine="openpyxl") as writer: results["regression"].to_excel(writer, sheet_name="Regression", index=False) results["domain_summary"].to_excel(writer, sheet_name="Domain Summary", index=False) results["status_summary"].to_excel(writer, sheet_name="Status Summary", index=False) results["safety"].to_excel(writer, sheet_name="Safety", index=False) results["quality_failures"].to_excel(writer, sheet_name="Quality Failures", index=False) print("Saved:", path) return path print("=" * 88) print("CELL 8A COMPREHENSIVE MULTIDOMAIN REGRESSION SUITE v2 LOADED") print("=" * 88) print("Aligned pipeline: 5 v4.1 -> 5A v3.4 -> 6 v5.1 -> 7 v4 -> 7A v2 -> 7B v2 -> 8 v4") print("Regression sentences:", len(REGRESSION_CORPUS)) print("Domains:", sorted({case['domain'] for case in REGRESSION_CORPUS})) print("Adversarial safety tests:", len(SAFETY_CASES)) print("Registered transformation rules:", len(RULES)) print("Must-paraphrase contract checks: active") print("Out-of-sample primary-clause cases: active") print("Run: RESULTS_8A = run_comprehensive_regression()") print("Optional export: export_comprehensive_regression(RESULTS_8A)") print("=" * 88)