| |
| |
| |
| |
| |
| |
|
|
| import json |
| import re |
| from collections import Counter |
|
|
| import pandas as pd |
|
|
| EXPECTED_8A_RULE_COUNT = 30 |
|
|
| _REQUIRED_8A = [ |
| "ENGINE", "Engine", "Analyzer", "Generator", "Validator", "Ranker", |
| "PROFILES", "RULES", "CFG", "clean", "surface_key", |
| "realization_issues", "expanded_regression", "safety_tests", |
| "relation_regression", |
| ] |
| _missing_8a = [name for name in _REQUIRED_8A if name not in globals()] |
| if _missing_8a: |
| raise RuntimeError( |
| "Cell 8A cannot run. Missing: " + ", ".join(_missing_8a) |
| + ". Run Cells 1–8 v4 in order first." |
| ) |
| if len(RULES) != EXPECTED_8A_RULE_COUNT: |
| raise RuntimeError( |
| f"Cell 8A expected {EXPECTED_8A_RULE_COUNT} rules but found {len(RULES)}." |
| ) |
| if getattr(Engine.one, "__name__", "") != "_one_with_must_paraphrase_v2": |
| raise RuntimeError("Cell 8A requires Cell 7B v2.") |
|
|
|
|
| |
| |
| |
|
|
| REGRESSION_CORPUS = [ |
| |
| {"id":"ACA-01","domain":"Academic","mode":"Balanced","text":"The next step is to examine how representation quality affects classifier robustness.","protected":"classifier robustness","expected_any":{"COPULAR_ACTION_FRAME"}}, |
| {"id":"ACA-02","domain":"Academic","mode":"Balanced","text":"These findings provide evidence that adversarial training improves robustness under the evaluated attack settings.","protected":"adversarial training, robustness","expected_any":{"EVIDENTIAL_COMPLEMENT"}}, |
| {"id":"ACA-03","domain":"Academic","mode":"Balanced","text":"The study requires consideration of whether the observed effect generalizes across datasets.","protected":"datasets","expected_any":{"WHETHER_COMPLEMENT"}}, |
| {"id":"ACA-04","domain":"Academic","mode":"Balanced","text":"Applying the same evaluation protocol across both platforms reduces methodological confounding.","protected":"evaluation protocol","expected_any":{"GERUND_SUBJECT"}}, |
| {"id":"ACA-05","domain":"Academic","mode":"Structural","text":"The framework preserves platform-specific constraints while applying a common evaluation procedure.","protected":"platform-specific constraints","expected_any":{"WHILE_PARTICIPIAL"}}, |
| {"id":"ACA-06","domain":"Academic","mode":"Balanced","text":"The purpose is not to identify a universally superior model but to compare robustness across feature spaces.","protected":"robustness, feature spaces","expected_any":{"NEGATED_PURPOSE_CONTRAST"}}, |
|
|
| |
| {"id":"TEC-01","domain":"Technical","mode":"Balanced","text":"The next step is to inspect how the deployed handler checks for GPT configuration.","protected":"GPT configuration","expected_any":{"COPULAR_ACTION_FRAME"}}, |
| {"id":"TEC-02","domain":"Technical","mode":"Balanced","text":"We need to distinguish those possibilities using the existing source.","protected":"existing source","expected_any":{"NEED_TO_ACTION","USING_METHOD_RELATION","METHOD_RELATION"}}, |
| {"id":"TEC-03","domain":"Technical","mode":"Balanced","text":"The fault could be in the environment-variable lookup, the initialization routine, or the error-reporting logic.","protected":"environment-variable lookup, initialization routine, error-reporting logic","expected_any":{"MODAL_COPULAR_ENUMERATION","ENUMERATION"}}, |
| {"id":"TEC-04","domain":"Technical","mode":"Balanced","text":"The service is evaluated using latency, throughput, and failure-rate measurements.","protected":"latency, throughput, failure-rate","expected_any":{"PASSIVE_USING","USING_METHOD_RELATION"}}, |
| {"id":"TEC-05","domain":"Technical","mode":"Structural","text":"The deployment was tested by the engineering team.","protected":"deployment, engineering team","expected_any":{"PASSIVE"}}, |
| {"id":"TEC-06","domain":"Technical","mode":"Balanced","text":"The process reduces configuration errors by validating required environment variables before startup.","protected":"environment variables","expected_any":{"BY_GERUND_METHOD_RELATION","METHOD_RELATION"}}, |
|
|
| |
| {"id":"BUS-01","domain":"Business","mode":"Balanced","text":"The next step is to review the supplier's revised implementation schedule.","protected":"implementation schedule","expected_any":{"COPULAR_ACTION_FRAME"}}, |
| {"id":"BUS-02","domain":"Business","mode":"Balanced","text":"The delay could be in procurement, vendor mobilization, or material delivery.","protected":"procurement, vendor mobilization, material delivery","expected_any":{"MODAL_COPULAR_ENUMERATION","ENUMERATION"}}, |
| {"id":"BUS-03","domain":"Business","mode":"Balanced","text":"We need to verify the invoice totals using the approved cost ledger.","protected":"invoice totals, approved cost ledger","expected_any":{"NEED_TO_ACTION","USING_METHOD_RELATION","METHOD_RELATION"}}, |
| {"id":"BUS-04","domain":"Business","mode":"Balanced","text":"The team intends to submit the report this week rather than waiting until month-end.","protected":"report, month-end","expected_any":{"FINITE_PURPOSE_INTENT","RATHER_THAN_GERUND"}}, |
| {"id":"BUS-05","domain":"Business","mode":"Structural","text":"The manager reviewed the estimate and approved the revised budget.","protected":"estimate, revised budget","expected_any":{"COORDINATED_PREDICATES"}}, |
| {"id":"BUS-06","domain":"Business","mode":"Structural","text":"The schedule identifies mobilization, procurement, installation, testing, and closeout; each stage has a separate milestone.","protected":"mobilization, procurement, installation, testing, closeout","expected_any":{"LONG_MULTI_CLAUSE","ENUMERATION"}}, |
|
|
| |
| {"id":"CON-01","domain":"Contract","mode":"Balanced","text":"For purposes of this plan, a Sub-RFR is an individual reimbursement request for eligible paid costs.","protected":"Sub-RFR, eligible paid costs","expected_any":{"FRONTED_ADJUNCT"},"must_remain_unchanged":True}, |
| {"id":"CON-02","domain":"Contract","mode":"Balanced","text":"To support reimbursement processing, the Contractor shall provide invoices, inspection reports, photographs, lien releases, and other supporting documentation.","protected":"Contractor, invoices, inspection reports, photographs, lien releases","expected_any":{"PURPOSE_INFINITIVE_FRONTING","ENUMERATION"}}, |
| {"id":"CON-03","domain":"Contract","mode":"Balanced","text":"Eligible invoices must be properly documented, reviewed, and approved by WSC.","protected":"Eligible invoices, WSC","expected_any":{"COORDINATED_PASSIVE_PARTICIPLES","COORDINATED_PREDICATES"}}, |
| {"id":"CON-04","domain":"Contract","mode":"Balanced","text":"WSC intends to submit reimbursement requests progressively rather than waiting until project completion.","protected":"WSC, reimbursement requests, project completion","expected_any":{"FINITE_PURPOSE_INTENT","RATHER_THAN_GERUND"}}, |
| {"id":"CON-05","domain":"Contract","mode":"Structural","text":"The work was inspected by the Owner's representative.","protected":"Owner's representative","expected_any":{"PASSIVE"}}, |
| {"id":"CON-06","domain":"Contract","mode":"Balanced","text":"Payments are based on approved work rather than on the passage of time.","protected":"approved work, passage of time","expected_any":{"PASSIVE","CONTRAST"}}, |
|
|
| |
| {"id":"GEN-01","domain":"General","mode":"Balanced","text":"General-purpose machine-learning systems often train separate models without explicit rules for competency assessment, evidence-grounded correction, and graduation.","protected":"competency assessment, evidence-grounded correction, graduation","expected_any":{"FINITE_COMPLEMENT_LIST"}}, |
| {"id":"GEN-02","domain":"General","mode":"Balanced","text":"We propose Framework X (FX), a domain-independent orchestration protocol consisting of contracts, examinations, peer study, and auditable promotion.","protected":"Framework X, FX, contracts, examinations, peer study, auditable promotion","expected_any":{"PROPOSAL_FRAME","APPOSITIVE_DEFINITION"}}, |
| {"id":"GEN-03","domain":"General","mode":"Balanced","text":"The experimental testbed applies this protocol to small cybersecurity learners spanning classification, generative drafting, and structured diagram planning.","protected":"small cybersecurity learners, classification, generative drafting, structured diagram planning","expected_any":{"PARTICIPIAL_POSTMODIFIER"}}, |
| {"id":"GEN-04","domain":"General","mode":"Balanced","text":"We pre-register comparable questions, disjoint data, matched baselines, ablations, and independently held-out testing.","protected":"comparable questions, disjoint data, matched baselines, ablations, independently held-out testing","expected_any":{"FINITE_OBJECT_LIST_FRAME"}}, |
| {"id":"GEN-05","domain":"General","mode":"Balanced","text":"Research claims about improvement, cost, generality, and deployment are contingent on execution and empirical analysis.","protected":"improvement, cost, generality, deployment, execution, empirical analysis","expected_any":{"CONTINGENCY_RELATION"}}, |
| ] |
|
|
|
|
| |
| |
| |
|
|
| def _terms(text): |
| return {part.strip() for part in re.split(r"[,;\n]", text or "") if part.strip()} |
|
|
|
|
| def _contains_term(text, term): |
| return re.search(re.escape(term), text, re.I) is not None |
|
|
|
|
| def _protected_missing(source, revised, protected_text): |
| return [ |
| term for term in _terms(protected_text) |
| if _contains_term(source, term) and not _contains_term(revised, term) |
| ] |
|
|
|
|
| def _candidate_trace_count(trace): |
| return len(trace.get("candidate_traces", []) or []) |
|
|
|
|
| def _realization_issues_safe(text): |
| try: |
| return list(realization_issues(text) or []) |
| except Exception as exc: |
| return [f"REALIZATION_CHECK_ERROR:{exc}"] |
|
|
|
|
| def _process_case(case): |
| profile, routing = ENGINE.route(case["text"], "Auto Detect") |
| result = ENGINE.one( |
| case["text"], |
| profile, |
| _terms(case.get("protected", "")), |
| case.get("mode", "Balanced"), |
| ) |
|
|
| trace = dict(getattr(result, "diagnostics", {}) or {}) |
| must = trace.get("must_paraphrase", {}) or {} |
| families = set(getattr(result, "families", []) or []) |
| revised = getattr(result, "revised", case["text"]) |
| status = getattr(result, "status", "UNKNOWN") |
| changed = surface_key(revised) != surface_key(case["text"]) |
| missing = _protected_missing(case["text"], revised, case.get("protected", "")) |
| realization = _realization_issues_safe(revised) |
| expected = set(case.get("expected_any", set())) |
| family_hit = not expected or bool(families & expected) |
| must_remain = bool(case.get("must_remain_unchanged", False)) |
|
|
| terminal_ok = status in { |
| "SUCCESS", "NO_PATTERN_COVERAGE", "CANDIDATE_GENERATION_FAILURE", |
| "TERMINOLOGY_REJECTION", "POLARITY_REJECTION", "MODALITY_REJECTION", |
| "QUANTITY_REJECTION", "ENUMERATION_REJECTION", "GRAMMAR_REJECTION", |
| "SEMANTIC_REJECTION", "INSUFFICIENT_REWRITE", "RANKING_EMPTY", |
| "PARSE_FAILURE", |
| } |
|
|
| |
| |
| |
| mp_outcome = must.get("outcome") |
| mp_contract = ( |
| (status == "SUCCESS" and changed and mp_outcome == "SAFE_CHANGED_OUTPUT") |
| or ( |
| status != "SUCCESS" |
| and not changed |
| and mp_outcome == "ORIGINAL_PRESERVED_AFTER_EXHAUSTIVE_SAFE_SEARCH" |
| ) |
| ) |
|
|
| pass_core = ( |
| terminal_ok |
| and not missing |
| and not realization |
| and (not must_remain or not changed) |
| and (family_hit or status == "NO_PATTERN_COVERAGE") |
| and mp_contract |
| ) |
|
|
| return { |
| "ID": case["id"], |
| "Domain": case["domain"], |
| "Mode": case.get("mode", "Balanced"), |
| "Status": status, |
| "Changed": changed, |
| "Detected families": " | ".join(sorted(families)), |
| "Expected family hit": family_hit, |
| "Selected family": getattr(result, "family", None), |
| "Requested mode": must.get("requested_mode"), |
| "Accepted mode": must.get("accepted_mode"), |
| "Attempt count": must.get("attempt_count"), |
| "Must-paraphrase outcome": mp_outcome, |
| "Must-paraphrase contract": mp_contract, |
| "Generated Minimal": (trace.get("generated_by_lane") or {}).get("Minimal", 0), |
| "Generated Balanced": (trace.get("generated_by_lane") or {}).get("Balanced", 0), |
| "Generated Structural": (trace.get("generated_by_lane") or {}).get("Structural", 0), |
| "Before final filter": trace.get("generated_before_final_filter"), |
| "After final filter": trace.get("generated_after_final_filter"), |
| "Candidate traces": _candidate_trace_count(trace), |
| "Missing protected": missing, |
| "Realization issues": realization, |
| "Must remain unchanged": must_remain, |
| "Core pass": pass_core, |
| "Original": case["text"], |
| "Revised": revised, |
| "Reason": getattr(result, "reason", ""), |
| "Rule errors": trace.get("rule_errors", []), |
| "Rejections": trace.get("rejections", {}), |
| "Routing evidence": routing, |
| } |
|
|
|
|
| |
| |
| |
|
|
| SAFETY_CASES = [ |
| {"id":"SAFE-01","name":"Subject/object reversal","source":"XGBoost outperformed DNN.","candidate":"DNN outperformed XGBoost.","expected":"SEMANTIC_REJECTION"}, |
| {"id":"SAFE-02","name":"Negation removal","source":"The model did not fail.","candidate":"The model failed.","expected":"POLARITY_REJECTION"}, |
| {"id":"SAFE-03","name":"Negation addition","source":"The model failed.","candidate":"The model did not fail.","expected":"POLARITY_REJECTION"}, |
| {"id":"SAFE-04","name":"Quantity change","source":"The study evaluated 26 attacks.","candidate":"The study evaluated 25 attacks.","expected":"QUANTITY_REJECTION"}, |
| {"id":"SAFE-05","name":"Modality strengthening","source":"The model may improve robustness.","candidate":"The model improves robustness.","expected":"MODALITY_REJECTION"}, |
| {"id":"SAFE-06","name":"Enumeration omission","source":"The system evaluated accuracy, precision, recall, and F1 score.","candidate":"The system evaluated accuracy, precision, and recall.","expected":"ENUMERATION_REJECTION"}, |
| {"id":"SAFE-07","name":"Protected terminology removal","source":"The XGBoost model was evaluated.","candidate":"The model was evaluated.","expected":"TERMINOLOGY_REJECTION","protected":{"XGBoost"}}, |
| {"id":"SAFE-08","name":"Citation removal","source":"Demetrio et al. (2021) evaluated PE-file manipulations.","candidate":"The study evaluated PE-file manipulations.","expected":"TERMINOLOGY_REJECTION"}, |
| ] |
|
|
|
|
| def _safety_case(case): |
| profile = PROFILES.get("General Academic", next(iter(PROFILES.values()))) |
| protected = set(case.get("protected", set())) |
| analysis = Analyzer().analyze(case["source"], profile, protected) |
| candidate = Candidate( |
| case["candidate"], "Balanced", "SAFETY_TEST", "Adversarial safety test", .5 |
| ) |
| accepted, failures, checks = Validator().validate( |
| case["source"], candidate, analysis, profile, protected, "Balanced" |
| ) |
| failures = list(failures or []) |
| return { |
| "ID": case["id"], |
| "Test": case["name"], |
| "Expected": case["expected"], |
| "Accepted": bool(accepted), |
| "Failures": failures, |
| "Pass": case["expected"] in failures, |
| "Checks": checks, |
| } |
|
|
|
|
| |
| |
| |
|
|
|
|
| def run_comprehensive_regression(show_all=True): |
| |
| previous = { |
| key: getattr(CFG, key, None) |
| for key in ["sts", "nli", "mlm", "lt", "online"] |
| if hasattr(CFG, key) |
| } |
| for key in previous: |
| setattr(CFG, key, False) |
|
|
| rows = [] |
| crashes = [] |
| try: |
| for case in REGRESSION_CORPUS: |
| try: |
| rows.append(_process_case(case)) |
| except Exception as exc: |
| crashes.append({"ID":case["id"],"Domain":case["domain"],"Error":repr(exc)}) |
| rows.append({ |
| "ID":case["id"],"Domain":case["domain"],"Mode":case.get("mode","Balanced"), |
| "Status":"TEST_CRASH","Changed":False,"Detected families":"", |
| "Expected family hit":False,"Selected family":None, |
| "Requested mode":None,"Accepted mode":None,"Attempt count":None, |
| "Must-paraphrase outcome":None,"Must-paraphrase contract":False, |
| "Generated Minimal":0,"Generated Balanced":0,"Generated Structural":0, |
| "Before final filter":0,"After final filter":0,"Candidate traces":0, |
| "Missing protected":[],"Realization issues":[repr(exc)], |
| "Must remain unchanged":False,"Core pass":False, |
| "Original":case["text"],"Revised":case["text"], |
| "Reason":"Regression test crashed.","Rule errors":[],"Rejections":{}, |
| "Routing evidence":[], |
| }) |
|
|
| safety_rows = [] |
| for case in SAFETY_CASES: |
| try: |
| safety_rows.append(_safety_case(case)) |
| except Exception as exc: |
| safety_rows.append({ |
| "ID":case["id"],"Test":case["name"],"Expected":case["expected"], |
| "Accepted":None,"Failures":[repr(exc)],"Pass":False,"Checks":{}, |
| }) |
| finally: |
| for key, value in previous.items(): |
| setattr(CFG, key, value) |
|
|
| regression_df = pd.DataFrame(rows) |
| safety_df = pd.DataFrame(safety_rows) |
|
|
| domain_summary = regression_df.groupby("Domain", dropna=False).agg( |
| Sentences=("ID","count"), |
| Successes=("Status",lambda values:int((values=="SUCCESS").sum())), |
| Changed=("Changed","sum"), |
| Core_passes=("Core pass","sum"), |
| Family_hits=("Expected family hit","sum"), |
| MP_contracts=("Must-paraphrase contract","sum"), |
| Crashes=("Status",lambda values:int((values=="TEST_CRASH").sum())), |
| ).reset_index() |
|
|
| status_summary = ( |
| regression_df.groupby(["Domain","Status"], dropna=False) |
| .size().reset_index(name="Count") |
| ) |
|
|
| quality_failures = regression_df[ |
| (~regression_df["Core pass"]) |
| | regression_df["Realization issues"].map(bool) |
| | regression_df["Missing protected"].map(bool) |
| | regression_df["Rule errors"].map(bool) |
| ].copy() |
|
|
| changed_without_trace = regression_df[ |
| regression_df["Changed"] & (regression_df["Candidate traces"] == 0) |
| ].copy() |
|
|
| summary = { |
| "regression_sentences": int(len(regression_df)), |
| "domains": int(regression_df["Domain"].nunique()), |
| "successes": int((regression_df["Status"] == "SUCCESS").sum()), |
| "changed": int(regression_df["Changed"].sum()), |
| "core_passes": int(regression_df["Core pass"].sum()), |
| "family_hits": int(regression_df["Expected family hit"].sum()), |
| "must_paraphrase_contracts": int(regression_df["Must-paraphrase contract"].sum()), |
| "crashes": int((regression_df["Status"] == "TEST_CRASH").sum()), |
| "quality_failures": int(len(quality_failures)), |
| "changed_without_trace": int(len(changed_without_trace)), |
| "safety_tests": int(len(safety_df)), |
| "safety_passes": int(safety_df["Pass"].sum()), |
| "registered_rules": int(len(RULES)), |
| } |
|
|
| print("=" * 88) |
| print("COMPREHENSIVE MULTIDOMAIN REGRESSION SUMMARY v2") |
| print("=" * 88) |
| print(json.dumps(summary, indent=2)) |
| print("=" * 88) |
|
|
| try: |
| display(domain_summary) |
| display(status_summary) |
| display(safety_df[["ID","Test","Expected","Accepted","Failures","Pass"]]) |
| if show_all: |
| display(regression_df) |
| if len(quality_failures): |
| print("QUALITY FAILURES") |
| display(quality_failures) |
| if len(changed_without_trace): |
| print("CHANGED OUTPUTS WITHOUT CANDIDATE TRACE") |
| display(changed_without_trace) |
| except NameError: |
| print(domain_summary.to_string(index=False)) |
| print(status_summary.to_string(index=False)) |
|
|
| return { |
| "summary":summary, |
| "regression":regression_df, |
| "domain_summary":domain_summary, |
| "status_summary":status_summary, |
| "safety":safety_df, |
| "quality_failures":quality_failures, |
| "changed_without_trace":changed_without_trace, |
| "crashes":crashes, |
| } |
|
|
|
|
| |
| |
| |
|
|
|
|
| def export_comprehensive_regression(results, filename="nll_comprehensive_regression_v2.xlsx"): |
| path = f"/content/{filename}" |
| with pd.ExcelWriter(path, engine="openpyxl") as writer: |
| results["regression"].to_excel(writer, sheet_name="Regression", index=False) |
| results["domain_summary"].to_excel(writer, sheet_name="Domain Summary", index=False) |
| results["status_summary"].to_excel(writer, sheet_name="Status Summary", index=False) |
| results["safety"].to_excel(writer, sheet_name="Safety", index=False) |
| results["quality_failures"].to_excel(writer, sheet_name="Quality Failures", index=False) |
| print("Saved:", path) |
| return path |
|
|
|
|
| print("=" * 88) |
| print("CELL 8A COMPREHENSIVE MULTIDOMAIN REGRESSION SUITE v2 LOADED") |
| print("=" * 88) |
| print("Aligned pipeline: 5 v4.1 -> 5A v3.4 -> 6 v5.1 -> 7 v4 -> 7A v2 -> 7B v2 -> 8 v4") |
| print("Regression sentences:", len(REGRESSION_CORPUS)) |
| print("Domains:", sorted({case['domain'] for case in REGRESSION_CORPUS})) |
| print("Adversarial safety tests:", len(SAFETY_CASES)) |
| print("Registered transformation rules:", len(RULES)) |
| print("Must-paraphrase contract checks: active") |
| print("Out-of-sample primary-clause cases: active") |
| print("Run: RESULTS_8A = run_comprehensive_regression()") |
| print("Optional export: export_comprehensive_regression(RESULTS_8A)") |
| print("=" * 88) |
|
|
