| """NLL V7.2 serialized engine source.""" |
|
|
|
|
| |
| |
| from __future__ import annotations |
| import os,re,json,html,time,traceback,statistics |
| from pathlib import Path |
| import uuid |
| from dataclasses import dataclass,field,asdict |
| from collections import Counter |
| from difflib import SequenceMatcher |
| import numpy as np,pandas as pd,spacy |
| from nltk.corpus import wordnet as wn |
| from lemminflect import getInflection |
| DISCLAIMER="Observable style-pattern index only; not proof/probability of AI authorship and no detector-bypass guarantee." |
| CODES={"NO_PATTERN_COVERAGE","PARSE_FAILURE","CANDIDATE_GENERATION_FAILURE","REALIZATION_FAILURE","SEMANTIC_REJECTION","TERMINOLOGY_REJECTION","POLARITY_REJECTION","MODALITY_REJECTION","QUANTITY_REJECTION","ENUMERATION_REJECTION","GRAMMAR_REJECTION","RANKING_EMPTY","INSUFFICIENT_REWRITE","SUCCESS","NOT_TARGETED"} |
| @dataclass |
| class Config: |
| stanza:bool=False;sts:bool=True;nli:bool=True;mlm:bool=False;lt:bool=False;online:bool=False;batch:int=20;near_copy:float=.965;min_sts:float=.72;min_nli:float=.58;out:str="/content/nll_outputs";cache:str="/content/nll_cache" |
| @dataclass |
| class Profile:name:str;keywords:set[str];phrases:set[str];immutable:set[str]=field(default_factory=set);concepts:set[str]=field(default_factory=set) |
| @dataclass |
| class Prop:predicate:str;subject:str="";objects:list=field(default_factory=list);complements:list=field(default_factory=list);polarity:str="positive";modals:list=field(default_factory=list);voice:str="active" |
| @dataclass |
| class Analysis:text:str;families:list;props:list;relations:list;enumerations:list;protected:list;notes:list=field(default_factory=list);constituency:str|None=None |
| @dataclass |
| class Finding:category:str;passage:str;start:int;end:int;sentence:int;severity:float;explanation:str |
| @dataclass |
| class Candidate:text:str;lane:str;family:str;reason:str;priority:float=.5;checks:dict=field(default_factory=dict);score:float=-999 |
| @dataclass |
| class SentenceResult:original:str;revised:str;status:str;families:list;family:str|None;reason:str;alternatives:list;validation:dict;diagnostics:dict |
| @dataclass |
| class DocumentResult:original:str;revised:str;workflow:str;mode:str;discipline:str;routing:list;pattern_score:float;band:str;findings:list;sentences:list;warnings:list;runtime:dict |
| CFG=Config();CFG.out='/tmp/nll_outputs';CFG.cache='/tmp/nll_cache';MODELS={};STATE={};Path(CFG.out).mkdir(parents=True,exist_ok=True);Path(CFG.cache).mkdir(parents=True,exist_ok=True) |
| def S(x):return {z.strip() for z in x.split("|") if z.strip()} |
| PROFILES={ |
| "General Academic":Profile("General Academic",S("study|research|analysis|result|method|evidence|sample|finding"),S("this study|the findings")), |
| "Computing / Cybersecurity":Profile("Computing / Cybersecurity",S("algorithm|classifier|malware|adversarial|dataset|network|feature|robustness|encryption|cloud"),S("machine learning|feature space|problem space|threat model|static analysis"),S("XGBoost|Linear SVM|DNN|Windows PE|Android|APK|API"),S("adversarial robustness|perturbation constraint")), |
| "Engineering":Profile("Engineering",S("design|system|load|stress|material|control|circuit|failure|efficiency"),S("finite element|control system"),S("CAD|FEA|HVAC")), |
| "Medicine / Health":Profile("Medicine / Health",S("patient|clinical|treatment|diagnosis|disease|therapy|trial|cohort|outcome"),S("randomized controlled trial|adverse event"),S("DNA|RNA|MRI|CT|ICD")), |
| "Biology":Profile("Biology",S("cell|gene|protein|organism|species|genome|enzyme|tissue|mutation"),S("gene expression|cell signaling"),S("DNA|RNA|PCR")), |
| "Chemistry":Profile("Chemistry",S("compound|reaction|molecule|catalyst|solvent|synthesis|concentration|ion"),S("reaction rate|chemical equilibrium"),S("pH|NMR|HPLC")), |
| "Physics":Profile("Physics",S("energy|force|particle|field|wave|quantum|velocity|mass|gravity"),S("boundary condition|quantum state")), |
| "Mathematics / Statistics":Profile("Mathematics / Statistics",S("theorem|proof|equation|distribution|variance|regression|probability|estimator|matrix"),S("null hypothesis|confidence interval"),S("ANOVA|OLS|MSE")), |
| "Social Sciences":Profile("Social Sciences",S("society|community|policy|culture|institution|inequality|survey|interview|qualitative"),S("social context|thematic analysis")), |
| "Psychology":Profile("Psychology",S("behavior|cognition|emotion|participant|memory|attention|motivation|construct"),S("cognitive load|self efficacy"),S("DSM")), |
| "Education":Profile("Education",S("student|teacher|learning|curriculum|instruction|classroom|assessment|school"),S("learning outcome|instructional strategy")), |
| "Business / Economics":Profile("Business / Economics",S("market|firm|revenue|cost|consumer|economy|finance|investment|management"),S("return on investment|market share"),S("ROI|GDP")), |
| "Law / Policy":Profile("Law / Policy",S("law|legal|court|statute|regulation|policy|rights|liability|compliance"),S("case law|due process"),S("FAR|CFR")), |
| "Humanities":Profile("Humanities",S("text|narrative|history|philosophy|literature|discourse|interpretation|rhetoric"),S("historical context|critical theory")), |
| "Environmental Science":Profile("Environmental Science",S("climate|ecosystem|pollution|emission|biodiversity|habitat|sustainability|carbon"),S("climate change|greenhouse gas"),S("CO2|LCA"))} |
| def nlp(): |
| if "spacy" not in MODELS: |
| try:MODELS["spacy"]=spacy.load("en_core_web_sm") |
| except Exception:MODELS["spacy"]=spacy.blank("en");MODELS["spacy"].add_pipe("sentencizer") |
| return MODELS["spacy"] |
| def optional(kind): |
| if kind=="sts" and CFG.sts: |
| try: |
| if kind not in MODELS: |
| from sentence_transformers import SentenceTransformer |
| MODELS[kind]=SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2") |
| except Exception as e:STATE["warnings"].append(f"STS unavailable: {e}");MODELS[kind]=None |
| if kind=="nli" and CFG.nli: |
| try: |
| if kind not in MODELS: |
| from transformers import AutoTokenizer,AutoModelForSequenceClassification |
| mid="cross-encoder/nli-deberta-v3-small";m=AutoModelForSequenceClassification.from_pretrained(mid);m.eval();MODELS[kind]=(AutoTokenizer.from_pretrained(mid),m) |
| except Exception as e:STATE["warnings"].append(f"NLI unavailable: {e}");MODELS[kind]=None |
| if kind=="mlm" and CFG.mlm: |
| try: |
| if kind not in MODELS: |
| import torch |
| from transformers import pipeline |
| MODELS[kind]=pipeline("fill-mask",model="distilroberta-base",device=0 if torch.cuda.is_available() else -1) |
| except Exception as e:STATE["warnings"].append(f"MLM unavailable: {e}");MODELS[kind]=None |
| if kind=="lt" and CFG.lt: |
| try: |
| if kind not in MODELS: |
| import language_tool_python |
| MODELS[kind]=language_tool_python.LanguageTool("en-US") |
| except Exception as e:STATE["warnings"].append(f"LanguageTool unavailable: {e}");MODELS[kind]=None |
| return MODELS.get(kind) |
| PATS={ |
| "citation":re.compile(r"\([^()\n]{0,90}(?:19|20)\d{2}[a-z]?[^()\n]{0,50}\)|\[[0-9,;\-\s]+\]|[A-Z][A-Za-z'\-]+ et al\. \((?:19|20)\d{2}[a-z]?\)"), |
| "url":re.compile(r"https?://\S+|doi:\s*\S+|10\.\d{4,9}/\S+",re.I), |
| "equation":re.compile(r"\$[^$]+\$"), |
| "code":re.compile(r"`[^`]+`|```[\s\S]+?```"), |
| "quantity":re.compile(r"(?<!\w)[<>≈~±]?\s*\d+(?:\.\d+)?(?:\s*[-–]\s*\d+(?:\.\d+)?)?\s*(?:%|mg|kg|mL|mm|cm|km|Hz|GHz|°C|MPa|kW|days?|years?)?\b",re.I) |
| } |
| def protect(text,terms): |
| out=[] |
| for k,p in PATS.items():out += [(k,m.group(),m.start(),m.end()) for m in p.finditer(text)] |
| for t in terms:out += [("term",m.group(),m.start(),m.end()) for m in re.finditer(r"(?<!\w)"+re.escape(t)+r"(?!\w)",text,re.I)] |
| return sorted(set(out),key=lambda x:(x[2],x[3],x[0])) |
| def sent_split(text):return [s.text.strip() for s in nlp()(text).sents if s.text.strip()] or [text.strip()] |
| def read_document(path): |
| if not path:return "",{"source":"textbox"} |
| p=Path(path);e=p.suffix.lower() |
| if e==".txt":return p.read_text(encoding="utf8",errors="replace"),{"type":"txt"} |
| if e==".docx": |
| from docx import Document |
| d=Document(str(p));return "\n\n".join(x.text for x in d.paragraphs if x.text.strip()),{"type":"docx"} |
| if e==".pdf": |
| import pymupdf |
| d=pymupdf.open(str(p));return "\n\n".join(x.get_text("text") for x in d),{"type":"pdf","pages":len(d)} |
| raise ValueError("Use TXT, DOCX, or text-based PDF.") |
|
|
|
|
| |
| |
| MODALS={"may","might","can","could","must","should","would","will","shall"} |
| def sub(t): |
| x=list(t.subtree);return t.doc[x[0].i:x[-1].i+1].text |
| class Analyzer: |
| def analyze(self,s,p,u): |
| d=nlp()(s);terms=p.immutable|p.concepts|u |
| if not d.has_annotation("DEP"):return Analysis(s,["PARSE_FAILURE"],[],[],[],protect(s,terms),["dependency parse unavailable"]) |
| fam=set();props=[];rels=[];enums=[];preds=[] |
| for r in [t for t in d if t.dep_=="ROOT"]:preds += [r]+[x for x in r.conjuncts if x.pos_ in {"VERB","AUX"}] |
| for v in preds: |
| ss=[x for x in v.children if x.dep_ in {"nsubj","nsubjpass","csubj","csubjpass"}] |
| if not ss and v.dep_=="conj":ss=[x for x in v.head.children if x.dep_ in {"nsubj","nsubjpass","csubj"}] |
| oo=[x for x in v.children if x.dep_ in {"obj","dobj","attr","oprd","dative"}];cc=[x for x in v.children if x.dep_ in {"ccomp","xcomp","advcl","acl","relcl"}];aux=[x.lemma_.lower() for x in v.children if x.dep_ in {"aux","auxpass"}] |
| props.append(Prop(v.lemma_.lower(),sub(ss[0]) if ss else "",[sub(x) for x in oo],[sub(x) for x in cc],"negative" if any(x.dep_=="neg" for x in v.children) else "positive",[x for x in aux if x in MODALS],"passive" if any(x.dep_ in {"nsubjpass","auxpass"} for x in v.children) else "active")) |
| lo=s.lower();tests={"COORDINATED_PREDICATES":any(t.dep_=="conj" and t.pos_=="VERB" for t in d),"COORDINATED_OBJECTS_OR_ENUMERATION":any(t.dep_=="conj" and t.pos_ in {"NOUN","PROPN"} for t in d),"NEGATED_PURPOSE_CONTRAST":bool(re.search(r"(?:purpose|aim|objective|goal).*is\s+not\s+to.+but\s+to",lo)),"EVIDENTIAL_COMPLEMENT":bool(re.search(r"(?:provide|provides|provided|offer|offers)\s+(?:clear\s+)?evidence\s+that",lo)),"WHETHER_COMPLEMENT":"whether" in lo,"WHEREBY_CLAUSE":"whereby" in lo,"PURPOSE_RESULT_CLAUSE":"so that" in lo,"WHILE_PARTICIPIAL":bool(re.search(r"while\s+\w+ing",lo)),"PURPOSE_INFINITIVE":bool(re.search(r"(?:intended|designed|used)\s+to",lo)),"REDUCED_RELATIVE_OR_NP_MODIFIER":bool(re.search(r"(?:obtained|derived|collected|trained|based|intended)\s+(?:from|on|to)",lo)),"FRONTED_ADJUNCT":bool(re.match(r"^(?:for|in|within|under|during|among|across)[^,]{1,80},",lo)),"WHEN_CLAUSE":bool(re.search(r"when",lo)),"PARENTHETICAL_EMPHASIS":bool(re.search(r"by (?:itself|themselves)|alone",lo)),"NOMINALIZATION":any(t.pos_=="NOUN" and t.suffix_.lower() in {"tion","sion","ment","ance","ence","ity"} for t in d),"METHOD_RELATION":bool(re.search(r"through.+(?:ing|tion|sion|ment)",lo)),"PASSIVE":any(x.voice=="passive" for x in props)} |
| if d and d[0].tag_=="VBG":fam|={"GERUND_SUBJECT","PARTICIPIAL_FRONTING"} |
| fam|={k for k,v in tests.items() if v};markers={"cause":{"because","since","therefore"},"purpose":{"so that","intended to"},"contrast":{"but","although","whereas","while"},"condition":{"if","unless","whether"},"temporal":{"when","while","before","after"}} |
| for typ,ms in markers.items(): |
| for x in ms: |
| if re.search(r"(?<!\w)"+re.escape(x)+r"(?!\w)",lo):rels.append({"type":typ,"marker":x}) |
| seen=set() |
| for t in d: |
| xs=[t]+list(t.conjuncts) |
| if len(xs)>=3 and t.i not in seen:enums.append([sub(x) for x in xs]);seen|={x.i for x in xs} |
| const=None |
| if CFG.stanza: |
| try: |
| if "stanza" not in MODELS: |
| import stanza |
| MODELS["stanza"]=stanza.Pipeline("en",processors="tokenize,pos,lemma,depparse,constituency",use_gpu=False,verbose=False) |
| const=str(MODELS["stanza"](s).sentences[0].constituency) |
| except Exception as e:STATE["warnings"].append(f"Stanza unavailable: {e}") |
| return Analysis(s,sorted(fam),props,rels,enums,protect(s,terms),[],const) |
| FORMULAIC=[r"it is important to note that",r"in today's rapidly evolving",r"delve(?:s|d)? into",r"multifaceted",r"pivotal role",r"seamlessly",r"in conclusion",r"furthermore",r"moreover",r"this highlights"] |
| class Scanner: |
| def scan(self,text): |
| ss=sent_split(text);fs=[];off=0;lens=[];starts=[] |
| for i,s in enumerate(ss): |
| st=max(off,text.find(s,off));off=st+len(s);d=nlp()(s);w=[t for t in d if t.is_alpha];lens.append(len(w));starts.append(" ".join(t.lemma_.lower() for t in w[:3])) |
| for p in FORMULAIC: |
| for z in re.finditer(p,s,re.I):fs.append(Finding("formulaic wording",z.group(),st+z.start(),st+z.end(),i,.65,"Formulaic academic wording; context may still justify it.")) |
| if len(w)>38:fs.append(Finding("sentence complexity",s,st,st+len(s),i,.6,"Long multi-clause sentence may be dense.")) |
| rep={x for x,n in Counter(starts).items() if x and n>=3} |
| for i,(s,o) in enumerate(zip(ss,starts)): |
| if o in rep:fs.append(Finding("repetitive opening",s[:80],text.find(s),text.find(s)+min(80,len(s)),i,.55,"Repeated sentence opening.")) |
| mean=statistics.mean(lens) if lens else 0;cv=statistics.pstdev(lens)/mean if len(lens)>1 and mean else 0;score=min(100,len(fs)*7+max(0,.28-cv)*40);return fs,float(score),"Low" if score<30 else "Moderate" if score<60 else "High",{"sentence_length_cv":cv,"findings":len(fs)} |
|
|
|
|
| |
| |
|
|
| from dataclasses import dataclass, field |
| from collections import Counter |
| import re |
|
|
| |
| |
| |
|
|
| PATTERN_TAXONOMY = { |
| "VOICE": { |
| "PASSIVE", "AGENTLESS_PASSIVE", "AGENTIVE_PASSIVE", |
| "MODAL_PASSIVE", "STACKED_PASSIVE", |
| "PASSIVE_EVALUATION_CONTEXT", "PASSIVE_PROCESS_CONTEXT", |
| "NEGATED_CLASSIFICATION_PASSIVE", |
| "COORDINATED_PASSIVE_PREDICATES", "PASSIVE_BASIS_FRAME", |
| }, |
| "CLAUSE_STRUCTURE": { |
| "WHEREBY_CLAUSE", "SO_THAT_CLAUSE", "PURPOSE_RESULT_CLAUSE", |
| "FINITE_WHILE_CLAUSE", "WHILE_PARTICIPIAL", |
| "PARTICIPIAL_RESULT_CLAUSE", "WHEN_CLAUSE", "WHEREAS_CLAUSE", |
| "ALTHOUGH_CLAUSE", "BECAUSE_CLAUSE", "IF_CLAUSE", |
| "CONTENT_COMPLEMENT", "RELATIVE_CLAUSE", |
| "MULTI_EMBEDDED_CLAUSES", "CONTRASTIVE_COORDINATED_CLAUSES", |
| "SEMICOLON_NEGATED_ASSUMPTION", |
| }, |
| "NOMINALIZATION": { |
| "GERUND_SUBJECT", "NOMINALIZATION", "ABSTRACT_NOUN_STACK", |
| "NOUN_HEAVY_PREDICATE", "NOMINALIZED_PROCESS", |
| "NOMINALIZED_EVALUATION", |
| }, |
| "CONTRAST": { |
| "CONTRAST", "NEGATED_PURPOSE_CONTRAST", "NOT_X_BUT_Y", |
| "RATHER_THAN_CONTRAST", "HOWEVER_CONTRAST", "WHEREAS_CONTRAST", |
| "CONTRASTIVE_COORDINATED_CLAUSES", |
| "SEMICOLON_NEGATED_ASSUMPTION", |
| }, |
| "REPORTING_EVIDENCE": { |
| "REPORTING_FRAME", "EVIDENTIAL_COMPLEMENT", "ATTRIBUTION_FRAME", |
| "RESULT_REPORTING_FRAME", "INDEPENDENT_REPORTING", |
| }, |
| "EVALUATION_METHOD": { |
| "EVALUATION_FRAME", "PASSIVE_EVALUATION_CONTEXT", |
| "PASSIVE_PROCESS_CONTEXT", "PASSIVE_BASIS_FRAME", |
| "METHOD_RELATION", "COMPARATIVE_FRAME", "METHOD_APPLICATION_FRAME", |
| "EVALUATION_CONDITION_FRAME", |
| }, |
| "COORDINATION": { |
| "COORDINATED_PREDICATES", "COORDINATED_OBJECTS_OR_ENUMERATION", |
| "COORDINATED_PROPOSITIONS", "LONG_ENUMERATION", |
| "PARALLEL_ENUMERATION", "COORDINATED_PASSIVE_PREDICATES", |
| "CONTRASTIVE_COORDINATED_CLAUSES", |
| }, |
| "DISCOURSE": { |
| "DISCOURSE_MARKER", "CONSEQUENCE", "RESULT_CLAUSE", |
| "REPETITIVE_TRANSITION", "FORMULAIC_TRANSITION", |
| "OVEREXPLICIT_LOGIC", "SEMICOLON_NEGATED_ASSUMPTION", |
| }, |
| "COMPLEXITY": { |
| "LONG_SENTENCE", "VERY_LONG_SENTENCE", "MULTI_EMBEDDED_CLAUSES", |
| "CLAUSE_OVERLOADING", "LONG_PREPOSITIONAL_CHAIN", |
| "HEAVY_POSTMODIFICATION", "EXCESSIVE_COORDINATION", |
| }, |
| "FORMULAIC_ACADEMIC": { |
| "FORMULAIC_PURPOSE", "FORMULAIC_RESULT", "FORMULAIC_COMPARISON", |
| "FORMULAIC_EVALUATION", "FORMULAIC_METHOD", |
| "FORMULAIC_CONCLUSION", "FORMULAIC_TRANSITION", |
| "OVEREXPLICIT_LOGIC", "REPETITIVE_SENTENCE_FRAME", |
| }, |
| "NATURALNESS": { |
| "AWKWARD_PASSIVE", "UNNATURAL_NOMINALIZATION", |
| "EXCESSIVE_SYMMETRY", "CLAUSE_OVERLOADING", "REPETITIVE_SYNTAX", |
| "REDUNDANT_EXPLICITNESS", "STILTED_CONNECTIVE", |
| "HEAVY_NOUN_PHRASE", "ABSTRACT_NOUN_STACK", |
| "MECHANICAL_PARALLELISM", |
| }, |
| } |
|
|
| |
| |
| |
|
|
| @dataclass |
| class PatternProfile: |
| grammar_families: set = field(default_factory=set) |
| formulaic_flags: set = field(default_factory=set) |
| naturalness_flags: set = field(default_factory=set) |
| complexity_flags: set = field(default_factory=set) |
| discourse_flags: set = field(default_factory=set) |
| metrics: dict = field(default_factory=dict) |
| evidence: dict = field(default_factory=dict) |
|
|
| |
| |
| |
|
|
| def _pt_clean(text): |
| return re.sub(r"\s+", " ", str(text or "")).strip() |
|
|
|
|
| def _pt_words(text): |
| return re.findall(r"\b[\w'-]+\b", str(text or "")) |
|
|
|
|
| def _add_evidence(profile, flag, evidence): |
| profile.evidence.setdefault(flag, []) |
| if evidence not in profile.evidence[flag]: |
| profile.evidence[flag].append(evidence) |
|
|
|
|
| def _detect_content_complement(lower_text): |
| if re.search(r"\bwhether\b", lower_text): |
| return True |
|
|
| determiner = re.search( |
| r"\bthat\s+(?:type|kind|form|method|model|approach|system|procedure|" |
| r"framework|process|design|strategy|technique|case|example|result|" |
| r"study|analysis|experiment|feature|representation|classifier|" |
| r"algorithm|dataset|environment|platform|component|domain|alignment|" |
| r"space|constraint|condition|measure|attack|sample|architecture|" |
| r"methodology)\b", |
| lower_text, |
| ) |
| if determiner: |
| return False |
|
|
| governor = re.search( |
| r"\b(?:show|shows|showed|indicate|indicates|indicated|suggest|" |
| r"suggests|suggested|demonstrate|demonstrates|demonstrated|report|" |
| r"reports|reported|find|finds|found|observe|observes|observed|confirm|" |
| r"confirms|confirmed|conclude|concludes|concluded|establish|" |
| r"establishes|established|assume|assumes|assumed|ensure|ensures|" |
| r"ensured|mean|means|meant|reveal|reveals|revealed|evidence|finding|" |
| r"findings|result|results)\s+that\b", |
| lower_text, |
| ) |
| if governor: |
| return True |
|
|
| clause = re.search( |
| r"\bthat\s+(?:the|a|an|this|these|those|it|they|we|researchers|" |
| r"participants|results|findings|evidence|data|models?|systems?|" |
| r"methods?|samples?|features?)\s+(?:is|are|was|were|has|have|had|" |
| r"can|could|may|might|must|should|will|would|does|do|did|" |
| r"[A-Za-z][A-Za-z-]*(?:s|ed))\b", |
| lower_text, |
| ) |
| return bool(clause) |
|
|
| |
| |
| |
|
|
| def detect_grammar_families(text): |
| text = _pt_clean(text) |
| lower_text = text.lower() |
| families = set() |
|
|
| if re.search(r"\bwhereby\b", lower_text): |
| families.add("WHEREBY_CLAUSE") |
|
|
| if re.search(r"\bso\s+that\b", lower_text): |
| families.update({"SO_THAT_CLAUSE", "PURPOSE_RESULT_CLAUSE"}) |
|
|
| if re.search( |
| r"\bwhile\s+(?:the\s+)?[A-Za-z][A-Za-z0-9_/-]*" |
| r"(?:\s+[A-Za-z][A-Za-z0-9_/-]*){0,5}\s+" |
| r"(?:is|are|was|were|has|have|had|does|do|did|tracks?|measures?|" |
| r"shows?|indicates?|preserves?|provides?|uses?|evaluates?|compares?|" |
| r"examines?|remains?|increases?|decreases?|continues?|represents?|" |
| r"performs?|captures?|retains?|allows?|enables?)\b", |
| lower_text, |
| ): |
| families.add("FINITE_WHILE_CLAUSE") |
|
|
| if re.search(r"\bwhile\s+[A-Za-z][A-Za-z-]*ing\b", lower_text): |
| families.add("WHILE_PARTICIPIAL") |
|
|
| if re.search(r",\s*(?:allowing|enabling|permitting)\b", lower_text): |
| families.add("PARTICIPIAL_RESULT_CLAUSE") |
|
|
| for family, pattern in { |
| "WHEN_CLAUSE": r"\bwhen\b", |
| "WHEREAS_CLAUSE": r"\bwhereas\b", |
| "ALTHOUGH_CLAUSE": r"\balthough\b", |
| "BECAUSE_CLAUSE": r"\bbecause\b", |
| "IF_CLAUSE": r"\bif\b", |
| }.items(): |
| if re.search(pattern, lower_text): |
| families.add(family) |
|
|
| if _detect_content_complement(lower_text): |
| families.add("CONTENT_COMPLEMENT") |
|
|
| if re.search(r"\bnot\b.+?\bbut\b", lower_text, re.S): |
| families.add("CONTRAST") |
|
|
| if re.search( |
| r"\b(?:purpose|aim|objective|goal)\b.+?\bis\s+not\s+to\b.+?\bbut\s+to\b", |
| lower_text, |
| re.S, |
| ): |
| families.add("NEGATED_PURPOSE_CONTRAST") |
|
|
| if re.match(r"^[A-Za-z][A-Za-z-]*ing\b", text, re.I): |
| families.add("GERUND_SUBJECT") |
|
|
| if re.search( |
| r"\b(?:is|are|was|were|has been|have been|had been)\s+(?:not\s+)?" |
| r"(?:\w+\s+){0,2}\w+(?:ed|en)\b", |
| lower_text, |
| ): |
| families.add("PASSIVE") |
|
|
| if re.search(r"\b(?:can|could|may|might|must|should)\s+be\s+\w+(?:ed|en)\b", lower_text): |
| families.add("MODAL_PASSIVE") |
|
|
| if re.search( |
| r"\b(?:is|are|was|were|has been|have been)\s+evaluated\s+" |
| r"(?:at|across|under|within|on|by|with|using)\b", |
| lower_text, |
| ): |
| families.add("PASSIVE_EVALUATION_CONTEXT") |
|
|
| if re.search( |
| r"\b(?:is|are|was|were|has been|have been|had been)\s+" |
| r"[A-Za-z][A-Za-z-]*(?:ed|en)(?:\s+(?:independently|separately|" |
| r"jointly|simultaneously|iteratively|individually|concurrently))?\s+" |
| r"(?:on|in|within|under|across|using|with|from|for)\b", |
| lower_text, |
| ): |
| families.add("PASSIVE_PROCESS_CONTEXT") |
|
|
| if re.search( |
| r"\b(?:is|are|was|were|has been|have been|had been)\s+not\s+" |
| r"(?:treated|regarded|interpreted|classified|considered)\s+as\b", |
| lower_text, |
| ): |
| families.add("NEGATED_CLASSIFICATION_PASSIVE") |
|
|
| |
| if re.search( |
| r"\b(?:is|are|was|were|has been|have been|had been)\s+" |
| r"[A-Za-z][A-Za-z-]*(?:ed|en)\s+(?:and|or)\s+" |
| r"[A-Za-z][A-Za-z-]*(?:ed|en)" |
| r"(?:\s+(?:independently|separately|jointly|simultaneously|individually))?\b", |
| lower_text, |
| ): |
| families.add("COORDINATED_PASSIVE_PREDICATES") |
|
|
| |
| if re.search(r",\s*(?:but|whereas|while)\s+.+", lower_text, re.S): |
| families.update({"CONTRASTIVE_COORDINATED_CLAUSES", "CONTRAST"}) |
|
|
| |
| if re.search( |
| r"\b(?:is|are|was|were|has been|have been)\s+based\s+on\s+", |
| lower_text, |
| ): |
| families.update({"PASSIVE_BASIS_FRAME", "PASSIVE"}) |
|
|
| |
| if re.search( |
| r";\s*(?:it|this|that|the\s+[A-Za-z][A-Za-z-]*)\s+" |
| r"(?:does|do|did)\s+not\s+" |
| r"(?:assume|imply|indicate|mean|require|establish|demonstrate|suggest)\s+that\b", |
| lower_text, |
| ): |
| families.update({"SEMICOLON_NEGATED_ASSUMPTION", "CONTRAST"}) |
|
|
| if re.search( |
| r"\b(?:is|are|was|were|has been|have been)\s+" |
| r"(?:commonly|frequently|generally|typically|often|usually)\s+" |
| r"evaluated\s+(?:using|with)\b", |
| lower_text, |
| ): |
| families.add("EVALUATION_FRAME") |
|
|
| if re.search( |
| r"\b(?:results|findings|outcomes|observations)\s+reported\s+" |
| r"(?:independently|separately)\b", |
| lower_text, |
| ): |
| families.add("REPORTING_FRAME") |
|
|
| if re.search(r"\bthrough\b", lower_text): |
| families.add("METHOD_RELATION") |
|
|
| if re.search(r"\b(?:intended|designed)\s+to\b", lower_text): |
| families.add("PURPOSE_INFINITIVE") |
|
|
| if re.search( |
| r"\bbasis\s+for\s+(?:determining|evaluating|assessing|comparing)\b", |
| lower_text, |
| ): |
| families.add("PURPOSE_RESULT_CLAUSE") |
|
|
| if re.match( |
| r"(?i)^(?:however|therefore|consequently|thus|accordingly|" |
| r"nevertheless|nonetheless|in contrast|by contrast|as a result)\b", |
| text, |
| ): |
| families.add("DISCOURSE_MARKER") |
|
|
| if re.match(r"(?i)^(?:therefore|consequently|thus|accordingly|as a result)\b", text): |
| families.update({"CONSEQUENCE", "RESULT_CLAUSE"}) |
|
|
| return families |
|
|
| |
| |
| |
|
|
| FORMULAIC_PATTERNS = { |
| "FORMULAIC_PURPOSE": [ |
| r"\bthe purpose (?:of this|is to)\b", |
| r"\bthe aim (?:of this|is to)\b", |
| r"\bthe objective (?:of this|is to)\b", |
| ], |
| "FORMULAIC_RESULT": [ |
| r"\bas a result\b", r"\bconsequently\b", r"\btherefore\b", |
| r"\bthis suggests that\b", r"\bthese results indicate that\b", |
| ], |
| "FORMULAIC_COMPARISON": [ |
| r"\bin contrast\b", r"\bby contrast\b", r"\bcompared with\b", r"\bcompared to\b", |
| ], |
| "FORMULAIC_EVALUATION": [ |
| r"\bis evaluated using\b", r"\bare evaluated using\b", |
| r"\bis evaluated across\b", r"\bare evaluated across\b", |
| r"\bis assessed using\b", r"\bare assessed using\b", |
| ], |
| "FORMULAIC_METHOD": [ |
| r"\busing this approach\b", r"\bunder this framework\b", |
| r"\bwithin this framework\b", r"\bthe methodology allows\b", |
| ], |
| "FORMULAIC_CONCLUSION": [ |
| r"\btaken together\b", r"\boverall, these\b", r"\bcollectively, these\b", |
| ], |
| "OVEREXPLICIT_LOGIC": [ |
| r"\bthis means that\b", r"\bthis indicates that\b", |
| r"\bthis demonstrates that\b", r"\bit is important to note that\b", |
| r"\bit should be noted that\b", |
| ], |
| "STILTED_CONNECTIVE": [ |
| r"\bfurthermore\b", r"\bmoreover\b", r"\bnevertheless\b", r"\baccordingly\b", |
| ], |
| } |
|
|
|
|
| def detect_formulaic_patterns(text, profile): |
| lower_text = _pt_clean(text).lower() |
| for flag, patterns in FORMULAIC_PATTERNS.items(): |
| for pattern in patterns: |
| match = re.search(pattern, lower_text, re.I) |
| if match: |
| profile.formulaic_flags.add(flag) |
| _add_evidence(profile, flag, match.group(0)) |
|
|
| |
| |
| |
|
|
| def detect_naturalness(text, profile): |
| text = _pt_clean(text) |
| words = _pt_words(text) |
| word_count = len(words) |
| profile.metrics["word_count"] = word_count |
|
|
| if word_count >= 35: |
| profile.complexity_flags.add("LONG_SENTENCE") |
| if word_count >= 50: |
| profile.complexity_flags.add("VERY_LONG_SENTENCE") |
|
|
| clause_markers = re.findall( |
| r"\b(?:that|which|whereby|whereas|although|because|while|when|whether|if|so that)\b", |
| text, |
| re.I, |
| ) |
| profile.metrics["clause_marker_count"] = len(clause_markers) |
| if len(clause_markers) >= 3: |
| profile.complexity_flags.add("MULTI_EMBEDDED_CLAUSES") |
| if word_count >= 35 and len(clause_markers) >= 2: |
| profile.naturalness_flags.add("CLAUSE_OVERLOADING") |
|
|
| prepositions = re.findall( |
| r"\b(?:of|in|within|under|across|among|through|between|for|from|with|by|on|at)\b", |
| text, |
| re.I, |
| ) |
| profile.metrics["preposition_count"] = len(prepositions) |
| if len(prepositions) >= 6 and word_count >= 25: |
| profile.complexity_flags.add("LONG_PREPOSITIONAL_CHAIN") |
|
|
| passive_hits = re.findall( |
| r"\b(?:is|are|was|were|has been|have been|had been)\s+(?:not\s+)?" |
| r"(?:\w+\s+){0,2}\w+(?:ed|en)\b", |
| text, |
| re.I, |
| ) |
| profile.metrics["passive_count"] = len(passive_hits) |
| if len(passive_hits) >= 2: |
| profile.naturalness_flags.add("AWKWARD_PASSIVE") |
|
|
| transitions = re.findall( |
| r"\b(?:however|therefore|consequently|thus|moreover|furthermore|" |
| r"accordingly|nevertheless|nonetheless)\b", |
| text, |
| re.I, |
| ) |
| profile.metrics["transition_count"] = len(transitions) |
| if len(transitions) >= 2: |
| profile.discourse_flags.add("REPETITIVE_TRANSITION") |
|
|
| nominalizations = re.findall( |
| r"\b[A-Za-z]+(?:tion|sion|ment|ance|ence|ity|ness)\b", |
| text, |
| re.I, |
| ) |
| profile.metrics["nominalization_count"] = len(nominalizations) |
| if len(nominalizations) >= 4 and word_count >= 25: |
| profile.naturalness_flags.add("UNNATURAL_NOMINALIZATION") |
|
|
| and_count = len(re.findall(r"\band\b", text, re.I)) |
| profile.metrics["and_count"] = and_count |
| if and_count >= 4 and word_count >= 30: |
| profile.naturalness_flags.add("MECHANICAL_PARALLELISM") |
|
|
| |
| |
| |
|
|
| def analyze_pattern_taxonomy(text): |
| profile = PatternProfile() |
| profile.grammar_families.update(detect_grammar_families(text)) |
| detect_formulaic_patterns(text, profile) |
| detect_naturalness(text, profile) |
| return profile |
|
|
|
|
| def taxonomy_categories(flag): |
| return [category for category, members in PATTERN_TAXONOMY.items() if flag in members] |
|
|
|
|
| def pattern_report(text): |
| profile = analyze_pattern_taxonomy(text) |
| return { |
| "grammar_families": sorted(profile.grammar_families), |
| "formulaic_flags": sorted(profile.formulaic_flags), |
| "naturalness_flags": sorted(profile.naturalness_flags), |
| "complexity_flags": sorted(profile.complexity_flags), |
| "discourse_flags": sorted(profile.discourse_flags), |
| "metrics": dict(profile.metrics), |
| "evidence": dict(profile.evidence), |
| } |
|
|
| |
| |
| |
|
|
| _PATTERN_TESTS = { |
| "while_participial": "The comparison preserves the constraints while applying the methodology.", |
| "while_finite": "Accuracy measures overall performance, while recall tracks missed examples.", |
| "whereby": "The design uses a common procedure whereby each model is evaluated so that comparisons remain consistent.", |
| "evaluation_context": "The models are evaluated across several conditions.", |
| "negated_purpose": "The purpose is not to rank the methods but to compare their behavior.", |
| "passive_process": "The models are trained independently on their respective feature representations.", |
| "negated_classification": "The representations are not treated as evidence of invariant features.", |
| "participial_result": "The models operate in separate environments, allowing their effects to be evaluated.", |
| "that_determiner": "The study does not implement that type of domain alignment.", |
| "that_content_complement": "The results indicate that the models preserve their original behavior.", |
| "coordinated_passive": "The models are trained and evaluated separately.", |
| "contrastive_coordination": "The models are trained separately, but both environments use the same procedure.", |
| "passive_basis": "The interpretation is based on changes in the measured outcomes.", |
| "semicolon_negated_assumption": "The comparison uses separate measurements; it does not assume that the representations are equivalent.", |
| } |
|
|
| _PATTERN_RESULTS = {name: pattern_report(sentence) for name, sentence in _PATTERN_TESTS.items()} |
|
|
|
|
| def _assert_pattern(test_name, expected_family): |
| detected = _PATTERN_RESULTS[test_name]["grammar_families"] |
| if expected_family not in detected: |
| raise RuntimeError( |
| f"Pattern taxonomy failed {expected_family} self-test for '{test_name}'. Detected: {detected}" |
| ) |
|
|
|
|
| for test_name, expected in [ |
| ("while_participial", "WHILE_PARTICIPIAL"), |
| ("while_finite", "FINITE_WHILE_CLAUSE"), |
| ("whereby", "WHEREBY_CLAUSE"), |
| ("evaluation_context", "PASSIVE_EVALUATION_CONTEXT"), |
| ("negated_purpose", "NEGATED_PURPOSE_CONTRAST"), |
| ("passive_process", "PASSIVE_PROCESS_CONTEXT"), |
| ("negated_classification", "NEGATED_CLASSIFICATION_PASSIVE"), |
| ("participial_result", "PARTICIPIAL_RESULT_CLAUSE"), |
| ("that_content_complement", "CONTENT_COMPLEMENT"), |
| ("coordinated_passive", "COORDINATED_PASSIVE_PREDICATES"), |
| ("contrastive_coordination", "CONTRASTIVE_COORDINATED_CLAUSES"), |
| ("passive_basis", "PASSIVE_BASIS_FRAME"), |
| ("semicolon_negated_assumption", "SEMICOLON_NEGATED_ASSUMPTION"), |
| ]: |
| _assert_pattern(test_name, expected) |
|
|
| if "CONTENT_COMPLEMENT" in _PATTERN_RESULTS["that_determiner"]["grammar_families"]: |
| raise RuntimeError("Pattern taxonomy incorrectly classified determiner 'that' as CONTENT_COMPLEMENT.") |
|
|
| |
| |
| |
|
|
| print("=" * 72) |
| print("CELL 4B PATTERN TAXONOMY SUCCESSFULLY LOADED") |
| print("=" * 72) |
| print("Grammar-family taxonomy: active") |
| print("Formulaic-expression analysis: active") |
| print("Naturalness analysis: active") |
| print("Complexity analysis: active") |
| print("Discourse analysis: active") |
| print("\nVerified grammar routes:") |
| for family in [ |
| "FINITE_WHILE_CLAUSE", "WHILE_PARTICIPIAL", "WHEREBY_CLAUSE", |
| "PASSIVE_EVALUATION_CONTEXT", "PASSIVE_PROCESS_CONTEXT", |
| "NEGATED_CLASSIFICATION_PASSIVE", "PARTICIPIAL_RESULT_CLAUSE", |
| "NEGATED_PURPOSE_CONTRAST", "CONTENT_COMPLEMENT", |
| "COORDINATED_PASSIVE_PREDICATES", "CONTRASTIVE_COORDINATED_CLAUSES", |
| "PASSIVE_BASIS_FRAME", "SEMICOLON_NEGATED_ASSUMPTION", |
| ]: |
| print(f" [OK] {family}") |
| print(" [OK] determiner 'that' exclusion") |
| print("\nCell 4B is ready. Update and run Cell 5 next.") |
|
|
|
|
| |
| |
| |
| |
| |
| |
|
|
| from dataclasses import dataclass |
| from typing import Callable |
| from collections import Counter |
| import re |
|
|
| _REQUIRED_CELL5 = ["Candidate", "CFG", "Analyzer", "nlp"] |
| _missing_cell5 = [name for name in _REQUIRED_CELL5 if name not in globals()] |
| if _missing_cell5: |
| raise RuntimeError( |
| "Standalone Cell 5 cannot run. Missing: " |
| + ", ".join(_missing_cell5) |
| + ". Run Cells 1–4/4B first." |
| ) |
|
|
| @dataclass |
| class Rule: |
| name: str |
| lanes: set |
| families: set |
| priority: float |
| fn: Callable |
|
|
| RULES = [] |
|
|
| def rule(name, lanes, families, priority=.5): |
| """Register or replace one deterministic transformation rule.""" |
| def deco(fn): |
| global RULES |
| RULES = [registered for registered in RULES if registered.name != name] |
| RULES.append( |
| Rule( |
| str(name), |
| set(lanes), |
| set(families), |
| float(priority), |
| fn, |
| ) |
| ) |
| return fn |
| return deco |
|
|
|
|
| def clean(text): |
| value = re.sub( |
| r"\s+([,.;:!?])", |
| r"\1", |
| re.sub(r"\s+", " ", str(text or "")), |
| ).strip() |
| if value: |
| value = value[0].upper() + value[1:] |
| return value if not value or value[-1] in ".!?" else value + "." |
|
|
|
|
| def bare(text): |
| return re.sub(r"[.?!]+$", "", clean(text)).strip() |
|
|
|
|
| def lower_initial(text): |
| value = str(text or "").strip() |
| if not value or re.match(r"^[A-Z]{2,}\b", value): |
| return value |
| return value[:1].lower() + value[1:] |
|
|
|
|
| def surface_key(text): |
| return re.sub(r"\W+", " ", str(text or "").lower()).strip() |
|
|
|
|
| _surface_key = surface_key |
|
|
|
|
| def K(text, lane, family, reason, priority=.5): |
| return Candidate( |
| clean(text), |
| lane, |
| family, |
| reason, |
| float(priority), |
| ) |
|
|
|
|
| def emit(text, lane, family, reason, priority=.5): |
| value = clean(text) |
| return [] if not value else [K(value, lane, family, reason, priority)] |
|
|
|
|
| def _safe_subtree_text(token): |
| """Return a token subtree without requiring a separate helper cell.""" |
| try: |
| items = list(token.subtree) |
| if not items: |
| return token.text |
| start = min(item.i for item in items) |
| end = max(item.i for item in items) + 1 |
| return token.doc[start:end].text |
| except Exception: |
| return getattr(token, "text", "") |
|
|
|
|
| |
| |
| |
|
|
| @rule("copular_action_frame", {"Balanced", "Structural"}, {"COPULAR_ACTION_FRAME"}, .92) |
| def tr_copular_action(analysis, profile, user_terms): |
| match = re.match( |
| r"(?is)^The\s+(?P<head>next|first|second|third|final|primary|main)\s+" |
| r"(?P<noun>step|task|action|objective|goal)\s+is\s+to\s+" |
| r"(?P<action>.+?)[.]?$", |
| analysis.text, |
| ) |
| if not match: |
| return [] |
| return [ |
| K( |
| f"The {match['head']} {match['noun']} involves {match['action']}", |
| "Balanced", |
| "COPULAR_ACTION_FRAME", |
| "Converted a copular action frame into an event frame.", |
| .92, |
| ) |
| ] |
|
|
|
|
| @rule("need_to_action", {"Balanced", "Structural"}, {"NEED_TO_ACTION"}, .88) |
| def tr_need_to(analysis, profile, user_terms): |
| match = re.match( |
| r"(?is)^(?P<subject>.+?)\s+(?P<need>need|needs|needed)\s+to\s+" |
| r"(?P<action>.+?)[.]?$", |
| analysis.text, |
| ) |
| if not match: |
| return [] |
| modal = "had to" if match["need"].lower() == "needed" else "must" |
| return [ |
| K( |
| f"{match['subject']} {modal} {match['action']}", |
| "Balanced", |
| "NEED_TO_ACTION", |
| "Recast a need-to construction while retaining its subject and action.", |
| .88, |
| ) |
| ] |
|
|
|
|
| @rule( |
| "finite_intent_rather_than", |
| {"Balanced", "Structural"}, |
| {"FINITE_PURPOSE_INTENT", "RATHER_THAN_GERUND"}, |
| .90, |
| ) |
| def tr_finite_intent(analysis, profile, user_terms): |
| match = re.match( |
| r"(?is)^(?P<subject>.+?)\s+" |
| r"(?P<intent>intend|intends|intended)\s+to\s+" |
| r"(?P<action>.+?)\s+rather\s+than\s+" |
| r"(?P<alternative>\w+ing\b.+?)[.]?$", |
| analysis.text, |
| ) |
| if not match: |
| return [] |
| return [ |
| K( |
| f"Rather than {match['alternative']}, {lower_initial(match['subject'])} " |
| f"{match['intent']} to {match['action']}", |
| "Balanced", |
| "FINITE_PURPOSE_INTENT", |
| "Fronted the rejected alternative while preserving the stated intent.", |
| .90, |
| ) |
| ] |
|
|
|
|
| @rule( |
| "modal_copular_enumeration", |
| {"Balanced", "Structural"}, |
| {"MODAL_COPULAR_ENUMERATION", "COORDINATED_OBJECTS_OR_ENUMERATION", "ENUMERATION"}, |
| .91, |
| ) |
| def tr_modal_enumeration(analysis, profile, user_terms): |
| match = re.match( |
| r"(?is)^(?P<subject>.+?)\s+" |
| r"(?P<modal>may|might|could|can)\s+be\s+in\s+" |
| r"(?P<items>.+?,\s*.+?\s*,?\s*(?:or|and)\s+.+?)[.]?$", |
| analysis.text, |
| ) |
| if not match: |
| return [] |
| return [ |
| K( |
| f"Possible sources of {lower_initial(match['subject'])} " |
| f"{match['modal'].lower()} include {match['items']}", |
| "Balanced", |
| "MODAL_COPULAR_ENUMERATION", |
| "Recast a modal copular list while preserving uncertainty and every member.", |
| .91, |
| ) |
| ] |
|
|
|
|
| @rule( |
| "enumeration_frame", |
| {"Minimal", "Balanced"}, |
| {"COORDINATED_OBJECTS_OR_ENUMERATION", "ENUMERATION"}, |
| .65, |
| ) |
| def tr_enumeration_frame(analysis, profile, user_terms): |
| match = re.match( |
| r"(?is)^(?P<subject>.+?)\s+(?:consists?\s+of|comprises?)\s+" |
| r"(?P<items>.+?,.+?(?:and|or).+?)[.]?$", |
| analysis.text, |
| ) |
| if not match: |
| return [] |
| return emit( |
| f"{match['subject']} includes {match['items']}", |
| "Minimal", |
| "ENUMERATION", |
| "Varied an explicit enumeration frame without changing its members.", |
| .65, |
| ) |
|
|
|
|
| @rule( |
| "using_method_relation", |
| {"Balanced", "Structural"}, |
| {"USING_METHOD_RELATION", "METHOD_RELATION"}, |
| .89, |
| ) |
| def tr_using_method(analysis, profile, user_terms): |
| match = re.match( |
| r"(?is)^(?P<main>.+?)\s+using\s+(?P<method>.+?)[.]?$", |
| analysis.text, |
| ) |
| if not match: |
| return [] |
| return [ |
| K( |
| f"Using {match['method']}, {lower_initial(match['main'])}", |
| "Balanced", |
| "USING_METHOD_RELATION", |
| "Fronted the explicitly stated method relation.", |
| .89, |
| ) |
| ] |
|
|
|
|
| @rule( |
| "through_method_relation", |
| {"Balanced", "Structural"}, |
| {"THROUGH_METHOD_RELATION", "METHOD_RELATION"}, |
| .86, |
| ) |
| def tr_through_method(analysis, profile, user_terms): |
| match = re.match( |
| r"(?is)^(?P<main>.+?)\s+through\s+(?P<method>.+?)[.]?$", |
| analysis.text, |
| ) |
| if not match: |
| return [] |
| return emit( |
| f"Through {match['method']}, {lower_initial(match['main'])}", |
| "Balanced", |
| "THROUGH_METHOD_RELATION", |
| "Fronted an explicitly stated through-method relation.", |
| .86, |
| ) |
|
|
|
|
| @rule( |
| "by_gerund_method", |
| {"Balanced", "Structural"}, |
| {"BY_GERUND_METHOD_RELATION", "METHOD_RELATION"}, |
| .84, |
| ) |
| def tr_by_gerund(analysis, profile, user_terms): |
| match = re.match( |
| r"(?is)^(?P<main>.+?)\s+by\s+(?P<method>\w+ing\b.+?)[.]?$", |
| analysis.text, |
| ) |
| if not match: |
| return [] |
| return emit( |
| f"By {match['method']}, {lower_initial(match['main'])}", |
| "Balanced", |
| "BY_GERUND_METHOD_RELATION", |
| "Fronted an explicitly stated by-gerund method relation.", |
| .84, |
| ) |
|
|
|
|
| @rule( |
| "purpose_infinitive_relocation", |
| {"Balanced", "Structural"}, |
| {"PURPOSE_INFINITIVE_FRONTING"}, |
| .88, |
| ) |
| def tr_purpose_front(analysis, profile, user_terms): |
| match = re.match( |
| r"(?is)^To\s+(?P<purpose>[^,]{2,160}),\s*(?P<main>.+?)[.]?$", |
| analysis.text, |
| ) |
| if not match: |
| return [] |
| return emit( |
| f"{match['main']} to {match['purpose']}", |
| "Balanced", |
| "PURPOSE_INFINITIVE_FRONTING", |
| "Moved the explicit purpose clause after the main proposition.", |
| .88, |
| ) |
|
|
|
|
| @rule( |
| "purpose_contrast", |
| {"Balanced", "Structural"}, |
| {"NEGATED_PURPOSE_CONTRAST"}, |
| .98, |
| ) |
| def tr_purpose(analysis, profile, user_terms): |
| match = re.match( |
| r"(?is)^(?P<head>.*?\b(?:purpose|aim|objective|goal)\b.*?)\s+" |
| r"is\s+not\s+to\s+(?P<x>.+?)\s+but\s+to\s+(?P<y>.+?)[.]?$", |
| analysis.text, |
| ) |
| if not match: |
| return [] |
| return [ |
| K( |
| f"{match['head']} is to {match['y']} rather than to {match['x']}", |
| "Balanced", |
| "NEGATED_PURPOSE_CONTRAST", |
| "Reordered contrasted objectives while preserving both scopes.", |
| .98, |
| ), |
| K( |
| f"{match['head']} does not seek to {match['x']}. " |
| f"Instead, it aims to {match['y']}", |
| "Structural", |
| "NEGATED_PURPOSE_CONTRAST", |
| "Separated the excluded and intended objectives.", |
| .90, |
| ), |
| ] |
|
|
|
|
| @rule( |
| "evidential_complement", |
| {"Balanced", "Structural"}, |
| {"EVIDENTIAL_COMPLEMENT"}, |
| .94, |
| ) |
| def tr_evidence(analysis, profile, user_terms): |
| match = re.match( |
| r"(?is)^(?P<src>.+?)\s+" |
| r"(?:provide|provides|provided|offer|offers|offered)\s+" |
| r"(?:clear\s+|strong\s+|empirical\s+)?evidence\s+that\s+" |
| r"(?P<claim>.+?)[.]?$", |
| analysis.text, |
| ) |
| if not match: |
| return [] |
| return [ |
| K( |
| f"Evidence from {lower_initial(match['src'])} indicates that {match['claim']}", |
| "Balanced", |
| "EVIDENTIAL_COMPLEMENT", |
| "Recast the evidential frame while preserving the claim.", |
| .94, |
| ), |
| K( |
| f"According to the evidence provided by {lower_initial(match['src'])}, " |
| f"{match['claim']}", |
| "Structural", |
| "EVIDENTIAL_COMPLEMENT", |
| "Reorganized the source and claim around the same evidential relation.", |
| .76, |
| ), |
| ] |
|
|
|
|
| @rule("whether_complement", {"Balanced", "Structural"}, {"WHETHER_COMPLEMENT"}, .89) |
| def tr_whether(analysis, profile, user_terms): |
| match = re.match( |
| r"(?is)^(?P<subject>.+?)\s+require(?:s|d)?\s+" |
| r"(?:an?\s+)?(?:consideration|assessment|evaluation|analysis)\s+" |
| r"of\s+whether\s+(?P<clause>.+?)[.]?$", |
| analysis.text, |
| ) |
| if not match: |
| return [] |
| subject = re.sub(r"\balso\b", "", match["subject"], flags=re.I).strip() |
| return emit( |
| f"Assessing whether {match['clause']} is also required when considering " |
| f"{lower_initial(subject)}", |
| "Balanced", |
| "WHETHER_COMPLEMENT", |
| "Converted a nominalized whether-complement into an assessment clause.", |
| .89, |
| ) |
|
|
|
|
| @rule("gerund_subject", {"Balanced", "Structural"}, {"GERUND_SUBJECT"}, .88) |
| def tr_gerund(analysis, profile, user_terms): |
| source = bare(analysis.text) |
| match = re.match( |
| r"(?is)^(?P<verb>Applying|Using|Evaluating|Comparing|Assessing|Examining|" |
| r"Analyzing|Implementing|Testing|Measuring)\s+(?P<object>.+?)\s+" |
| r"(?P<predicate>reduces?|provides?|enables?|supports?|allows?|improves?|" |
| r"limits?|increases?|decreases?|preserves?|reveals?|shows?)\b(?P<tail>.*)$", |
| source, |
| ) |
| if not match: |
| return [] |
| nouns = { |
| "applying": "application", |
| "using": "use", |
| "evaluating": "evaluation", |
| "comparing": "comparison", |
| "assessing": "assessment", |
| "examining": "examination", |
| "analyzing": "analysis", |
| "implementing": "implementation", |
| "testing": "testing", |
| "measuring": "measurement", |
| } |
| event = nouns[match["verb"].lower()] |
| return emit( |
| f"The {event} of {match['object']} {match['predicate']}{match['tail']}", |
| "Balanced", |
| "GERUND_SUBJECT", |
| "Converted a gerund event into an explicit nominal subject.", |
| .88, |
| ) |
|
|
|
|
| @rule("whereby_mechanism", {"Balanced", "Structural"}, {"WHEREBY_CLAUSE"}, .84) |
| def tr_whereby(analysis, profile, user_terms): |
| match = re.match( |
| r"(?is)^(?P<main>.+?)\s+whereby\s+(?P<x>.+?)" |
| r"(?:\s+so\s+that\s+(?P<y>.+?))?[.]?$", |
| analysis.text, |
| ) |
| if not match: |
| return [] |
| middle = ( |
| f"{match['main']}; under this design, {lower_initial(match['x'])}" |
| + (f", so that {match['y']}" if match.group("y") else "") |
| ) |
| output = [ |
| K( |
| middle, |
| "Balanced", |
| "WHEREBY_CLAUSE", |
| "Separated the main proposition from its mechanism.", |
| .84, |
| ) |
| ] |
| if match.group("y"): |
| output.append( |
| K( |
| f"{match['main']}. Under this design, {match['x']}. " |
| f"This arrangement is intended so that {match['y']}", |
| "Structural", |
| "WHEREBY_CLAUSE", |
| "Separated the claim, mechanism, and purpose.", |
| .72, |
| ) |
| ) |
| return output |
|
|
|
|
| @rule("while_participial", {"Balanced", "Structural"}, {"WHILE_PARTICIPIAL"}, .84) |
| def tr_while(analysis, profile, user_terms): |
| match = re.match( |
| r"(?is)^(?P<main>.+?)\s+while\s+(?P<x>\w+ing\b.+?)[.]?$", |
| analysis.text, |
| ) |
| if not match: |
| return [] |
| return emit( |
| f"While {match['x']}, {lower_initial(match['main'])}", |
| "Balanced", |
| "WHILE_PARTICIPIAL", |
| "Reordered a simultaneous participial clause without changing its relation.", |
| .84, |
| ) |
|
|
|
|
| @rule("fronted_adjunct", {"Balanced", "Structural"}, {"FRONTED_ADJUNCT"}, .73) |
| def tr_front(analysis, profile, user_terms): |
| if re.match( |
| r"(?is)^\s*(?:for purposes of|as used in|for the purposes of|" |
| r"in this (?:agreement|contract|plan|section|policy|specification))\b", |
| analysis.text, |
| ): |
| return [] |
| match = re.match( |
| r"(?is)^(?P<adj>(?:For|In|Within|Under|During|Among|Across|Throughout)\s+" |
| r"[^,]+),\s*(?P<main>.+?)[.]?$", |
| analysis.text, |
| ) |
| if not match: |
| return [] |
| return emit( |
| f"{match['main']} {lower_initial(match['adj'])}", |
| "Balanced", |
| "FRONTED_ADJUNCT", |
| "Moved an existing domain adjunct after the main proposition.", |
| .73, |
| ) |
|
|
|
|
| @rule("discourse_marker", {"Minimal", "Balanced"}, {"DISCOURSE_FRONTING"}, .72) |
| def tr_discourse(analysis, profile, user_terms): |
| match = re.match( |
| r"(?is)^(?P<mark>However|Therefore|Consequently|Thus|Moreover|Furthermore|" |
| r"Nevertheless|Accordingly|Additionally),\s+(?P<main>.+?)[.]?$", |
| analysis.text, |
| ) |
| if not match: |
| return [] |
| synonyms = { |
| "However": "Nevertheless", |
| "Therefore": "For this reason", |
| "Consequently": "As a result", |
| "Thus": "Accordingly", |
| "Moreover": "In addition", |
| "Furthermore": "Additionally", |
| "Nevertheless": "Even so", |
| "Accordingly": "Therefore", |
| "Additionally": "In addition", |
| } |
| return emit( |
| f"{synonyms[match['mark']]}, {match['main']}", |
| "Minimal", |
| "DISCOURSE_FRONTING", |
| "Varied an existing discourse marker within the same relation class.", |
| .62, |
| ) |
|
|
|
|
| @rule( |
| "coordinated_passive_participles", |
| {"Minimal", "Balanced"}, |
| {"COORDINATED_PASSIVE_PARTICIPLES", "COORDINATED_PREDICATES"}, |
| .86, |
| ) |
| def tr_passive_series(analysis, profile, user_terms): |
| source = bare(analysis.text) |
| match = re.search( |
| r"(?is)\b(?P<noun>[A-Za-z][\w-]*(?:\s+[A-Za-z][\w-]*){0,5})\s+that\s+" |
| r"(?:have|has|had)\s+been\s+" |
| r"(?P<series>(?:\w+ly\s+)?\w+(?:ed|en)\s*,\s*" |
| r"\w+(?:ed|en)\s*,\s*and\s*\w+(?:ed|en)(?:\s+by\s+[^.;]+)?)", |
| source, |
| ) |
| if not match: |
| return [] |
| revised = source[:match.start()] + f"{match['noun']} {match['series']}" + source[match.end():] |
| return emit( |
| revised, |
| "Minimal", |
| "COORDINATED_PASSIVE_PARTICIPLES", |
| "Reduced a relative passive clause while preserving its coordinated series.", |
| .86, |
| ) |
|
|
|
|
| @rule("passive_using_nominal", {"Balanced", "Structural"}, {"PASSIVE_USING"}, .91) |
| def tr_passive_using(analysis, profile, user_terms): |
| match = re.match( |
| r"(?is)^(?P<subject>.+?)\s+" |
| r"(?:is|are|was|were|has been|have been|had been)\s+" |
| r"(?P<adverb>\w+ly\s+)?evaluated\s+using\s+" |
| r"(?P<object>.+?)[.]?$", |
| analysis.text, |
| ) |
| if not match: |
| return [] |
| adverb = (match.group("adverb") or "").strip() |
| return emit( |
| f"Evaluations of {lower_initial(match['subject'])} " |
| f"{adverb + ' ' if adverb else ''}use {match['object']}", |
| "Balanced", |
| "PASSIVE_USING", |
| "Recast a passive evaluation without inventing an actor.", |
| .91, |
| ) |
|
|
|
|
| @rule("agentless_study_passive", {"Balanced", "Structural"}, {"PASSIVE"}, .70) |
| def tr_agentless(analysis, profile, user_terms): |
| match = re.match( |
| r"(?is)^(?P<topic>.+?)\s+(?:has|have)\s+(?P<also>also\s+)?been\s+" |
| r"(?:studied|examined|investigated)\s+" |
| r"(?P<context>(?:within|in|across)\s+.+?)[.]?$", |
| analysis.text, |
| ) |
| if not match: |
| return [] |
| return emit( |
| f"Studies {match['context']} have {match['also'] or ''}examined {match['topic']}", |
| "Balanced", |
| "PASSIVE", |
| "Recast an agentless research passive without inventing a specific actor.", |
| .70, |
| ) |
|
|
|
|
| @rule("explicit_agent_active", {"Structural"}, {"PASSIVE"}, .82) |
| def tr_explicit_agent(analysis, profile, user_terms): |
| match = re.match( |
| r"(?is)^(?P<theme>.+?)\s+" |
| r"(?:was|were|is|are|has been|have been|had been)\s+" |
| r"(?P<verb>evaluated|examined|studied|investigated|tested|measured|reported|" |
| r"implemented|applied|modified|trained|collected|derived|obtained|inspected|" |
| r"approved|reviewed|documented)\s+by\s+(?P<agent>.+?)[.]?$", |
| analysis.text, |
| ) |
| if not match: |
| return [] |
| return emit( |
| f"{match['agent']} {match['verb']} {lower_initial(match['theme'])}", |
| "Structural", |
| "PASSIVE", |
| "Converted passive to active because the source supplied the actor.", |
| .82, |
| ) |
|
|
|
|
| @rule( |
| "reduced_modifier", |
| {"Minimal", "Balanced"}, |
| {"REDUCED_RELATIVE_OR_NP_MODIFIER"}, |
| .68, |
| ) |
| def tr_reduced(analysis, profile, user_terms): |
| source = bare(analysis.text) |
| output = [] |
| for pattern, replacement in [ |
| (r"\bresults?\s+obtained\s+from\b", "results derived from"), |
| (r"\bsamples?\s+obtained\s+from\b", "samples drawn from"), |
| (r"\bdata\s+collected\s+from\b", "data gathered from"), |
| (r"\bresults?\s+reported\s+independently\b", "independently reported results"), |
| ]: |
| if re.search(pattern, source, re.I): |
| output.append( |
| K( |
| re.sub(pattern, replacement, source, count=1, flags=re.I), |
| "Minimal", |
| "REDUCED_RELATIVE_OR_NP_MODIFIER", |
| "Recast a reduced participial noun modifier.", |
| .68, |
| ) |
| ) |
| return output |
|
|
|
|
| @rule( |
| "parenthetical_emphasis", |
| {"Minimal", "Balanced"}, |
| {"PARENTHETICAL_EMPHASIS"}, |
| .78, |
| ) |
| def tr_parenthetical(analysis, profile, user_terms): |
| revised = re.sub( |
| r",\s*by\s+(?:itself|themselves)\s*,", |
| " alone", |
| analysis.text, |
| flags=re.I, |
| ) |
| if surface_key(revised) == surface_key(analysis.text): |
| return [] |
| return emit( |
| revised, |
| "Minimal", |
| "PARENTHETICAL_EMPHASIS", |
| "Integrated parenthetical emphasis into the clause.", |
| .78, |
| ) |
|
|
|
|
| @rule("coordinated_predicate_split", {"Structural"}, {"COORDINATED_PREDICATES"}, .80) |
| def tr_coord_base(analysis, profile, user_terms): |
| try: |
| document = nlp()(bare(analysis.text)) |
| if not document.has_annotation("DEP"): |
| return [] |
| root = next((token for token in document if token.dep_ == "ROOT"), None) |
| if root is None: |
| return [] |
| conjuncts = [ |
| token for token in root.conjuncts |
| if token.pos_ in {"VERB", "AUX"} |
| ] |
| subjects = [ |
| child for child in root.children |
| if child.dep_ in {"nsubj", "nsubjpass", "csubj"} |
| ] |
| if len(conjuncts) != 1 or not subjects: |
| return [] |
| conjunct = conjuncts[0] |
| cut = min(token.i for token in conjunct.subtree) |
| left = re.sub( |
| r"(?:,\s*)?\b(?:and|or)\s*$", |
| "", |
| document[:cut].text.strip(), |
| flags=re.I, |
| ).strip() |
| subject = _safe_subtree_text(subjects[0]).strip() |
| right = _safe_subtree_text(conjunct).strip() |
| if not left or not subject or not right: |
| return [] |
| return emit( |
| f"{left}. {subject} {right}", |
| "Structural", |
| "COORDINATED_PREDICATES", |
| "Split coordinated predicates and repeated only the source subject.", |
| .80, |
| ) |
| except Exception: |
| return [] |
|
|
|
|
| @rule("semicolon_split", {"Structural"}, {"LONG_MULTI_CLAUSE"}, .65) |
| def tr_semicolon(analysis, profile, user_terms): |
| parts = [item.strip() for item in bare(analysis.text).split(";") if item.strip()] |
| if len(parts) != 2 or not all(len(item.split()) >= 3 for item in parts): |
| return [] |
| return emit( |
| f"{parts[0]}. {parts[1]}", |
| "Structural", |
| "LONG_MULTI_CLAUSE", |
| "Split two propositions already delimited by a semicolon.", |
| .65, |
| ) |
|
|
|
|
| |
| |
| |
|
|
| class Generator: |
| def applicable(self, analysis, lane): |
| detected = set(getattr(analysis, "families", []) or []) |
| return sorted( |
| [ |
| registered |
| for registered in RULES |
| if lane in registered.lanes |
| and bool(registered.families & detected) |
| ], |
| key=lambda registered: -registered.priority, |
| ) |
|
|
| def lane(self, analysis, profile, user_terms, lane, trace): |
| output = [] |
| seen = {surface_key(analysis.text)} |
| for registered in self.applicable(analysis, lane): |
| try: |
| generated = registered.fn(analysis, profile, user_terms) or [] |
| except Exception as exc: |
| trace["rule_errors"].append( |
| { |
| "rule": registered.name, |
| "lane": lane, |
| "error": str(exc), |
| } |
| ) |
| continue |
| for candidate in generated: |
| if candidate is None or candidate.lane != lane: |
| continue |
| key = surface_key(candidate.text) |
| if key and key not in seen: |
| seen.add(key) |
| output.append(candidate) |
| if lane == "Minimal" and getattr(CFG, "mlm", False): |
| output.extend(self.lexical(analysis, profile, user_terms, trace)) |
| return output[:16] |
|
|
| def generate(self, analysis, profile, user_terms, mode): |
| if mode not in {"Minimal", "Balanced", "Structural"}: |
| mode = "Balanced" |
| trace = { |
| "detected_families": list(getattr(analysis, "families", []) or []), |
| "applicable": {}, |
| "generated_by_lane": {}, |
| "rule_errors": [], |
| "candidate_traces": [], |
| } |
| pools = {} |
| for lane in ["Minimal", "Balanced", "Structural"]: |
| trace["applicable"][lane] = [ |
| registered.name |
| for registered in self.applicable(analysis, lane) |
| ] |
| pools[lane] = self.lane( |
| analysis, |
| profile, |
| user_terms, |
| lane, |
| trace, |
| ) |
| trace["generated_by_lane"][lane] = len(pools[lane]) |
|
|
| order = { |
| "Minimal": ["Minimal"], |
| "Balanced": ["Balanced", "Minimal", "Structural"], |
| "Structural": ["Structural", "Balanced", "Minimal"], |
| }[mode] |
|
|
| output = [] |
| seen = set() |
| for lane in order: |
| for candidate in pools[lane]: |
| key = surface_key(candidate.text) |
| if key and key not in seen: |
| seen.add(key) |
| output.append(candidate) |
|
|
| |
| |
| if mode in {"Balanced", "Structural"}: |
| composed = [] |
| for base in list(output[:8]): |
| try: |
| second_analysis = Analyzer().analyze( |
| base.text, |
| profile, |
| user_terms, |
| ) |
| except Exception: |
| continue |
| next_lanes = ["Minimal"] if mode == "Balanced" else ["Balanced", "Minimal"] |
| for next_lane in next_lanes: |
| for registered in self.applicable(second_analysis, next_lane)[:4]: |
| try: |
| second_candidates = registered.fn( |
| second_analysis, |
| profile, |
| user_terms, |
| ) or [] |
| except Exception: |
| continue |
| for second in second_candidates[:1]: |
| key = surface_key(second.text) |
| if not key or key in seen: |
| continue |
| seen.add(key) |
| second.lane = mode |
| second.family = f"{base.family}+{second.family}" |
| continuation = second.reason[:1].lower() + second.reason[1:] |
| second.reason = f"{base.reason} Then {continuation}" |
| second.priority = (base.priority + second.priority) / 2.0 |
| composed.append(second) |
| output.extend(composed[:8]) |
|
|
| trace["routes"] = [ |
| name |
| for lane in order |
| for name in trace["applicable"].get(lane, []) |
| ] |
| trace["generated"] = len(output) |
| return output, trace |
|
|
| def lexical(self, analysis, profile, user_terms, trace=None): |
| optional_fn = globals().get("optional") |
| if not callable(optional_fn): |
| return [] |
| pipeline = optional_fn("mlm") |
| if not pipeline: |
| return [] |
| if "wn" not in globals() or "getInflection" not in globals(): |
| return [] |
| try: |
| document = nlp()(analysis.text) |
| except Exception as exc: |
| if trace is not None: |
| trace["rule_errors"].append( |
| { |
| "rule": "contextual_lexical", |
| "lane": "Minimal", |
| "error": str(exc), |
| } |
| ) |
| return [] |
|
|
| protected = { |
| str(term).lower() |
| for term in ( |
| set(getattr(profile, "immutable", set()) or set()) |
| | set(user_terms or set()) |
| ) |
| } |
| output = [] |
| seen = {surface_key(analysis.text)} |
| for token in document: |
| if token.pos_ not in {"ADJ", "ADV", "VERB"}: |
| continue |
| if token.is_stop or not token.is_alpha: |
| continue |
| if token.lemma_.lower() in protected: |
| continue |
| try: |
| allowed = { |
| lemma.name().replace("_", " ").lower() |
| for synset in wn.synsets(token.lemma_)[:4] |
| for lemma in synset.lemmas() |
| } |
| mask = pipeline.tokenizer.mask_token |
| predictions = pipeline( |
| analysis.text[:token.idx] |
| + mask |
| + analysis.text[token.idx + len(token.text):], |
| top_k=12, |
| ) |
| except Exception: |
| continue |
| for prediction in predictions: |
| word = str(prediction.get("token_str", "")).strip().lower() |
| if not word.isalpha() or word == token.lower_ or word not in allowed: |
| continue |
| try: |
| inflected = getInflection(word, tag=token.tag_) |
| replacement = inflected[0] if inflected else word |
| except Exception: |
| replacement = word |
| revised = ( |
| analysis.text[:token.idx] |
| + replacement |
| + analysis.text[token.idx + len(token.text):] |
| ) |
| candidate = K( |
| revised, |
| "Minimal", |
| "CONTEXTUAL_LEXICAL", |
| "Contextual lexical substitution supported by WordNet and the optional masked-language model.", |
| .45, |
| ) |
| key = surface_key(candidate.text) |
| if key and key not in seen: |
| seen.add(key) |
| output.append(candidate) |
| break |
| if len(output) >= 3: |
| break |
| return output |
|
|
|
|
| |
| _BASE_RULE_NAMES = [registered.name for registered in RULES] |
| _BASE_DUPLICATES = sorted( |
| name |
| for name, count in Counter(_BASE_RULE_NAMES).items() |
| if count > 1 |
| ) |
| if _BASE_DUPLICATES: |
| raise RuntimeError( |
| "Standalone Cell 5 base registry has duplicate rules: " |
| + ", ".join(_BASE_DUPLICATES) |
| ) |
| if len(RULES) != 25: |
| raise RuntimeError( |
| "Standalone Cell 5 base registry expected 25 rules but found " |
| + str(len(RULES)) |
| ) |
|
|
| print("=" * 78) |
| print("CELL 5 BASE UNIVERSAL TRANSFORMATION ENGINE LOADED") |
| print("=" * 78) |
| print("Base rules:", len(RULES)) |
| print("Base duplicate rule names:", _BASE_DUPLICATES) |
| print("Generator: defined") |
| print("=" * 78) |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
|
|
| from collections import Counter |
| import re |
|
|
|
|
| |
| |
| |
|
|
| _V41_PARTICIPLE_TO_BASE = { |
| "spanning": "span", |
| "covering": "cover", |
| "including": "include", |
| "containing": "contain", |
| "involving": "involve", |
| } |
|
|
| _V41_IRREGULAR_3SG = { |
| "have": "has", |
| "do": "does", |
| "go": "goes", |
| } |
|
|
|
|
| def _v41_distinct(source, candidate): |
| return bool(candidate and surface_key(source) != surface_key(candidate)) |
|
|
|
|
| def _v41_balanced_parentheses(text): |
| pairs = {"(": ")", "[": "]", "{": "}"} |
| stack = [] |
| for char in str(text or ""): |
| if char in pairs: |
| stack.append(pairs[char]) |
| elif char in pairs.values(): |
| if not stack or stack.pop() != char: |
| return False |
| return not stack |
|
|
|
|
| def _v41_sentence_doc(text): |
| try: |
| doc = nlp()(str(text or "")) |
| if len(doc): |
| return doc |
| except Exception: |
| pass |
| return None |
|
|
|
|
| def _v41_subtree_span(token): |
| items = list(token.subtree) |
| if not items: |
| return None |
| start = min(item.i for item in items) |
| end = max(item.i for item in items) + 1 |
| return token.doc[start:end] |
|
|
|
|
| def _v41_noun_chunk_for_token(doc, token): |
| try: |
| chunks = list(doc.noun_chunks) |
| except Exception: |
| chunks = [] |
|
|
| containing = [chunk for chunk in chunks if chunk.start <= token.i < chunk.end] |
| if not containing: |
| return None |
|
|
| |
| return min(containing, key=lambda chunk: chunk.end - chunk.start) |
|
|
|
|
| def _v41_present_verb(base, singular): |
| base = str(base or "").lower() |
| if not singular: |
| return base |
| if base in _V41_IRREGULAR_3SG: |
| return _V41_IRREGULAR_3SG[base] |
| if re.search(r"(?:s|x|z|ch|sh|o)$", base): |
| return base + "es" |
| if re.search(r"[^aeiou]y$", base): |
| return base[:-1] + "ies" |
| return base + "s" |
|
|
|
|
| def _v41_token_is_singular(token): |
| try: |
| number = token.morph.get("Number") |
| if "Sing" in number: |
| return True |
| if "Plur" in number: |
| return False |
| except Exception: |
| pass |
|
|
| |
| return not token.text.lower().endswith("s") |
|
|
|
|
| def _v41_list_preserved(source_fragment, candidate): |
| """Require list-bearing source fragments to survive literally.""" |
| fragment = re.sub(r"\s+", " ", str(source_fragment or "")).strip(" .") |
| candidate_norm = re.sub(r"\s+", " ", str(candidate or "")) |
| return not fragment or fragment.lower() in candidate_norm.lower() |
|
|
|
|
| def _v41_emit_if_safe(source, text, lane, family, reason, priority): |
| candidate = clean(text) |
| if not _v41_distinct(source, candidate): |
| return [] |
| if not _v41_balanced_parentheses(candidate): |
| return [] |
| return [K(candidate, lane, family, reason, priority)] |
|
|
|
|
| |
| |
| |
|
|
| @rule( |
| "without_complement_list", |
| {"Balanced", "Structural"}, |
| { |
| "FINITE_COMPLEMENT_LIST", |
| "COORDINATED_OBJECTS_OR_ENUMERATION", |
| "ENUMERATION", |
| "LONG_MULTI_CLAUSE", |
| }, |
| .92, |
| ) |
| def tr_without_complement_list(analysis, profile, user_terms): |
| source = bare(analysis.text) |
|
|
| match = re.match( |
| r"(?is)^(?P<main>.+?)\s+without\s+" |
| r"(?P<missing>.+?,\s*.+?(?:,\s*)?(?:and|or)\s+.+?)$", |
| source, |
| ) |
| if not match: |
| return [] |
|
|
| main = match["main"].strip() |
| missing = match["missing"].strip() |
| outputs = [] |
|
|
| |
| text_a = f"Without {missing}, {lower_initial(main)}" |
| if _v41_list_preserved(missing, text_a): |
| outputs.extend(_v41_emit_if_safe( |
| source, |
| text_a, |
| "Balanced", |
| "FINITE_COMPLEMENT_LIST", |
| "Fronted the existing without-complement while preserving the exact polarity marker and complete list.", |
| .92, |
| )) |
|
|
| |
| |
| |
| text_b = f"{main}; this occurs without {missing}" |
| if _v41_list_preserved(missing, text_b): |
| outputs.extend(_v41_emit_if_safe( |
| source, |
| text_b, |
| "Structural", |
| "FINITE_COMPLEMENT_LIST", |
| "Separated the main proposition from its existing without-complement without changing negative scope.", |
| .86, |
| )) |
|
|
| return outputs |
|
|
|
|
| |
| |
| |
|
|
| @rule( |
| "proposal_appositive_definition", |
| {"Balanced", "Structural"}, |
| { |
| "PROPOSAL_FRAME", |
| "APPOSITIVE_DEFINITION", |
| "COORDINATED_OBJECTS_OR_ENUMERATION", |
| "ENUMERATION", |
| "LONG_MULTI_CLAUSE", |
| }, |
| .98, |
| ) |
| def tr_proposal_appositive_definition(analysis, profile, user_terms): |
| source = bare(analysis.text) |
|
|
| match = re.match( |
| r"(?is)^(?P<subject>We|I|The\s+(?:study|paper|article|report|team|authors?))\s+" |
| r"(?P<verb>propose|proposes|introduce|introduces|present|presents)\s+" |
| r"(?P<name>.+?)\s*,\s*" |
| r"(?P<article>an?|the)\s+" |
| r"(?P<definition>.+)$", |
| source, |
| ) |
| if not match: |
| return [] |
|
|
| name = match["name"].strip() |
| definition = match["definition"].strip() |
| article = match["article"].lower() |
|
|
| if not _v41_balanced_parentheses(name): |
| return [] |
|
|
| outputs = [] |
|
|
| |
| |
| text_a = ( |
| f"{name} is {article} {definition} proposed by " |
| f"{lower_initial(match['subject'])}" |
| ) |
| if _v41_list_preserved(definition, text_a): |
| outputs.extend(_v41_emit_if_safe( |
| source, |
| text_a, |
| "Balanced", |
| "APPOSITIVE_DEFINITION", |
| "Converted the proposal-plus-appositive frame into one definitional sentence while preserving the complete definition.", |
| .98, |
| )) |
|
|
| |
| text_b = ( |
| f"{match['subject']} {match['verb']} {name} as " |
| f"{article} {definition}" |
| ) |
| if _v41_list_preserved(definition, text_b): |
| outputs.extend(_v41_emit_if_safe( |
| source, |
| text_b, |
| "Structural", |
| "PROPOSAL_FRAME", |
| "Recast the appositive as an explicit as-complement while preserving the name and complete definition.", |
| .94, |
| )) |
|
|
| return outputs |
|
|
|
|
| |
| |
| |
|
|
| @rule( |
| "participial_postmodifier", |
| {"Balanced", "Structural"}, |
| { |
| "PARTICIPIAL_POSTMODIFIER", |
| "COORDINATED_OBJECTS_OR_ENUMERATION", |
| "ENUMERATION", |
| "NOMINALIZATION", |
| }, |
| .94, |
| ) |
| def tr_participial_postmodifier(analysis, profile, user_terms): |
| source = bare(analysis.text) |
| doc = _v41_sentence_doc(source) |
| outputs = [] |
|
|
| if doc is not None and doc.has_annotation("DEP"): |
| targets = [ |
| token |
| for token in doc |
| if token.text.lower() in _V41_PARTICIPLE_TO_BASE |
| and token.pos_ in {"VERB", "ADJ"} |
| ] |
|
|
| for participle in targets: |
| |
| |
| head = participle.head |
| if head is participle or head.pos_ not in {"NOUN", "PROPN", "PRON"}: |
| continue |
|
|
| chunk = _v41_noun_chunk_for_token(doc, head) |
| if chunk is None: |
| chunk_start = head.left_edge.i |
| chunk_end = head.i + 1 |
| else: |
| chunk_start = chunk.start |
| chunk_end = chunk.end |
|
|
| if participle.i < chunk_end: |
| continue |
|
|
| |
| |
| subtree = _v41_subtree_span(participle) |
| if subtree is None: |
| continue |
|
|
| complement_start = participle.i + 1 |
| complement_end = subtree.end |
| if complement_start >= complement_end: |
| continue |
|
|
| complement = doc[complement_start:complement_end].text.strip() |
| if not complement: |
| continue |
|
|
| base = _V41_PARTICIPLE_TO_BASE[participle.text.lower()] |
| finite = _v41_present_verb(base, _v41_token_is_singular(head)) |
|
|
| prefix = doc[:participle.i].text.rstrip() |
| suffix = doc[complement_end:].text.lstrip() |
| candidate = f"{prefix} that {finite} {complement}" |
| if suffix: |
| candidate += " " + suffix |
|
|
| if _v41_list_preserved(complement, candidate): |
| outputs.extend(_v41_emit_if_safe( |
| source, |
| candidate, |
| "Balanced", |
| "PARTICIPIAL_POSTMODIFIER", |
| "Converted an attested participial noun modifier into a finite relative clause while preserving its head and complement.", |
| .94, |
| )) |
|
|
| if outputs: |
| return outputs |
|
|
| |
| fallback = re.match( |
| r"(?is)^(?P<prefix>.+?\b(?:to|for|with|among|across)\s+)" |
| r"(?P<head>(?:[A-Za-z][\w-]*\s+){0,5}[A-Za-z][\w-]*)\s+" |
| r"(?P<participle>spanning|covering|including|containing|involving)\s+" |
| r"(?P<items>.+?,\s*.+?(?:,\s*)?(?:and|or)\s+.+?)$", |
| source, |
| ) |
| if not fallback: |
| return [] |
|
|
| head = fallback["head"].strip() |
| final_noun = head.split()[-1] |
| singular = not final_noun.lower().endswith("s") |
| base = _V41_PARTICIPLE_TO_BASE[fallback["participle"].lower()] |
| finite = _v41_present_verb(base, singular) |
| candidate = ( |
| f"{fallback['prefix']}{head} that {finite} {fallback['items']}" |
| ) |
|
|
| if not _v41_list_preserved(fallback["items"], candidate): |
| return [] |
|
|
| return _v41_emit_if_safe( |
| source, |
| candidate, |
| "Balanced", |
| "PARTICIPIAL_POSTMODIFIER", |
| "Converted a bounded participial postmodifier into a finite relative clause while preserving its noun phrase and complete list.", |
| .88, |
| ) |
|
|
|
|
| |
| |
| |
|
|
| @rule( |
| "finite_object_list_frame", |
| {"Balanced", "Structural"}, |
| { |
| "FINITE_OBJECT_LIST_FRAME", |
| "COORDINATED_OBJECTS_OR_ENUMERATION", |
| "ENUMERATION", |
| "LONG_MULTI_CLAUSE", |
| }, |
| .88, |
| ) |
| def tr_finite_object_list_frame(analysis, profile, user_terms): |
| source = bare(analysis.text) |
|
|
| match = re.match( |
| r"(?is)^(?P<subject>We|I|They|The\s+[A-Za-z][\w-]*(?:\s+[A-Za-z][\w-]*){0,4})\s+" |
| r"(?P<verb>pre-register|preregister|document|record|evaluate|assess|review|" |
| r"analyze|compare|test|measure|include|identify|specify|report)\s+" |
| r"(?P<items>.+?,\s*.+?(?:,\s*)?(?:and|or)\s+.+?)$", |
| source, |
| ) |
| if not match: |
| return [] |
|
|
| nominal = { |
| "pre-register": "preregistration", |
| "preregister": "preregistration", |
| "document": "documentation", |
| "record": "record", |
| "evaluate": "evaluation", |
| "assess": "assessment", |
| "review": "review", |
| "analyze": "analysis", |
| "compare": "comparison", |
| "test": "testing", |
| "measure": "measurement", |
| "include": "scope", |
| "identify": "identification", |
| "specify": "specification", |
| "report": "report", |
| }.get(match["verb"].lower()) |
|
|
| if not nominal: |
| return [] |
|
|
| subject = match["subject"].strip() |
| possessive = {"we": "Our", "i": "My", "they": "Their"}.get(subject.lower()) |
|
|
| if possessive: |
| candidate = f"{possessive} {nominal} covers {match['items']}" |
| else: |
| candidate = ( |
| f"The {nominal} conducted by {lower_initial(subject)} " |
| f"covers {match['items']}" |
| ) |
|
|
| if not _v41_list_preserved(match["items"], candidate): |
| return [] |
|
|
| return _v41_emit_if_safe( |
| source, |
| candidate, |
| "Balanced", |
| "FINITE_OBJECT_LIST_FRAME", |
| "Recast the finite predicate as an event frame while preserving every listed object verbatim.", |
| .88, |
| ) |
|
|
|
|
| |
| |
| |
|
|
| @rule( |
| "contingency_relation", |
| {"Minimal", "Balanced", "Structural"}, |
| { |
| "CONTINGENCY_RELATION", |
| "COORDINATED_OBJECTS_OR_ENUMERATION", |
| "ENUMERATION", |
| "NOMINALIZATION", |
| }, |
| .96, |
| ) |
| def tr_contingency_relation(analysis, profile, user_terms): |
| source = bare(analysis.text) |
|
|
| match = re.match( |
| r"(?is)^(?P<subject>.+?)\s+" |
| r"(?P<copula>is|are|was|were)\s+contingent\s+on\s+" |
| r"(?P<basis>.+)$", |
| source, |
| ) |
| if not match: |
| return [] |
|
|
| verb = { |
| "is": "depends", |
| "are": "depend", |
| "was": "depended", |
| "were": "depended", |
| }[match["copula"].lower()] |
|
|
| return _v41_emit_if_safe( |
| source, |
| f"{match['subject']} {verb} on {match['basis']}", |
| "Minimal", |
| "CONTINGENCY_RELATION", |
| "Recast the contingent-on relation as an equivalent dependency relation.", |
| .96, |
| ) |
|
|
|
|
| |
| |
| |
|
|
| @rule( |
| "coordinated_predicate_split", |
| {"Structural"}, |
| {"COORDINATED_PREDICATES"}, |
| .86, |
| ) |
| def tr_coord_safe(analysis, profile, user_terms): |
| try: |
| document = nlp()(bare(analysis.text)) |
| if not document.has_annotation("DEP"): |
| return [] |
|
|
| root = next((token for token in document if token.dep_ == "ROOT"), None) |
| if root is None or root.pos_ not in {"VERB", "AUX"}: |
| return [] |
|
|
| subjects = [ |
| child for child in root.children |
| if child.dep_ in {"nsubj", "nsubjpass", "csubj"} |
| ] |
| conjuncts = [ |
| token for token in root.conjuncts |
| if token.pos_ in {"VERB", "AUX"} |
| and token.dep_ == "conj" |
| and token.i > root.i |
| ] |
| if len(subjects) != 1 or len(conjuncts) != 1: |
| return [] |
|
|
| conjunct = conjuncts[0] |
| between = document[root.i + 1:conjunct.i].text |
| if not re.search(r"(?:,\s*)?\b(?:and|or)\b", between, re.I): |
| return [] |
|
|
| subject = sub(subjects[0]).strip() |
| right = sub(conjunct).strip() |
| cut = min(token.i for token in conjunct.subtree) |
| left = re.sub( |
| r"(?:,\s*)?\b(?:and|or)\s*$", |
| "", |
| document[:cut].text.strip(), |
| flags=re.I, |
| ).strip() |
|
|
| if not left or not subject or not right: |
| return [] |
| if len(left.split()) < 4: |
| return [] |
| if re.match(r"(?is)^(?:propose|introduce|present)\b", right): |
| return [] |
|
|
| return _v41_emit_if_safe( |
| bare(analysis.text), |
| f"{left}. {subject} {right}", |
| "Structural", |
| "COORDINATED_PREDICATES", |
| "Split two dependency-verified coordinated predicates and repeated only the attested subject.", |
| .86, |
| ) |
| except Exception: |
| return [] |
|
|
|
|
| |
| |
| |
|
|
| _V41_REQUIRED_RULES = { |
| "without_complement_list", |
| "proposal_appositive_definition", |
| "participial_postmodifier", |
| "finite_object_list_frame", |
| "contingency_relation", |
| "coordinated_predicate_split", |
| } |
|
|
| _v41_names = [registered.name for registered in RULES] |
| _v41_missing = sorted(_V41_REQUIRED_RULES - set(_v41_names)) |
| _v41_duplicates = sorted( |
| name for name, count in Counter(_v41_names).items() if count > 1 |
| ) |
|
|
| if _v41_missing: |
| raise RuntimeError("Cell 5 v4.1 missing rules: " + ", ".join(_v41_missing)) |
| if _v41_duplicates: |
| raise RuntimeError("Cell 5 v4.1 duplicate rules: " + ", ".join(_v41_duplicates)) |
| if len(RULES) != 30: |
| raise RuntimeError( |
| "Cell 5 v4.1 expected 30 rules but found " + str(len(RULES)) |
| ) |
|
|
| |
| if not _v41_balanced_parentheses("Framework (ALCEF-RSF)"): |
| raise RuntimeError("Cell 5 v4.1 parenthesis invariant failed.") |
| if _v41_balanced_parentheses("Framework (ALCEF-RSF"): |
| raise RuntimeError("Cell 5 v4.1 unbalanced-parenthesis test failed.") |
| if not _v41_list_preserved("A, B, and C", "The scope covers A, B, and C."): |
| raise RuntimeError("Cell 5 v4.1 list-preservation invariant failed.") |
| if _v41_present_verb("span", True) != "spans": |
| raise RuntimeError("Cell 5 v4.1 singular conjugation test failed.") |
| if _v41_present_verb("span", False) != "span": |
| raise RuntimeError("Cell 5 v4.1 plural conjugation test failed.") |
|
|
| print("=" * 78) |
| print("CELL 5 UNIVERSAL TRANSFORMATION ENGINE v4.1 LOADED") |
| print("=" * 78) |
| print("Dependency-aware noun-head extraction: active") |
| print("Bounded surface fallback: active") |
| print("Literal list-preservation checks: active") |
| print("One-sentence proposal/appositive recasts: active") |
| print("Polarity-marker preservation candidates: active") |
| print("Finite relative-clause postmodifier recast: active") |
| print("Safer coordinated-predicate split: active") |
| print("Missing v4.1 rules:", _v41_missing) |
| print("Duplicate rule names:", _v41_duplicates) |
| print("Registered transformation rules:", len(RULES)) |
| print("Expected rule count: 30") |
| print("Invariant self-tests: PASSED") |
| print("=" * 78) |
|
|
| |
| |
| |
| |
| |
| |
|
|
| UNIVERSAL_MAX_CANDIDATES_PER_LANE = 24 |
| UNIVERSAL_MAX_COMPOSED_CANDIDATES = 12 |
| UNIVERSAL_RULE_NAMES = set() |
|
|
|
|
| def universal_rule(name, lanes, families, priority=.60): |
| """Register one universal dependency rule and record its identity.""" |
| def deco(fn): |
| UNIVERSAL_RULE_NAMES.add(name) |
| return rule(name, lanes, families, priority)(fn) |
| return deco |
|
|
|
|
| def _u_doc(text): |
| try: |
| document = nlp()(str(text or "")) |
| if document.has_annotation("DEP"): |
| return document |
| except Exception: |
| pass |
| return None |
|
|
|
|
| def _u_root(document): |
| if document is None: |
| return None |
| return next((token for token in document if token.dep_ == "ROOT"), None) |
|
|
|
|
| def _u_span(token): |
| try: |
| items = list(token.subtree) |
| if not items: |
| return token.text |
| start = min(item.i for item in items) |
| end = max(item.i for item in items) + 1 |
| return token.doc[start:end].text.strip() |
| except Exception: |
| return getattr(token, "text", "").strip() |
|
|
|
|
| def _u_subject(root): |
| if root is None: |
| return None |
| return next( |
| ( |
| child for child in root.children |
| if child.dep_ in {"nsubj", "nsubjpass", "nsubj:pass", "csubj"} |
| ), |
| None, |
| ) |
|
|
|
|
| def _u_objects(root): |
| if root is None: |
| return [] |
| return [ |
| child for child in root.children |
| if child.dep_ in {"obj", "dobj", "attr", "oprd"} |
| ] |
|
|
|
|
| def _u_agent(root): |
| """Return the explicit by-agent object for a passive root.""" |
| if root is None: |
| return None |
| for child in root.children: |
| if child.dep_ in {"agent", "obl:agent"}: |
| pobj = next( |
| ( |
| item for item in child.children |
| if item.dep_ in {"pobj", "obj"} |
| ), |
| None, |
| ) |
| return pobj or child |
| if child.lower_ == "by" and child.dep_ in {"prep", "agent"}: |
| pobj = next( |
| ( |
| item for item in child.children |
| if item.dep_ in {"pobj", "obj"} |
| ), |
| None, |
| ) |
| if pobj is not None: |
| return pobj |
| return None |
|
|
|
|
| def _u_is_passive(root): |
| if root is None: |
| return False |
| passive_subject = any( |
| child.dep_ in {"nsubjpass", "nsubj:pass"} |
| for child in root.children |
| ) |
| passive_aux = any( |
| child.dep_ in {"auxpass", "aux:pass"} |
| for child in root.children |
| ) |
| return passive_subject or passive_aux |
|
|
|
|
| def _u_number(token): |
| try: |
| values = token.morph.get("Number") |
| if "Plur" in values: |
| return "plural" |
| if "Sing" in values: |
| return "singular" |
| except Exception: |
| pass |
| text = getattr(token, "text", "").lower() |
| return "plural" if text.endswith("s") and not text.endswith("ss") else "singular" |
|
|
|
|
| def _u_tense(root): |
| try: |
| values = root.morph.get("Tense") |
| if "Past" in values: |
| return "past" |
| if "Pres" in values: |
| return "present" |
| except Exception: |
| pass |
| return "past" if getattr(root, "tag_", "") == "VBD" else "present" |
|
|
|
|
| def _u_inflect(lemma, tag): |
| """Use lemminflect when available; otherwise apply conservative forms.""" |
| lemma = str(lemma or "").strip() |
| if not lemma: |
| return lemma |
| get_inflection = globals().get("getInflection") |
| if callable(get_inflection): |
| try: |
| values = get_inflection(lemma, tag=tag) |
| if values: |
| return values[0] |
| except Exception: |
| pass |
| irregular = { |
| ("be", "VBD"): "was", |
| ("be", "VBZ"): "is", |
| ("be", "VBP"): "are", |
| ("have", "VBD"): "had", |
| ("have", "VBZ"): "has", |
| ("do", "VBD"): "did", |
| ("do", "VBZ"): "does", |
| ("go", "VBD"): "went", |
| ("go", "VBZ"): "goes", |
| } |
| if (lemma.lower(), tag) in irregular: |
| return irregular[(lemma.lower(), tag)] |
| if tag == "VBD": |
| if lemma.endswith("e"): |
| return lemma + "d" |
| if re.search(r"[^aeiou]y$", lemma): |
| return lemma[:-1] + "ied" |
| return lemma + "ed" |
| if tag == "VBZ": |
| return _v41_present_verb(lemma, True) |
| return lemma |
|
|
|
|
| def _u_active_verb(root, agent): |
| tense = _u_tense(root) |
| if tense == "past": |
| tag = "VBD" |
| else: |
| tag = "VBZ" if _u_number(agent) == "singular" else "VBP" |
| return _u_inflect(root.lemma_.lower(), tag) |
|
|
|
|
| def _u_relative_pronoun(head): |
| if getattr(head, "ent_type_", "") in {"PERSON", "NORP"}: |
| return "who" |
| return "that" |
|
|
|
|
| def _u_clean_join(*parts): |
| return re.sub(r"\s+", " ", " ".join(str(part or "").strip() for part in parts if str(part or "").strip())).strip() |
|
|
|
|
| def _u_clause_without_span(document, start, end): |
| return _u_clean_join(document[:start].text, document[end:].text) |
|
|
|
|
| @universal_rule( |
| "universal_explicit_agent_passive", |
| {"Structural", "Balanced"}, |
| {"PASSIVE"}, |
| .89, |
| ) |
| def tr_universal_explicit_agent_passive(analysis, profile, user_terms): |
| source = bare(analysis.text) |
| document = _u_doc(source) |
| root = _u_root(document) |
| if document is None or root is None or root.pos_ not in {"VERB", "AUX"}: |
| return [] |
| if not _u_is_passive(root): |
| return [] |
| theme = _u_subject(root) |
| agent = _u_agent(root) |
| if theme is None or agent is None: |
| return [] |
| theme_text = _u_span(theme) |
| agent_text = _u_span(agent) |
| if not theme_text or not agent_text: |
| return [] |
| verb = _u_active_verb(root, agent) |
| candidate = f"{agent_text} {verb} {lower_initial(theme_text)}" |
| return _v41_emit_if_safe( |
| source, |
| candidate, |
| "Structural", |
| "PASSIVE", |
| "Converted a dependency-confirmed passive to active using the explicit source agent and root lemma.", |
| .89, |
| ) |
|
|
|
|
| @universal_rule( |
| "universal_participial_modifier", |
| {"Balanced", "Structural"}, |
| {"PARTICIPIAL_POSTMODIFIER", "REDUCED_RELATIVE_OR_NP_MODIFIER"}, |
| .87, |
| ) |
| def tr_universal_participial_modifier(analysis, profile, user_terms): |
| source = bare(analysis.text) |
| document = _u_doc(source) |
| if document is None: |
| return [] |
| outputs = [] |
| for token in document: |
| if token.dep_ not in {"acl", "relcl"}: |
| continue |
| if token.tag_ not in {"VBG", "VBN"} or token.head.pos_ not in {"NOUN", "PROPN", "PRON"}: |
| continue |
| head = token.head |
| subtree = list(token.subtree) |
| if not subtree: |
| continue |
| end = max(item.i for item in subtree) + 1 |
| complement = document[token.i + 1:end].text.strip() |
| if not complement: |
| continue |
| finite_tag = "VBZ" if _u_number(head) == "singular" else "VBP" |
| finite = _u_inflect(token.lemma_.lower(), finite_tag) |
| relative = _u_relative_pronoun(head) |
| candidate = _u_clean_join( |
| document[:token.i].text, |
| relative, |
| finite, |
| complement, |
| document[end:].text, |
| ) |
| outputs.extend( |
| _v41_emit_if_safe( |
| source, |
| candidate, |
| "Balanced", |
| "PARTICIPIAL_POSTMODIFIER", |
| "Converted a dependency-confirmed participial modifier into a finite relative clause without requiring a known verb list.", |
| .87, |
| ) |
| ) |
| return outputs |
|
|
|
|
| @universal_rule( |
| "universal_relative_clause_reduction", |
| {"Minimal", "Balanced"}, |
| {"REDUCED_RELATIVE_OR_NP_MODIFIER"}, |
| .72, |
| ) |
| def tr_universal_relative_clause_reduction(analysis, profile, user_terms): |
| source = bare(analysis.text) |
| document = _u_doc(source) |
| if document is None: |
| return [] |
| outputs = [] |
| for token in document: |
| if token.dep_ != "relcl" or token.pos_ not in {"VERB", "AUX"}: |
| continue |
| rel_subject = next( |
| ( |
| child for child in token.children |
| if child.dep_ in {"nsubj", "nsubjpass", "nsubj:pass"} |
| and child.lower_ in {"who", "which", "that"} |
| ), |
| None, |
| ) |
| if rel_subject is None: |
| continue |
| span = list(token.subtree) |
| if not span: |
| continue |
| start = min(item.i for item in span) |
| end = max(item.i for item in span) + 1 |
| reduced = document[start:end].text |
| reduced = re.sub(r"(?i)^\s*(?:who|which|that)\s+", "", reduced) |
| reduced = re.sub(r"(?i)^\s*(?:is|are|was|were|has been|have been|had been)\s+", "", reduced) |
| if not reduced.strip(): |
| continue |
| candidate = _u_clean_join(document[:start].text, reduced, document[end:].text) |
| outputs.extend( |
| _v41_emit_if_safe( |
| source, |
| candidate, |
| "Minimal", |
| "REDUCED_RELATIVE_OR_NP_MODIFIER", |
| "Reduced a dependency-confirmed relative clause while preserving its lexical predicate.", |
| .72, |
| ) |
| ) |
| return outputs |
|
|
|
|
| @universal_rule( |
| "universal_method_adjunct_fronting", |
| {"Balanced", "Structural"}, |
| {"METHOD_RELATION", "USING_METHOD_RELATION", "THROUGH_METHOD_RELATION", "BY_GERUND_METHOD_RELATION"}, |
| .83, |
| ) |
| def tr_universal_method_adjunct_fronting(analysis, profile, user_terms): |
| source = bare(analysis.text) |
| document = _u_doc(source) |
| root = _u_root(document) |
| if document is None or root is None: |
| return [] |
| outputs = [] |
| for child in root.children: |
| if child.dep_ not in {"prep", "obl", "advcl"}: |
| continue |
| marker = child.lower_ |
| if marker not in {"using", "through", "via", "with", "by"}: |
| continue |
| span_tokens = list(child.subtree) |
| if not span_tokens: |
| continue |
| start = min(item.i for item in span_tokens) |
| end = max(item.i for item in span_tokens) + 1 |
| adjunct = document[start:end].text.strip(" ,") |
| main = _u_clause_without_span(document, start, end).strip(" ,") |
| if not adjunct or not main: |
| continue |
| candidate = f"{adjunct[:1].upper() + adjunct[1:]}, {lower_initial(main)}" |
| outputs.extend( |
| _v41_emit_if_safe( |
| source, |
| candidate, |
| "Balanced", |
| "METHOD_RELATION", |
| "Fronted a dependency-confirmed method adjunct without relying on its domain vocabulary.", |
| .83, |
| ) |
| ) |
| return outputs |
|
|
|
|
| @universal_rule( |
| "universal_purpose_clause_relocation", |
| {"Balanced", "Structural"}, |
| {"PURPOSE_INFINITIVE_FRONTING", "PURPOSE_INFINITIVE"}, |
| .84, |
| ) |
| def tr_universal_purpose_clause_relocation(analysis, profile, user_terms): |
| source = bare(analysis.text) |
| document = _u_doc(source) |
| if document is None: |
| return [] |
| outputs = [] |
| for token in document: |
| if token.tag_ != "VB" or token.dep_ not in {"advcl", "xcomp"}: |
| continue |
| marker = next((child for child in token.children if child.lower_ == "to" and child.dep_ in {"aux", "mark"}), None) |
| if marker is None: |
| continue |
| span_tokens = list(token.subtree) |
| span_tokens.append(marker) |
| start = min(item.i for item in span_tokens) |
| end = max(item.i for item in span_tokens) + 1 |
| purpose = document[start:end].text.strip(" ,") |
| main = _u_clause_without_span(document, start, end).strip(" ,") |
| if not purpose.lower().startswith("to ") or not main: |
| continue |
| if start == 0: |
| candidate = f"{main} {lower_initial(purpose)}" |
| else: |
| candidate = f"{purpose[:1].upper() + purpose[1:]}, {lower_initial(main)}" |
| outputs.extend( |
| _v41_emit_if_safe( |
| source, |
| candidate, |
| "Balanced", |
| "PURPOSE_INFINITIVE_FRONTING", |
| "Relocated a dependency-confirmed infinitival purpose clause without requiring a known main verb.", |
| .84, |
| ) |
| ) |
| return outputs |
|
|
|
|
| @universal_rule( |
| "universal_adverbial_clause_reordering", |
| {"Balanced", "Structural"}, |
| {"CONTRAST", "LONG_MULTI_CLAUSE", "WHILE_PARTICIPIAL"}, |
| .78, |
| ) |
| def tr_universal_adverbial_clause_reordering(analysis, profile, user_terms): |
| source = bare(analysis.text) |
| document = _u_doc(source) |
| root = _u_root(document) |
| if document is None or root is None: |
| return [] |
| outputs = [] |
| allowed_markers = {"although", "though", "while", "when", "because", "since", "if", "unless", "after", "before"} |
| for child in root.children: |
| if child.dep_ != "advcl": |
| continue |
| marker = next((item for item in child.children if item.dep_ == "mark"), None) |
| if marker is None or marker.lower_ not in allowed_markers: |
| continue |
| span_tokens = list(child.subtree) |
| start = min(item.i for item in span_tokens) |
| end = max(item.i for item in span_tokens) + 1 |
| subordinate = document[start:end].text.strip(" ,") |
| main = _u_clause_without_span(document, start, end).strip(" ,") |
| if not subordinate or not main: |
| continue |
| if start == 0: |
| candidate = f"{main}; {lower_initial(subordinate)}" |
| else: |
| candidate = f"{subordinate[:1].upper() + subordinate[1:]}, {lower_initial(main)}" |
| outputs.extend( |
| _v41_emit_if_safe( |
| source, |
| candidate, |
| "Structural", |
| "LONG_MULTI_CLAUSE", |
| "Reordered a dependency-confirmed adverbial clause while preserving its explicit relation marker.", |
| .78, |
| ) |
| ) |
| return outputs |
|
|
|
|
| @universal_rule( |
| "universal_prepositional_adjunct_relocation", |
| {"Balanced", "Structural"}, |
| {"FRONTED_ADJUNCT", "LONG_MULTI_CLAUSE"}, |
| .70, |
| ) |
| def tr_universal_prepositional_adjunct_relocation(analysis, profile, user_terms): |
| source = bare(analysis.text) |
| document = _u_doc(source) |
| root = _u_root(document) |
| if document is None or root is None: |
| return [] |
| outputs = [] |
| for child in root.children: |
| if child.dep_ not in {"prep", "obl"}: |
| continue |
| span_tokens = list(child.subtree) |
| if not span_tokens: |
| continue |
| start = min(item.i for item in span_tokens) |
| end = max(item.i for item in span_tokens) + 1 |
| adjunct = document[start:end].text.strip(" ,") |
| if len(adjunct.split()) < 3 or len(adjunct.split()) > 16: |
| continue |
| main = _u_clause_without_span(document, start, end).strip(" ,") |
| if not main: |
| continue |
| candidate = ( |
| f"{main} {lower_initial(adjunct)}" |
| if start == 0 |
| else f"{adjunct[:1].upper() + adjunct[1:]}, {lower_initial(main)}" |
| ) |
| outputs.extend( |
| _v41_emit_if_safe( |
| source, |
| candidate, |
| "Balanced", |
| "FRONTED_ADJUNCT", |
| "Relocated a dependency-confirmed prepositional adjunct without depending on its lexical content.", |
| .70, |
| ) |
| ) |
| return outputs[:2] |
|
|
|
|
| @universal_rule( |
| "universal_coordinated_predicate_split", |
| {"Structural"}, |
| {"COORDINATED_PREDICATES"}, |
| .88, |
| ) |
| def tr_universal_coordinated_predicate_split(analysis, profile, user_terms): |
| source = bare(analysis.text) |
| document = _u_doc(source) |
| root = _u_root(document) |
| subject = _u_subject(root) |
| if document is None or root is None or subject is None or root.pos_ not in {"VERB", "AUX"}: |
| return [] |
| conjuncts = [ |
| token for token in root.conjuncts |
| if token.dep_ == "conj" and token.pos_ in {"VERB", "AUX"} and token.i > root.i |
| ] |
| if len(conjuncts) != 1: |
| return [] |
| conjunct = conjuncts[0] |
| cut = min(item.i for item in conjunct.subtree) |
| left = re.sub(r"(?:,\s*)?\b(?:and|or)\s*$", "", document[:cut].text.strip(), flags=re.I).strip() |
| right = _u_span(conjunct) |
| subject_text = _u_span(subject) |
| if not left or not right or not subject_text: |
| return [] |
| candidate = f"{left}. {subject_text[:1].upper() + subject_text[1:]} {right}" |
| return _v41_emit_if_safe( |
| source, |
| candidate, |
| "Structural", |
| "COORDINATED_PREDICATES", |
| "Split dependency-confirmed coordinated predicates without requiring either predicate verb to be prelisted.", |
| .88, |
| ) |
|
|
|
|
| @universal_rule( |
| "universal_object_enumeration_frame", |
| {"Balanced", "Structural"}, |
| {"FINITE_OBJECT_LIST_FRAME", "COORDINATED_OBJECTS_OR_ENUMERATION", "ENUMERATION"}, |
| .76, |
| ) |
| def tr_universal_object_enumeration_frame(analysis, profile, user_terms): |
| source = bare(analysis.text) |
| document = _u_doc(source) |
| root = _u_root(document) |
| subject = _u_subject(root) |
| objects = _u_objects(root) |
| if document is None or root is None or subject is None or not objects: |
| return [] |
| head = objects[0] |
| conjuncts = [item for item in head.conjuncts if item.dep_ == "conj"] |
| if len(conjuncts) < 2: |
| return [] |
| subject_text = _u_span(subject) |
| object_start = min(item.i for item in head.subtree) |
| object_end = max(max(item.i for item in item.subtree) for item in [head] + conjuncts) + 1 |
| object_text = document[object_start:object_end].text.strip() |
| root_lemma = root.lemma_.lower() |
| |
| |
| candidate = f"The following items are what {lower_initial(subject_text)} {root.text}: {object_text}" |
| if not root_lemma or not object_text: |
| return [] |
| return _v41_emit_if_safe( |
| source, |
| candidate, |
| "Structural", |
| "FINITE_OBJECT_LIST_FRAME", |
| "Reframed a dependency-confirmed coordinated object inventory without deriving an unseen nominalization.", |
| .76, |
| ) |
|
|
|
|
| @universal_rule( |
| "universal_appositive_definition", |
| {"Balanced", "Structural"}, |
| {"APPOSITIVE_DEFINITION", "PROPOSAL_FRAME"}, |
| .82, |
| ) |
| def tr_universal_appositive_definition(analysis, profile, user_terms): |
| source = bare(analysis.text) |
| document = _u_doc(source) |
| if document is None: |
| return [] |
| outputs = [] |
| for token in document: |
| if token.dep_ != "appos" or token.head.pos_ not in {"NOUN", "PROPN"}: |
| continue |
| appos = _u_span(token) |
| head = _u_span(token.head) |
| if not appos or not head: |
| continue |
| span_tokens = list(token.subtree) |
| start = min(item.i for item in span_tokens) |
| end = max(item.i for item in span_tokens) + 1 |
| remainder = _u_clause_without_span(document, start, end).strip(" ,") |
| candidate = f"{remainder}. {head[:1].upper() + head[1:]} is {lower_initial(appos)}" |
| outputs.extend( |
| _v41_emit_if_safe( |
| source, |
| candidate, |
| "Structural", |
| "APPOSITIVE_DEFINITION", |
| "Expanded a dependency-confirmed appositive into an explicit definitional clause without relying on its terminology.", |
| .82, |
| ) |
| ) |
| return outputs |
|
|
|
|
| @universal_rule( |
| "universal_copular_complement_reframe", |
| {"Balanced", "Structural"}, |
| {"COPULAR_ACTION_FRAME", "NOMINALIZATION"}, |
| .68, |
| ) |
| def tr_universal_copular_complement_reframe(analysis, profile, user_terms): |
| source = bare(analysis.text) |
| document = _u_doc(source) |
| root = _u_root(document) |
| if document is None or root is None: |
| return [] |
| copula = next((child for child in root.children if child.dep_ == "cop"), None) |
| subject = _u_subject(root) |
| if copula is None or subject is None or root.pos_ not in {"NOUN", "ADJ", "PROPN"}: |
| return [] |
| subject_text = _u_span(subject) |
| complement = _u_span(root) |
| if not subject_text or not complement: |
| return [] |
| candidate = f"{complement[:1].upper() + complement[1:]} describes {lower_initial(subject_text)}" |
| return _v41_emit_if_safe( |
| source, |
| candidate, |
| "Structural", |
| "COPULAR_ACTION_FRAME", |
| "Reframed a dependency-confirmed copular complement without requiring a known complement noun or adjective.", |
| .68, |
| ) |
|
|
|
|
| @universal_rule( |
| "universal_reporting_complement", |
| {"Balanced", "Structural"}, |
| {"CONTENT_COMPLEMENT", "EVIDENTIAL_COMPLEMENT"}, |
| .74, |
| ) |
| def tr_universal_reporting_complement(analysis, profile, user_terms): |
| source = bare(analysis.text) |
| document = _u_doc(source) |
| root = _u_root(document) |
| subject = _u_subject(root) |
| if document is None or root is None or subject is None: |
| return [] |
| complement = next((child for child in root.children if child.dep_ in {"ccomp", "xcomp"}), None) |
| if complement is None: |
| return [] |
| marker = next((child for child in complement.children if child.dep_ == "mark"), None) |
| claim = _u_span(complement) |
| subject_text = _u_span(subject) |
| if not claim or not subject_text: |
| return [] |
| if marker is not None and marker.lower_ == "that": |
| claim = re.sub(r"(?i)^that\s+", "", claim).strip() |
| candidate = f"According to {lower_initial(subject_text)}, {claim}" |
| return _v41_emit_if_safe( |
| source, |
| candidate, |
| "Balanced", |
| "CONTENT_COMPLEMENT", |
| "Reorganized a dependency-confirmed reporting complement without requiring a prelisted reporting verb.", |
| .74, |
| ) |
|
|
|
|
| |
| if not hasattr(Generator, "_nll_v5_base_lane"): |
| Generator._nll_v5_base_lane = Generator.lane |
|
|
|
|
| def _v5_lane(self, analysis, profile, user_terms, lane, trace): |
| candidates = Generator._nll_v5_base_lane( |
| self, |
| analysis, |
| profile, |
| user_terms, |
| lane, |
| trace, |
| ) |
| return list(candidates)[:UNIVERSAL_MAX_CANDIDATES_PER_LANE] |
|
|
|
|
| Generator.lane = _v5_lane |
|
|
|
|
| |
| |
| |
|
|
| _V5_EXPECTED_UNIVERSAL_RULES = { |
| "universal_explicit_agent_passive", |
| "universal_participial_modifier", |
| "universal_relative_clause_reduction", |
| "universal_method_adjunct_fronting", |
| "universal_purpose_clause_relocation", |
| "universal_adverbial_clause_reordering", |
| "universal_prepositional_adjunct_relocation", |
| "universal_coordinated_predicate_split", |
| "universal_object_enumeration_frame", |
| "universal_appositive_definition", |
| "universal_copular_complement_reframe", |
| "universal_reporting_complement", |
| } |
|
|
| _v5_rule_names = [registered.name for registered in RULES] |
| _v5_duplicates = sorted( |
| name for name, count in Counter(_v5_rule_names).items() if count > 1 |
| ) |
| _v5_missing_universal = sorted(_V5_EXPECTED_UNIVERSAL_RULES - set(_v5_rule_names)) |
|
|
| if _v5_duplicates: |
| raise RuntimeError("Cell 5 v5 duplicate rules: " + ", ".join(_v5_duplicates)) |
| if _v5_missing_universal: |
| raise RuntimeError("Cell 5 v5 missing universal rules: " + ", ".join(_v5_missing_universal)) |
| if len(RULES) != 42: |
| raise RuntimeError("Cell 5 v5 expected 42 rules but found " + str(len(RULES))) |
| if _u_inflect("calibrate", "VBD") not in {"calibrated"}: |
| raise RuntimeError("Cell 5 v5 generic past-tense realization failed.") |
| if _u_inflect("triangulate", "VBZ") not in {"triangulates"}: |
| raise RuntimeError("Cell 5 v5 generic present-tense realization failed.") |
|
|
| print("=" * 84) |
| print("CELL 5 UNIVERSAL DEPENDENCY TRANSFORMATION ENGINE v5 LOADED") |
| print("=" * 84) |
| print("Validated specialized rules retained: 30") |
| print("Universal dependency rules added:", len(_V5_EXPECTED_UNIVERSAL_RULES)) |
| print("Total registered transformation rules:", len(RULES)) |
| print("Expected rule count: 42") |
| print("Vocabulary-independent passive handling: active") |
| print("Vocabulary-independent participial handling: active") |
| print("Vocabulary-independent adjunct handling: active") |
| print("Vocabulary-independent clause handling: active") |
| print("Vocabulary-independent coordination handling: active") |
| print("Unicode-safe parsing delegated to spaCy pipeline: active") |
| print("Universal lane candidate cap:", UNIVERSAL_MAX_CANDIDATES_PER_LANE) |
| print("Missing universal rules:", _v5_missing_universal) |
| print("Duplicate rule names:", _v5_duplicates) |
| print("Universal morphology invariants: PASSED") |
| print("=" * 84) |
|
|
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
|
|
| import re |
| from collections import Counter |
|
|
|
|
| |
| |
| |
|
|
| _REQUIRED_5A = [ |
| "RULES", "rule", "Candidate", "Generator", "clean", "K", |
| "lower_initial", "surface_key", "nlp", |
| ] |
|
|
| _missing_5a = [name for name in _REQUIRED_5A if name not in globals()] |
| if _missing_5a: |
| raise RuntimeError( |
| "Cell 5A cannot run. Missing: " + ", ".join(_missing_5a) |
| + ". Run the complete Cell 5 v5 first." |
| ) |
|
|
| EXPECTED_5A_RULE_COUNT = 42 |
| if len(RULES) != EXPECTED_5A_RULE_COUNT: |
| raise RuntimeError( |
| f"Cell 5A expected {EXPECTED_5A_RULE_COUNT} Cell 5 v5 rules, " |
| f"but found {len(RULES)}. Rerun Cell 5 v5 first." |
| ) |
|
|
|
|
| |
| |
| |
|
|
| _SUBJECT_PRONOUNS = {"i", "we", "he", "she", "they", "who"} |
| _OBJECT_PRONOUNS = {"me", "us", "him", "her", "them", "whom"} |
| _FINITE_TAGS = {"VBD", "VBP", "VBZ", "MD"} |
| _DECLARATIVE_STARTERS = { |
| "i", "we", "you", "he", "she", "it", "they", |
| "this", "that", "these", "those", "the", "a", "an", |
| } |
| _RELATIVE_MARKERS = {"that", "which", "who", "whose", "where"} |
|
|
|
|
| def _r34_split_sentences(text): |
| """Return bounded sentence units without modifying source content.""" |
| value = clean(text) |
| try: |
| document = nlp()(value) |
| units = [sent.text.strip() for sent in document.sents if sent.text.strip()] |
| if units: |
| return units |
| except Exception: |
| pass |
|
|
| return [ |
| part.strip() |
| for part in re.split(r"(?<=[.!?])\s+", value) |
| if part.strip() |
| ] |
|
|
|
|
| def _r34_parse(text): |
| try: |
| document = nlp()(str(text or "")) |
| return document if len(document) else None |
| except Exception: |
| return None |
|
|
|
|
| def _r45_recover_misparsed_finite_predicate(document): |
| """Recover a finite predicate from a known parser ambiguity. |
| |
| Some English past-tense forms are surface-identical to past participles. |
| A parser can therefore analyze a normal clause such as |
| ``The model operated within its established configuration`` as a noun ROOT |
| (model) plus an ``acl`` VBN (operated). This helper accepts that structure |
| only when strong clause evidence is present and passive/reduced-relative |
| evidence is absent. |
| """ |
| if document is None: |
| return False |
|
|
| |
| |
| annotation_owner = document if hasattr(document, "has_annotation") else getattr(document, "doc", None) |
| if annotation_owner is None or not annotation_owner.has_annotation("DEP"): |
| return False |
|
|
| tokens = list(document) |
| root = next((token for token in tokens if token.dep_ == "ROOT"), None) |
| if root is None or root.pos_ not in {"NOUN", "PROPN", "PRON"}: |
| return False |
|
|
| |
| has_nominal_anchor = any( |
| child.dep_ in {"det", "poss", "compound", "amod"} |
| for child in root.children |
| ) or root.i == 0 |
| if not has_nominal_anchor: |
| return False |
|
|
| candidates = [ |
| child for child in root.children |
| if child.dep_ in {"acl", "relcl"} |
| and child.pos_ == "VERB" |
| and child.tag_ in {"VBD", "VBN"} |
| and child.i > root.i |
| ] |
|
|
| for verb in candidates: |
| |
| |
| child_deps = {child.dep_ for child in verb.children} |
| has_passive_evidence = bool( |
| {"auxpass", "aux:pass", "agent", "obl:agent"} & child_deps |
| ) or any( |
| child.lower_ == "by" and child.dep_ in {"prep", "agent"} |
| for child in verb.children |
| ) |
| if has_passive_evidence: |
| continue |
|
|
| |
| |
| has_predicate_complement = any( |
| child.dep_ in { |
| "obj", "dobj", "attr", "oprd", "prep", "obl", "advmod", |
| "advcl", "ccomp", "xcomp", "dative", |
| } |
| for child in verb.children |
| ) |
| if has_predicate_complement: |
| return True |
|
|
| return False |
|
|
|
|
| def _r34_has_finite_predicate(document): |
| """Detect finite predicates by syntax before trusting ambiguous tags. |
| |
| English present-tense verbs with plural subjects are surface-identical to |
| the bare infinitive. spaCy may therefore tag a finite root such as |
| ``Models ... inform ...`` as VB/VerbForm=Inf. A ROOT verb with an explicit |
| subject and no infinitival marker is treated as finite. This is structural, |
| vocabulary-independent, and does not depend on a verb dictionary. |
| """ |
| if document is None: |
| return False |
|
|
| if _r45_recover_misparsed_finite_predicate(document): |
| return True |
|
|
| tokens = list(document) |
| for token in tokens: |
| if token.tag_ in _FINITE_TAGS: |
| return True |
|
|
| try: |
| if "Fin" in token.morph.get("VerbForm"): |
| return True |
| except Exception: |
| pass |
|
|
| if token.pos_ in {"VERB", "AUX"} and any( |
| child.dep_ in {"aux", "auxpass", "aux:pass"} |
| and ( |
| child.tag_ in _FINITE_TAGS |
| or "Fin" in child.morph.get("VerbForm") |
| ) |
| for child in token.children |
| ): |
| return True |
|
|
| if token.dep_ == "ROOT" and token.pos_ in {"VERB", "AUX"}: |
| subjects = [ |
| child for child in token.children |
| if child.dep_ in {"nsubj", "nsubjpass", "nsubj:pass", "csubj", "expl"} |
| ] |
| infinitive_markers = [ |
| child for child in token.children |
| if child.lower_ == "to" and child.dep_ in {"aux", "mark"} |
| ] |
| |
| |
| |
| if subjects and not infinitive_markers: |
| return True |
|
|
| verb_form = set() |
| try: |
| verb_form = set(token.morph.get("VerbForm")) |
| except Exception: |
| pass |
| if token.tag_ not in {"VBG", "VBN", "VB"} and not ({"Inf", "Ger", "Part"} & verb_form): |
| return True |
|
|
| return False |
|
|
|
|
| def _r34_relative_clause_is_finite(document): |
| """Validate finite predicates after explicit relative markers.""" |
| if document is None: |
| return True |
|
|
| for token in document: |
| if token.lower_ not in _RELATIVE_MARKERS: |
| continue |
|
|
| following = document[token.i + 1:] |
| if not following: |
| return False |
|
|
| if not _r34_has_finite_predicate(following): |
| return False |
|
|
| return True |
|
|
|
|
| def _r34_orphan_participial_fragment(sentence): |
| """Detect vocabulary-independent orphan VBG/VBN modifier fragments. |
| |
| The check does not depend on a list such as spanning/covering/including. |
| It uses dependency labels, verb tags, finite-predicate evidence, and the |
| presence of an explicit relative-clause marker. |
| """ |
| value = str(sentence or "").strip() |
| low = value.lower() |
| document = _r34_parse(value) |
|
|
| if document is not None and document.has_annotation("DEP"): |
| if _r45_recover_misparsed_finite_predicate(document): |
| return False |
| root = next((token for token in document if token.dep_ == "ROOT"), None) |
| if root is not None and root.pos_ in {"VERB", "AUX"}: |
| subjects = [ |
| child for child in root.children |
| if child.dep_ in {"nsubj", "nsubjpass", "nsubj:pass", "csubj", "expl"} |
| ] |
| infinitive_markers = [ |
| child for child in root.children |
| if child.lower_ == "to" and child.dep_ in {"aux", "mark"} |
| ] |
| if subjects and not infinitive_markers: |
| return False |
| if _r34_has_finite_predicate(document): |
| return False |
|
|
| |
| |
| |
| for marker in document: |
| if marker.lower_ not in _RELATIVE_MARKERS: |
| continue |
| following = document[marker.i + 1:] |
| if any(token.tag_ in _FINITE_TAGS or token.pos_ == "AUX" for token in following): |
| return False |
|
|
| participles = [ |
| token for token in document |
| if token.tag_ in {"VBG", "VBN"} |
| and token.dep_ in {"ROOT", "acl", "relcl", "advcl", "amod"} |
| ] |
| if participles and not _r34_has_finite_predicate(document): |
| return True |
|
|
| |
| |
| |
| |
| return False |
|
|
| |
| |
| |
| |
| if re.search( |
| r"(?iu)\b(?:that|which|who)\s+[^\W\d_]+(?:ing|ed)\b", |
| value, |
| ): |
| return True |
| if re.search( |
| r"(?iu)\b(?:am|is|are|was|were|has|have|had)\s+" |
| r"(?:am|is|are|was|were|has|have|had)\b", |
| value, |
| ): |
| return True |
| return False |
|
|
|
|
| def _r34_clause_issues(text): |
| issues = [] |
|
|
| for index, sentence in enumerate(_r34_split_sentences(text), start=1): |
| words = re.findall(r"[^\W\d_]+(?:[-’'][^\W\d_]+)*", sentence, flags=re.UNICODE) |
| if not words: |
| issues.append(f"EMPTY_SENTENCE_{index}") |
| continue |
|
|
| first = words[0].lower() |
| document = _r34_parse(sentence) |
|
|
| if first in _DECLARATIVE_STARTERS and not _r34_has_finite_predicate(document): |
| issues.append(f"NONFINITE_DECLARATIVE_FRAGMENT_{index}") |
|
|
| if document is not None: |
| root = next((token for token in document if token.dep_ == "ROOT"), None) |
| if ( |
| first in _DECLARATIVE_STARTERS |
| and root is not None |
| and root.pos_ not in {"VERB", "AUX"} |
| and not _r34_has_finite_predicate(document) |
| ): |
| issues.append(f"NOMINAL_ROOT_FRAGMENT_{index}") |
|
|
| if not _r34_relative_clause_is_finite(document): |
| issues.append(f"NONFINITE_RELATIVE_CLAUSE_{index}") |
| else: |
| if re.match( |
| r"(?is)^(?:I|We|You|He|She|It|They|The|A|An|This|That|These|Those)\s+" |
| r"(?:[A-Z][\w-]*\s+){1,8}" |
| r"(?:Framework|System|Model|Protocol|Method|Procedure|Process|Team|Study)\b", |
| sentence, |
| ): |
| issues.append(f"LIKELY_NOMINAL_FRAGMENT_{index}") |
|
|
| if _r34_orphan_participial_fragment(sentence): |
| issues.append(f"ORPHAN_PARTICIPIAL_FRAGMENT_{index}") |
|
|
| return issues |
|
|
|
|
| def _r34_balanced_delimiters(text): |
| pairs = {"(": ")", "[": "]", "{": "}"} |
| stack = [] |
| for char in str(text or ""): |
| if char in pairs: |
| stack.append(pairs[char]) |
| elif char in pairs.values(): |
| if not stack or stack.pop() != char: |
| return False |
| return not stack |
|
|
|
|
| def _r4_dependency_realization_issues(text): |
| """Vocabulary-independent dependency and morphology realization checks.""" |
| issues = [] |
| document = _r34_parse(text) |
| if document is None or not document.has_annotation("DEP"): |
| return issues |
|
|
| for sentence_index, sentence in enumerate(document.sents, start=1): |
| root = next((token for token in sentence if token.dep_ == "ROOT"), None) |
| if root is None: |
| issues.append(f"MISSING_ROOT_{sentence_index}") |
| continue |
|
|
| |
| |
| token_count = sum(not token.is_punct and not token.is_space for token in sentence) |
| recovered_finite = _r45_recover_misparsed_finite_predicate(sentence) |
| if ( |
| token_count >= 3 |
| and root.pos_ not in {"VERB", "AUX"} |
| and not recovered_finite |
| ): |
| first = next( |
| (token for token in sentence if not token.is_punct and not token.is_space), |
| None, |
| ) |
| if ( |
| first is not None |
| and first.lower_ in _DECLARATIVE_STARTERS |
| and not _r34_has_finite_predicate(sentence) |
| ): |
| issues.append(f"NONVERBAL_DECLARATIVE_ROOT_{sentence_index}") |
|
|
| |
| |
| for token in sentence: |
| if token.lower_ in _RELATIVE_MARKERS: |
| tail = sentence.doc[token.i + 1:sentence.end] |
| if not _r34_has_finite_predicate(tail): |
| issues.append(f"NONFINITE_RELATIVE_CLAUSE_{sentence_index}") |
| break |
|
|
| |
| |
| previous_aux = None |
| for token in sentence: |
| if token.pos_ == "AUX": |
| if previous_aux is not None and previous_aux.lemma_ == token.lemma_: |
| issues.append(f"DUPLICATED_AUXILIARY_SEQUENCE_{sentence_index}") |
| break |
| previous_aux = token |
| elif not token.is_punct: |
| previous_aux = None |
|
|
| return issues |
|
|
|
|
| def _r44_possessive_determiner_issues(text): |
| """Detect high-confidence use of object pronoun *it* where *its* is required. |
| |
| Example rejected: |
| "under it prespecified positive-mean criterion" |
| |
| This is a rejection check, not an automatic correction. The generator is |
| allowed to try another candidate instead of silently changing content. |
| """ |
| issues = [] |
| document = _r34_parse(text) |
|
|
| if document is not None and document.has_annotation("DEP"): |
| tokens = list(document) |
| for index, token in enumerate(tokens[:-2]): |
| if token.lower_ != "it" or token.pos_ != "PRON": |
| continue |
|
|
| following = tokens[index + 1:index + 5] |
| |
| |
| |
| |
| modifier_count = 0 |
| noun_found = False |
| for item in following: |
| if item.pos_ in {"ADJ"} or item.tag_ in {"VBN", "VBG"}: |
| modifier_count += 1 |
| continue |
| if item.pos_ in {"NOUN", "PROPN"} and modifier_count: |
| noun_found = True |
| break |
|
|
| previous = tokens[index - 1] if index > 0 else None |
| if ( |
| noun_found |
| and previous is not None |
| and previous.lower_ in { |
| "under", "within", "according", "based", "from", "in", |
| "on", "through", "with", "by", "after", "before", |
| } |
| ): |
| issues.append("OBJECT_PRONOUN_IN_POSSESSIVE_DETERMINER_SLOT") |
| break |
|
|
| |
| |
| |
| if re.search( |
| r"(?iu)\b(?:under|within|according\s+to|based\s+on|from|in|on|through|with|by)\s+" |
| r"it\s+(?:[^\W\d_]+(?:[-’'][^\W\d_]+)*\s+){1,3}" |
| r"(?:criterion|criteria|framework|procedure|configuration|method|model|" |
| r"policy|protocol|design|analysis|assessment|evaluation|standard|rule|" |
| r"requirement|approach|context|structure|setting|settings)\b", |
| str(text or ""), |
| ): |
| issues.append("OBJECT_PRONOUN_IN_POSSESSIVE_DETERMINER_SLOT") |
|
|
| return list(dict.fromkeys(issues)) |
|
|
|
|
| def _r44_result_adjunct_punctuation_issues(text): |
| """Detect a missing comma before a supplementary 'thereby + VBG' result.""" |
| issues = [] |
| for sentence in _r34_split_sentences(text): |
| match = re.search(r"(?iu)\bthereby\s+[^\W\d_]+ing\b", sentence) |
| if not match: |
| continue |
| prefix = sentence[:match.start()].rstrip() |
| |
| |
| if ( |
| len(re.findall(r"[^\W\d_]+", prefix, flags=re.UNICODE)) >= 5 |
| and not prefix.endswith((",", ";", ":")) |
| ): |
| issues.append("MISSING_COMMA_BEFORE_THEREBY_RESULT_ADJUNCT") |
| return issues |
|
|
|
|
| def realization_issues(text): |
| """Return only high-confidence grammatical/realization defects.""" |
| value = clean(text) |
| low = value.lower() |
| issues = [] |
|
|
| if not value: |
| return ["EMPTY_OUTPUT"] |
|
|
| if not _r34_balanced_delimiters(value): |
| issues.append("UNBALANCED_DELIMITERS") |
|
|
| if re.search( |
| r"\b(?:for|to|with|by|from|of|about)\s+(?:i|we|he|she|they|who)\b", |
| low, |
| ): |
| issues.append("SUBJECT_PRONOUN_AFTER_PREPOSITION") |
|
|
| if re.match( |
| r"^(?:me|us|him|her|them|whom)\s+" |
| r"(?:am|is|are|was|were|must|should|can|could|may|might|will|would)\b", |
| low, |
| ): |
| issues.append("OBJECT_PRONOUN_AS_SUBJECT") |
|
|
| if re.match( |
| r"^(?:inspect|distinguish|evaluate|examine|analyze|compare|apply|use|" |
| r"test|measure|determine|identify|review|assess|implement|investigate)" |
| r"\b.+\s+is\s+the\s+(?:next|first|second|third|final|primary|main)\s+" |
| r"(?:step|task|action|objective|goal)\b", |
| low, |
| ): |
| issues.append("BARE_ACTION_AS_NOMINAL_SUBJECT") |
|
|
| if re.search(r"\b(?:is|are|was|were)\s+(?:is|are|was|were)\b", low): |
| issues.append("DUPLICATED_COPULA") |
|
|
| if re.search(r"\b(?:has|have|had)\s+been\s+been\b", low): |
| issues.append("DUPLICATED_AUXILIARY") |
|
|
| if re.search(r"\b(?:a|an)\s+(?:are|were|have|do)\b", low): |
| issues.append("DETERMINER_BEFORE_FINITE_VERB") |
|
|
| if re.search( |
| r"\b(?:of|for|with|by|from|to|in|on|under|within|across)\s+" |
| r"(?:The|A|An)\s+\w+", |
| value, |
| ): |
| issues.append("INTERNAL_CAPITALIZED_DETERMINER") |
|
|
| if re.search(r"\s+[,.!?;:]", value): |
| issues.append("SPACE_BEFORE_PUNCTUATION") |
|
|
| if re.search(r"[.!?]{2,}", value): |
| issues.append("DUPLICATED_TERMINAL_PUNCTUATION") |
|
|
| if len(re.findall(r"\b\w+\b", value)) < 3: |
| issues.append("TOO_SHORT") |
|
|
| issues.extend(_r34_clause_issues(value)) |
| issues.extend(_r4_dependency_realization_issues(value)) |
| issues.extend(_r44_possessive_determiner_issues(value)) |
| issues.extend(_r44_result_adjunct_punctuation_issues(value)) |
| return list(dict.fromkeys(issues)) |
|
|
|
|
| |
| |
| |
|
|
| @rule( |
| "copular_action_frame", |
| {"Balanced", "Structural"}, |
| {"COPULAR_ACTION_FRAME"}, |
| .96, |
| ) |
| def tr_copular_action_reliable(analysis, profile, user_terms): |
| match = re.match( |
| r"(?is)^The\s+(?P<order>next|first|second|third|final|primary|main)\s+" |
| r"(?P<label>step|task|action|objective|goal)\s+is\s+to\s+" |
| r"(?P<action>.+?)[.]?$", |
| analysis.text, |
| ) |
| if not match: |
| return [] |
|
|
| action = match["action"].strip() |
| first_match = re.match(r"(?is)^(?P<verb>[A-Za-z-]+)(?P<rest>.*)$", action) |
| if not first_match: |
| return [] |
|
|
| verb = first_match["verb"] |
| rest = first_match["rest"] |
| low = verb.lower() |
| irregular = { |
| "be": "being", "have": "having", "begin": "beginning", |
| "run": "running", "set": "setting", "get": "getting", |
| } |
|
|
| if low in irregular: |
| gerund = irregular[low] |
| elif low.endswith("ie"): |
| gerund = verb[:-2] + "ying" |
| elif low.endswith("e") and not low.endswith(("ee", "ye", "oe")): |
| gerund = verb[:-1] + "ing" |
| else: |
| gerund = verb + "ing" |
|
|
| candidate = K( |
| f"The {match['order']} {match['label']} involves {gerund}{rest}", |
| "Balanced", |
| "COPULAR_ACTION_FRAME", |
| "Converted a copular action frame into a grammatical event frame.", |
| .96, |
| ) |
| return [] if realization_issues(candidate.text) else [candidate] |
|
|
|
|
| @rule( |
| "need_to_action", |
| {"Balanced", "Structural"}, |
| {"NEED_TO_ACTION"}, |
| .94, |
| ) |
| def tr_need_to_reliable(analysis, profile, user_terms): |
| match = re.match( |
| r"(?is)^(?P<subject>.+?)\s+(?P<need>need|needs|needed)\s+to\s+" |
| r"(?P<action>.+?)[.]?$", |
| analysis.text, |
| ) |
| if not match: |
| return [] |
|
|
| modal = "had to" if match["need"].lower() == "needed" else "must" |
| candidate = K( |
| f"{match['subject'].strip()} {modal} {match['action'].strip()}", |
| "Balanced", |
| "NEED_TO_ACTION", |
| "Recast a need-to construction as a subject-preserving necessity frame.", |
| .94, |
| ) |
| return [] if realization_issues(candidate.text) else [candidate] |
|
|
|
|
| @rule( |
| "modal_copular_enumeration", |
| {"Balanced", "Structural"}, |
| {"MODAL_COPULAR_ENUMERATION", "COORDINATED_OBJECTS_OR_ENUMERATION", "ENUMERATION"}, |
| .95, |
| ) |
| def tr_modal_copular_enumeration_reliable(analysis, profile, user_terms): |
| match = re.match( |
| r"(?is)^(?P<subject>.+?)\s+(?P<modal>may|might|could|can)\s+be\s+in\s+" |
| r"(?P<items>.+?,\s*.+?\s*,?\s*(?:or|and)\s+.+?)[.]?$", |
| analysis.text, |
| ) |
| if not match: |
| return [] |
|
|
| candidate = K( |
| f"Possible sources of {lower_initial(match['subject'])} " |
| f"{match['modal'].lower()} include {match['items']}", |
| "Balanced", |
| "MODAL_COPULAR_ENUMERATION", |
| "Recast a modal copular list while preserving uncertainty and every member.", |
| .95, |
| ) |
| return [] if realization_issues(candidate.text) else [candidate] |
|
|
|
|
| @rule( |
| "passive_using_nominal", |
| {"Balanced", "Structural"}, |
| {"PASSIVE_USING"}, |
| .96, |
| ) |
| def tr_passive_using_reliable(analysis, profile, user_terms): |
| match = re.match( |
| r"(?is)^(?:(?P<marker>However|Therefore|Consequently|Thus|Moreover|" |
| r"Furthermore|Nevertheless|Accordingly),\s+)?" |
| r"(?P<subject>.+?)\s+" |
| r"(?:is|are|was|were|has been|have been|had been)\s+" |
| r"(?P<adverb>\w+ly\s+)?evaluated\s+using\s+" |
| r"(?P<object>.+?)[.]?$", |
| analysis.text, |
| ) |
| if not match: |
| return [] |
|
|
| marker = (match.group("marker") or "").strip() |
| adverb = (match.group("adverb") or "").strip() |
| prefix = f"{marker}, " if marker else "" |
| adverb_text = f"{adverb} " if adverb else "" |
|
|
| candidate = K( |
| f"{prefix}Evaluations of {lower_initial(match['subject'])} " |
| f"{adverb_text}use {match['object']}", |
| "Balanced", |
| "PASSIVE_USING", |
| "Recast a passive evaluation as an evaluation event without inventing an actor.", |
| .96, |
| ) |
| return [] if realization_issues(candidate.text) else [candidate] |
|
|
|
|
| |
| |
| |
|
|
| if not hasattr(Generator, "_nll_reliability_base_lane"): |
| Generator._nll_reliability_base_lane = Generator.lane |
| else: |
| Generator.lane = Generator._nll_reliability_base_lane |
|
|
|
|
| def _lane_with_realization_filter(self, analysis, profile, user_terms, lane, trace): |
| candidates = Generator._nll_reliability_base_lane( |
| self, analysis, profile, user_terms, lane, trace |
| ) |
| accepted = [] |
|
|
| for candidate in candidates: |
| issues = realization_issues(candidate.text) |
| if issues: |
| trace.setdefault("realization_rejections", []).append({ |
| "stage": "lane", |
| "rule": getattr(candidate, "family", ""), |
| "lane": getattr(candidate, "lane", lane), |
| "text": getattr(candidate, "text", ""), |
| "issues": issues, |
| }) |
| continue |
| accepted.append(candidate) |
|
|
| return accepted |
|
|
|
|
| Generator.lane = _lane_with_realization_filter |
|
|
|
|
| |
| |
| |
|
|
| if not hasattr(Generator, "_nll_reliability_base_generate"): |
| Generator._nll_reliability_base_generate = Generator.generate |
| else: |
| Generator.generate = Generator._nll_reliability_base_generate |
|
|
|
|
| def _generate_with_final_realization_filter( |
| self, analysis, profile, user_terms, mode |
| ): |
| candidates, trace = Generator._nll_reliability_base_generate( |
| self, analysis, profile, user_terms, mode |
| ) |
|
|
| accepted = [] |
| seen = set() |
|
|
| for candidate in candidates: |
| text = getattr(candidate, "text", "") |
| issues = realization_issues(text) |
| if issues: |
| trace.setdefault("realization_rejections", []).append({ |
| "stage": "final_pool", |
| "rule": getattr(candidate, "family", ""), |
| "lane": getattr(candidate, "lane", mode), |
| "text": text, |
| "issues": issues, |
| }) |
| continue |
|
|
| key = surface_key(text) |
| if not key or key in seen: |
| continue |
| seen.add(key) |
| accepted.append(candidate) |
|
|
| trace["generated_before_final_filter"] = len(candidates) |
| trace["generated_after_final_filter"] = len(accepted) |
| trace["generated"] = len(accepted) |
| return accepted, trace |
|
|
|
|
| Generator.generate = _generate_with_final_realization_filter |
|
|
|
|
| |
| |
| |
|
|
| _BAD_REALIZATIONS = [ |
| "Inspect how the handler works is the next step.", |
| "Distinguish the possibilities is necessary for we.", |
| "For they, the review is necessary.", |
| "The system is is evaluated.", |
| "Evaluations of The service use latency measurements.", |
| "We propose. We Adaptive Learning and Competency Evaluation Framework.", |
| "Protocol to small cybersecurity learners spanning classification, drafting, and planning.", |
| "The learners that spanning classification, drafting, and planning.", |
| "The learners that classification, drafting, and planning.", |
| "Sensors calibrating atmospheric pressure.", |
| "The devices that calibrating atmospheric pressure.", |
| "The alloy was was cryogenically stabilized.", |
| "H3 was supported under it prespecified positive-mean criterion.", |
| "The model operated within it established configuration.", |
| "The conditions met the prespecified thresholds thereby supporting H1.", |
| "The model operated by the laboratory.", |
| ] |
|
|
| _GOOD_REALIZATIONS = [ |
| "The next step involves inspecting how the handler works.", |
| "We must distinguish the possibilities using the existing source.", |
| "Possible sources of the problem could include lookup, initialization, or reporting logic.", |
| "Evaluations of the service use latency, throughput, and failure-rate measurements.", |
| "We propose the Adaptive Learning and Competency Evaluation Framework.", |
| "The experimental testbed applies this protocol to small cybersecurity learners that span classification, generative drafting, and structured diagram planning.", |
| "ALCEF-RSF is a domain-independent orchestration protocol consisting of native model contracts, protected examinations, and auditable promotion proposed by us.", |
| "Sensors that calibrate atmospheric pressure support the monitoring system.", |
| "The laboratory cryogenically stabilized the unfamiliar alloy.", |
| "Models predicting regional demand inform the allocation process.", |
| "Systems processing satellite imagery support emergency planning.", |
| "Devices monitoring volatile compounds trigger the ventilation system.", |
| "Contracts governing equipment maintenance define the service obligations.", |
| "Teams reconciling conflicting records produce the final ledger.", |
| "Plants decarbonizing industrial heat reduce lifecycle emissions.", |
| "Systems that triangulate satellite signals improve positional accuracy.", |
| "H3 was supported under its prespecified positive-mean criterion.", |
| "The model operated within its established configuration.", |
| "The controller operated under its documented procedure.", |
| "The system functioned within its approved settings.", |
| "The conditions met the prespecified thresholds, thereby supporting H1.", |
| "The team kept it available for review.", |
| ] |
|
|
| _RECOVERY_SENTENCE = "The model operated within its established configuration." |
| _recovery_doc = _r34_parse(_RECOVERY_SENTENCE) |
| if not _r45_recover_misparsed_finite_predicate(_recovery_doc): |
| raise RuntimeError( |
| "Cell 5A v4.7 failed: noun-root finite-predicate recovery did not activate." |
| ) |
| if not _r34_has_finite_predicate(_recovery_doc): |
| raise RuntimeError( |
| "Cell 5A v4.7 failed: recovered predicate was not treated as finite." |
| ) |
| _recovery_span = next(iter(_recovery_doc.sents), None) if _recovery_doc is not None else None |
| if _recovery_span is None or not _r45_recover_misparsed_finite_predicate(_recovery_span): |
| raise RuntimeError( |
| "Cell 5A v4.7 failed: sentence-Span finite-predicate recovery did not activate." |
| ) |
| if _r4_dependency_realization_issues(_RECOVERY_SENTENCE): |
| raise RuntimeError( |
| "Cell 5A v4.7 failed: dependency realization layer ignored recovered predicate: " |
| + str(_r4_dependency_realization_issues(_RECOVERY_SENTENCE)) |
| ) |
|
|
| for bad_text in _BAD_REALIZATIONS: |
| if not realization_issues(bad_text): |
| raise RuntimeError( |
| "Cell 5A v4.7 failed to reject malformed text: " + bad_text |
| ) |
|
|
| for good_text in _GOOD_REALIZATIONS: |
| detected = realization_issues(good_text) |
| if detected: |
| debug_doc = _r34_parse(good_text) |
| debug_tokens = [] |
| if debug_doc is not None: |
| debug_tokens = [ |
| { |
| "text": token.text, |
| "pos": token.pos_, |
| "tag": token.tag_, |
| "dep": token.dep_, |
| "head": token.head.text, |
| "verb_form": list(token.morph.get("VerbForm")), |
| } |
| for token in debug_doc |
| ] |
| raise RuntimeError( |
| "Cell 5A v4.7 incorrectly rejected valid text: " |
| + good_text + " -> " + str(detected) |
| + " | parse=" + str(debug_tokens) |
| ) |
|
|
| _rule_names = [registered.name for registered in RULES] |
| _duplicates = sorted( |
| name for name, count in Counter(_rule_names).items() if count > 1 |
| ) |
| if _duplicates: |
| raise RuntimeError( |
| "Cell 5A v4.7 duplicate rule registration: " + ", ".join(_duplicates) |
| ) |
| if len(RULES) != EXPECTED_5A_RULE_COUNT: |
| raise RuntimeError( |
| f"Cell 5A v4.7 expected {EXPECTED_5A_RULE_COUNT} rules but found {len(RULES)}." |
| ) |
|
|
| if getattr(Generator.lane, "__name__", "") != "_lane_with_realization_filter": |
| raise RuntimeError("Cell 5A v4.7 lane wrapper was not installed.") |
| if getattr(Generator.generate, "__name__", "") != "_generate_with_final_realization_filter": |
| raise RuntimeError("Cell 5A v4.7 generate wrapper was not installed.") |
|
|
| print("=" * 78) |
| print("CELL 5A UNIVERSAL CANDIDATE RELIABILITY v4.7 LOADED") |
| print("=" * 78) |
| print("Aligned Cell 5 version: v5") |
| print("Finite declarative-clause checks: active") |
| print("Finite relative-clause checks: active") |
| print("Vocabulary-independent VBG/VBN checks: active") |
| print("Finite-root precedence over participial fallback: active") |
| print("VerbForm + ROOT finite-predicate detection: active") |
| print("Finite bare-root + explicit-subject disambiguation: active") |
| print("Misparsed noun-root + finite-predicate recovery: active") |
| print("Cross-layer recovered-predicate consistency: active") |
| print("Doc/Span dependency-annotation compatibility: active") |
| print("Passive/reduced-relative recovery guard: active") |
| print("High-precision no-parse fallback: active") |
| print("Dependency-root realization checks: active") |
| print("Unicode-aware token checks: active") |
| print("Orphan participial-fragment checks: active") |
| print("Delimiter-balance checks: active") |
| print("Pronoun-case checks: active") |
| print("Internal capitalization checks: active") |
| print("Possessive-determiner integrity checks: active") |
| print("Thereby-result adjunct punctuation checks: active") |
| print("First-stage filtering: active") |
| print("Final/composed filtering: active") |
| print("Rerun-safe Generator.lane wrapping: active") |
| print("Rerun-safe Generator.generate wrapping: active") |
| print("Malformed candidate self-tests: PASSED") |
| print("Valid candidate self-tests: PASSED") |
| print("Duplicate rule registrations: none") |
| print("Registered transformation rules:", len(RULES)) |
| print("Expected rule count:", EXPECTED_5A_RULE_COUNT) |
| print("=" * 78) |
|
|
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
|
|
| import re |
|
|
| EXPECTED_CELL6_RULE_COUNT = 42 |
|
|
|
|
| |
| |
| |
|
|
| _REQUIRED_6 = [ |
| "Analyzer", "RULES", "sent_split", "nlp", |
| ] |
| _missing_6 = [name for name in _REQUIRED_6 if name not in globals()] |
| if _missing_6: |
| raise RuntimeError( |
| "Cell 6 cannot run. Missing: " + ", ".join(_missing_6) |
| + ". Run Cells 1–5A first." |
| ) |
|
|
| if len(RULES) != EXPECTED_CELL6_RULE_COUNT: |
| raise RuntimeError( |
| f"Cell 6 v6.1 expected {EXPECTED_CELL6_RULE_COUNT} Cell 5 v5 rules, " |
| f"but found {len(RULES)}." |
| ) |
|
|
|
|
| |
| |
| |
|
|
| if "_NLL_CELL6_ORIGINAL_SENT_SPLIT" not in globals(): |
| _NLL_CELL6_ORIGINAL_SENT_SPLIT = sent_split |
|
|
| if not hasattr(Analyzer, "_nll_cell6_original_analyze"): |
| Analyzer._nll_cell6_original_analyze = Analyzer.analyze |
|
|
| _BASE_ANALYZE = Analyzer._nll_cell6_original_analyze |
|
|
|
|
| |
| |
| |
|
|
| _CELL6_ABBREVIATIONS = { |
| r"\bet\s+al\.": "et al<NLL_DOT>", |
| r"\be\.g\.": "e<NLL_DOT>g<NLL_DOT>", |
| r"\bi\.e\.": "i<NLL_DOT>e<NLL_DOT>", |
| r"\bFig\.": "Fig<NLL_DOT>", |
| r"\bFigs\.": "Figs<NLL_DOT>", |
| r"\bEq\.": "Eq<NLL_DOT>", |
| r"\bEqs\.": "Eqs<NLL_DOT>", |
| r"\bDr\.": "Dr<NLL_DOT>", |
| r"\bProf\.": "Prof<NLL_DOT>", |
| r"\bMr\.": "Mr<NLL_DOT>", |
| r"\bMs\.": "Ms<NLL_DOT>", |
| r"\bMrs\.": "Mrs<NLL_DOT>", |
| r"\bNo\.": "No<NLL_DOT>", |
| } |
|
|
|
|
| def sent_split(text): |
| protected = str(text or "") |
|
|
| for pattern, replacement in _CELL6_ABBREVIATIONS.items(): |
| protected = re.sub(pattern, replacement, protected, flags=re.I) |
|
|
| parts = _NLL_CELL6_ORIGINAL_SENT_SPLIT(protected) |
|
|
| return [ |
| str(part).replace("<NLL_DOT>", ".").strip() |
| for part in parts |
| if str(part).strip() |
| ] |
|
|
|
|
| |
| |
| |
|
|
| _PRIMARY_FAMILIES_V51 = { |
| "COPULAR_ACTION_FRAME", |
| "NEED_TO_ACTION", |
| "FINITE_PURPOSE_INTENT", |
| "RATHER_THAN_GERUND", |
| "MODAL_COPULAR_ENUMERATION", |
| "USING_METHOD_RELATION", |
| "THROUGH_METHOD_RELATION", |
| "BY_GERUND_METHOD_RELATION", |
| "PURPOSE_INFINITIVE_FRONTING", |
| "NEGATED_PURPOSE_CONTRAST", |
| "EVIDENTIAL_COMPLEMENT", |
| "WHETHER_COMPLEMENT", |
| "GERUND_SUBJECT", |
| "WHEREBY_CLAUSE", |
| "WHILE_PARTICIPIAL", |
| "FRONTED_ADJUNCT", |
| "DISCOURSE_FRONTING", |
| "COORDINATED_PASSIVE_PARTICIPLES", |
| "PASSIVE_USING", |
| "PASSIVE", |
| "REDUCED_RELATIVE_OR_NP_MODIFIER", |
| "PARENTHETICAL_EMPHASIS", |
| "COORDINATED_PREDICATES", |
| "FINITE_COMPLEMENT_LIST", |
| "PROPOSAL_FRAME", |
| "APPOSITIVE_DEFINITION", |
| "PARTICIPIAL_POSTMODIFIER", |
| "FINITE_OBJECT_LIST_FRAME", |
| "CONTINGENCY_RELATION", |
| } |
|
|
| _SECONDARY_FAMILIES_V51 = { |
| "ENUMERATION", |
| "COORDINATED_OBJECTS_OR_ENUMERATION", |
| "LONG_MULTI_CLAUSE", |
| "NOMINALIZATION", |
| "CONTRAST", |
| "CONTENT_COMPLEMENT", |
| "PURPOSE_RESULT_CLAUSE", |
| "PURPOSE_INFINITIVE", |
| "MODAL_PASSIVE", |
| } |
|
|
| _PARTICIPIAL_LEMMAS = {"span", "cover", "include", "contain", "involve"} |
|
|
|
|
| def _v51_dependency_context(text): |
| try: |
| doc = nlp()(text) |
| if not len(doc): |
| return None, False |
| return doc, bool(doc.has_annotation("DEP")) |
| except Exception: |
| return None, False |
|
|
|
|
| def _v51_dependency_participial_postmodifier(doc): |
| """Vocabulary-independent participial modifier detection.""" |
| if doc is None or not doc.has_annotation("DEP"): |
| return False |
| for token in doc: |
| if token.tag_ not in {"VBG", "VBN"}: |
| continue |
| if token.head is token or token.head.pos_ not in {"NOUN", "PROPN", "PRON"}: |
| continue |
| if token.dep_ in {"acl", "relcl", "amod", "advcl", "dep"}: |
| return True |
| return False |
|
|
|
|
| def _v6_dependency_passive(doc): |
| if doc is None or not doc.has_annotation("DEP"): |
| return False |
| for token in doc: |
| if token.pos_ not in {"VERB", "AUX"}: |
| continue |
| deps = {child.dep_ for child in token.children} |
| if {"nsubjpass", "auxpass"} & deps or {"nsubj:pass", "aux:pass"} & deps: |
| return True |
| return False |
|
|
|
|
| def _v6_dependency_explicit_agent_passive(doc): |
| if not _v6_dependency_passive(doc): |
| return False |
| return any( |
| token.dep_ in {"agent", "obl:agent"} |
| or (token.lower_ == "by" and token.dep_ in {"prep", "agent"}) |
| for token in doc |
| ) |
|
|
|
|
| def _v6_dependency_method(doc): |
| if doc is None or not doc.has_annotation("DEP"): |
| return False |
| for token in doc: |
| if token.lower_ in {"using", "through", "via", "with", "by"} and token.dep_ in {"prep", "obl", "advcl"}: |
| return True |
| if token.dep_ == "advcl" and token.tag_ == "VBG": |
| marker = next((c for c in token.children if c.dep_ in {"mark", "prep"}), None) |
| if marker is not None and marker.lower_ in {"by", "using"}: |
| return True |
| return False |
|
|
|
|
| def _v6_dependency_purpose(doc): |
| if doc is None or not doc.has_annotation("DEP"): |
| return False |
| for token in doc: |
| if token.pos_ != "VERB" or token.tag_ != "VB" or token.dep_ not in {"advcl", "xcomp"}: |
| continue |
| if any(c.lower_ == "to" and c.dep_ in {"aux", "mark"} for c in token.children): |
| return True |
| return False |
|
|
|
|
| def _v6_dependency_adverbial_clause(doc): |
| if doc is None or not doc.has_annotation("DEP"): |
| return False |
| return any(token.dep_ == "advcl" for token in doc) |
|
|
|
|
| def _v6_dependency_prepositional_adjunct(doc): |
| if doc is None or not doc.has_annotation("DEP"): |
| return False |
| root = next((t for t in doc if t.dep_ == "ROOT"), None) |
| if root is None: |
| return False |
| return any(c.dep_ in {"prep", "obl"} for c in root.children) |
|
|
|
|
| def _v6_dependency_object_enumeration(doc): |
| if doc is None or not doc.has_annotation("DEP"): |
| return False |
| for token in doc: |
| if token.dep_ not in {"obj", "dobj", "attr", "oprd"}: |
| continue |
| if len([x for x in token.conjuncts if x.dep_ == "conj"]) >= 2: |
| return True |
| return False |
|
|
|
|
| def _v6_dependency_appositive(doc): |
| if doc is None or not doc.has_annotation("DEP"): |
| return False |
| return any(token.dep_ == "appos" and token.head.pos_ in {"NOUN", "PROPN"} for token in doc) |
|
|
|
|
| def _v6_dependency_copular(doc): |
| if doc is None or not doc.has_annotation("DEP"): |
| return False |
| root = next((t for t in doc if t.dep_ == "ROOT"), None) |
| return bool(root is not None and any(c.dep_ == "cop" for c in root.children)) |
|
|
|
|
| def _v6_dependency_reporting_complement(doc): |
| if doc is None or not doc.has_annotation("DEP"): |
| return False |
| root = next((t for t in doc if t.dep_ == "ROOT"), None) |
| return bool(root is not None and any(c.dep_ in {"ccomp", "xcomp"} for c in root.children)) |
|
|
|
|
| def _v51_dependency_coord_predicates(doc): |
| if doc is None or not doc.has_annotation("DEP"): |
| return False |
|
|
| roots = [token for token in doc if token.dep_ == "ROOT"] |
| if len(roots) != 1: |
| return False |
|
|
| root = roots[0] |
| if root.pos_ not in {"VERB", "AUX"}: |
| return False |
|
|
| subjects = [ |
| child for child in root.children |
| if child.dep_ in {"nsubj", "nsubjpass", "csubj"} |
| ] |
| if len(subjects) != 1: |
| return False |
|
|
| conjuncts = [ |
| token for token in root.conjuncts |
| if token.dep_ == "conj" |
| and token.pos_ in {"VERB", "AUX"} |
| and token.i > root.i |
| ] |
| if not conjuncts: |
| return False |
|
|
| return True |
|
|
|
|
| def _v51_dependency_noun_enumeration(doc): |
| if doc is None or not doc.has_annotation("DEP"): |
| return False |
|
|
| return any( |
| token.dep_ == "conj" |
| and token.head.pos_ in {"NOUN", "PROPN", "PRON", "ADJ"} |
| for token in doc |
| ) |
|
|
|
|
| |
| |
| |
|
|
|
|
| def _analyze_v51(self, sentence, profile, user_terms): |
| analysis = _BASE_ANALYZE(self, sentence, profile, user_terms) |
|
|
| text = str(sentence or "").strip() |
| low = text.lower() |
| families = set(getattr(analysis, "families", []) or []) |
| notes = getattr(analysis, "notes", None) |
| if notes is None: |
| notes = [] |
| analysis.notes = notes |
|
|
| comma_count = len(re.findall(r",", text)) |
| semicolon_count = text.count(";") |
| has_coord_word = bool(re.search(r"\b(?:and|or)\b", low)) |
| has_enumeration = comma_count >= 2 and has_coord_word |
|
|
| doc, dependency_available = _v51_dependency_context(text) |
|
|
| tests = { |
| "CONTRAST": bool(re.search( |
| r"\bnot\b.+?\bbut\b|\b(?:but|although|whereas|however)\b", |
| low, |
| re.S, |
| )), |
| "CONTENT_COMPLEMENT": bool(re.search( |
| r"\b(?:evidence|shows?|indicates?|suggests?|demonstrates?)\s+that\b", |
| low, |
| )), |
| "EVIDENTIAL_COMPLEMENT": bool(re.search( |
| r"\b(?:provide|provides|provided|offer|offers|offered)\s+" |
| r"(?:clear\s+|strong\s+|empirical\s+)?evidence\s+that\b", |
| low, |
| )), |
| "WHETHER_COMPLEMENT": bool(re.search( |
| r"\brequire(?:s|d)?\s+(?:an?\s+)?" |
| r"(?:consideration|assessment|evaluation|analysis)\s+of\s+whether\b", |
| low, |
| )), |
| "NEGATED_PURPOSE_CONTRAST": bool(re.search( |
| r"\b(?:purpose|aim|objective|goal)\b.*?\bis\s+not\s+to\b" |
| r".+?\bbut\s+to\b", |
| low, |
| re.S, |
| )), |
| "GERUND_SUBJECT": bool(re.match( |
| r"^(?:Applying|Using|Evaluating|Comparing|Assessing|Examining|" |
| r"Analyzing|Implementing|Testing|Measuring)\b", |
| text, |
| re.I, |
| )), |
| "WHEREBY_CLAUSE": bool(re.search(r"\bwhereby\b", low)), |
| "PURPOSE_RESULT_CLAUSE": bool(re.search(r"\bso\s+that\b", low)), |
| "WHILE_PARTICIPIAL": bool(re.search(r"\bwhile\s+\w+ing\b", low)), |
| "PARENTHETICAL_EMPHASIS": bool(re.search( |
| r",\s*by\s+(?:itself|themselves)\s*,|" |
| r"\b(?:by itself|by themselves|alone)\b", |
| low, |
| )), |
| "FRONTED_ADJUNCT": bool(re.match( |
| r"^(?:for|in|within|under|during|among|across|throughout)\b[^,]{1,160},", |
| low, |
| )), |
| "METHOD_RELATION": bool(re.search( |
| r"\bthrough\s+.+|\bby\s+\w+ing\b|\busing\s+.+", |
| low, |
| )), |
| "PURPOSE_INFINITIVE": bool(re.search( |
| r"\b(?:intended|designed|used|developed|implemented)\s+to\b", |
| low, |
| )), |
| "MODAL_PASSIVE": bool(re.search( |
| r"\b(?:can|could|may|might|must|should|would|will)\s+be\s+" |
| r"(?:\w+ly\s+)?\w+(?:ed|en)\b", |
| low, |
| )), |
| "PASSIVE_USING": bool(re.search( |
| r"\b(?:is|are|was|were|has been|have been|had been)\s+" |
| r"(?:\w+ly\s+)?evaluated\s+using\b", |
| low, |
| )), |
| "PASSIVE": bool(re.search( |
| r"\b(?:is|are|was|were|has|have|had)\s+(?:\w+ly\s+)?(?:been\s+)?" |
| r"(?:studied|examined|investigated|evaluated|reported|obtained|" |
| r"derived|collected|trained|modified|implemented|applied|tested|" |
| r"inspected|approved|reviewed|documented)\b", |
| low, |
| )), |
| "REDUCED_RELATIVE_OR_NP_MODIFIER": bool(re.search( |
| r"\b(?:results?|evidence|data|samples?|features?|models?|constraints?|" |
| r"methods?|files?|representations?)\s+(?:obtained|derived|collected|" |
| r"trained|reported|based|intended|designed|used|modified|implemented)\b", |
| low, |
| )), |
| "DISCOURSE_FRONTING": bool(re.match( |
| r"^(?:however|therefore|consequently|thus|moreover|furthermore|" |
| r"nevertheless|accordingly|additionally),", |
| low, |
| )), |
| "LONG_MULTI_CLAUSE": ( |
| len(text.split()) >= 32 |
| or semicolon_count >= 1 |
| or comma_count >= 4 |
| ), |
| "COPULAR_ACTION_FRAME": bool(re.match( |
| r"^the\s+(?:next|first|second|third|final|primary|main)\s+" |
| r"(?:step|task|action|objective|goal)\s+is\s+to\b", |
| low, |
| )), |
| "NEED_TO_ACTION": bool(re.search(r"\bneed(?:s)?\s+to\b", low)), |
| "MODAL_COPULAR_ENUMERATION": bool(re.search( |
| r"\b(?:may|might|could|can)\s+be\s+in\s+" |
| r".+?,\s*.+?\s*,?\s*(?:or|and)\s+.+", |
| low, |
| re.S, |
| )), |
| "USING_METHOD_RELATION": bool(re.search(r"\busing\s+.+", low)), |
| "THROUGH_METHOD_RELATION": bool(re.search(r"\bthrough\s+.+", low)), |
| "BY_GERUND_METHOD_RELATION": bool(re.search(r"\bby\s+\w+ing\b", low)), |
| "FINITE_PURPOSE_INTENT": bool(re.search( |
| r"\b(?:intend|intends|intended)\s+to\b", |
| low, |
| )), |
| "RATHER_THAN_GERUND": bool(re.search(r"\brather\s+than\s+\w+ing\b", low)), |
| "COORDINATED_PASSIVE_PARTICIPLES": bool(re.search( |
| r"\b(?:(?:have|has|had)\s+been|" |
| r"(?:can|could|may|might|must|should|would|will)\s+be)\s+" |
| r"(?:\w+ly\s+)?\w+(?:ed|en)\s*,\s*" |
| r"\w+(?:ed|en)\s*,\s*and\s*\w+(?:ed|en)(?:\s+by\b)?", |
| low, |
| )), |
| "PURPOSE_INFINITIVE_FRONTING": bool(re.match( |
| r"^to\s+[^,]{2,160},\s+", |
| low, |
| )), |
| "COORDINATED_OBJECTS_OR_ENUMERATION": has_enumeration, |
| "ENUMERATION": has_enumeration, |
|
|
| |
| "FINITE_COMPLEMENT_LIST": bool(re.search( |
| r"\bwithout\s+.+?,\s*.+?(?:,\s*)?(?:and|or)\s+.+", |
| low, |
| re.S, |
| )), |
| "PROPOSAL_FRAME": bool(re.match( |
| r"^(?:we|i|the\s+(?:study|paper|article|report|team|authors?))\s+" |
| r"(?:propose|proposes|introduce|introduces|present|presents)\b", |
| low, |
| )), |
| "APPOSITIVE_DEFINITION": bool(re.search( |
| r"\([^)]{2,120}\)\s*,\s*(?:an?|the)\s+.+", |
| text, |
| re.S, |
| )), |
| "PARTICIPIAL_POSTMODIFIER": ( |
| _v51_dependency_participial_postmodifier(doc) |
| or bool(re.search( |
| r"\b(?:to|for|with|among|across)\s+" |
| r"(?:[A-Za-z][\w-]*\s+){0,5}[A-Za-z][\w-]*\s+" |
| r"(?:spanning|covering|including|containing|involving)\s+" |
| r".+?,\s*.+?(?:,\s*)?(?:and|or)\s+.+", |
| text, |
| re.I | re.S, |
| )) |
| ), |
| "FINITE_OBJECT_LIST_FRAME": bool(re.match( |
| r"^(?:we|i|they|the\s+[A-Za-z][\w-]*(?:\s+[A-Za-z][\w-]*){0,4})\s+" |
| r"(?:pre-register|preregister|document|record|evaluate|assess|review|" |
| r"analyze|compare|test|measure|include|identify|specify|report)\s+" |
| r".+?,\s*.+?(?:,\s*)?(?:and|or)\s+.+", |
| text, |
| re.I | re.S, |
| )), |
| "CONTINGENCY_RELATION": bool(re.search( |
| r"\b(?:is|are|was|were)\s+contingent\s+on\b", |
| low, |
| )), |
| } |
|
|
| families.update(name for name, matched in tests.items() if matched) |
|
|
| |
| try: |
| if dependency_available: |
| if _v51_dependency_coord_predicates(doc): |
| families.add("COORDINATED_PREDICATES") |
| if _v51_dependency_noun_enumeration(doc): |
| families.add("COORDINATED_OBJECTS_OR_ENUMERATION") |
| families.add("ENUMERATION") |
| if _v51_dependency_participial_postmodifier(doc): |
| families.add("PARTICIPIAL_POSTMODIFIER") |
| families.add("REDUCED_RELATIVE_OR_NP_MODIFIER") |
| if _v6_dependency_passive(doc): |
| families.add("PASSIVE") |
| if _v6_dependency_method(doc): |
| families.add("METHOD_RELATION") |
| if _v6_dependency_purpose(doc): |
| families.add("PURPOSE_INFINITIVE") |
| families.add("PURPOSE_INFINITIVE_FRONTING") |
| if _v6_dependency_adverbial_clause(doc): |
| families.add("LONG_MULTI_CLAUSE") |
| if _v6_dependency_prepositional_adjunct(doc): |
| families.add("FRONTED_ADJUNCT") |
| if _v6_dependency_object_enumeration(doc): |
| families.add("FINITE_OBJECT_LIST_FRAME") |
| families.add("COORDINATED_OBJECTS_OR_ENUMERATION") |
| families.add("ENUMERATION") |
| if _v6_dependency_appositive(doc): |
| families.add("APPOSITIVE_DEFINITION") |
| if _v6_dependency_copular(doc): |
| families.add("COPULAR_ACTION_FRAME") |
| if _v6_dependency_reporting_complement(doc): |
| families.add("CONTENT_COMPLEMENT") |
| except Exception as exc: |
| notes.append(f"Dependency-family overlay failed: {exc}") |
|
|
| safe_families = families - {"PARSE_FAILURE"} |
| if "PARSE_FAILURE" in families and safe_families: |
| families.remove("PARSE_FAILURE") |
| notes.append( |
| "Dependency parsing was unavailable or incomplete, but " |
| "surface-safe grammar families were detected." |
| ) |
|
|
| primary = sorted(family for family in families if family in _PRIMARY_FAMILIES_V51) |
| secondary = sorted(family for family in families if family in _SECONDARY_FAMILIES_V51) |
|
|
| analysis.families = sorted(families) |
| try: |
| analysis.metadata = dict(getattr(analysis, "metadata", {}) or {}) |
| analysis.metadata.update({ |
| "dependency_available": dependency_available, |
| "routing_overlay": "NLL_CELL_6_V6_1_CELL5_V5_RERUN_SAFE", |
| "primary_families": primary, |
| "secondary_structures": secondary, |
| "primary_family_count": len(primary), |
| "secondary_structure_count": len(secondary), |
| }) |
| except Exception: |
| pass |
|
|
| return analysis |
|
|
|
|
| Analyzer.analyze = _analyze_v51 |
|
|
|
|
| |
| |
| |
|
|
| |
| |
| |
| |
| _UNIVERSAL_ROUTE_FAMILIES_V61 = { |
| "PASSIVE", |
| "PARTICIPIAL_POSTMODIFIER", |
| "REDUCED_RELATIVE_OR_NP_MODIFIER", |
| "METHOD_RELATION", |
| "PURPOSE_INFINITIVE", |
| "PURPOSE_INFINITIVE_FRONTING", |
| "LONG_MULTI_CLAUSE", |
| "FRONTED_ADJUNCT", |
| "COORDINATED_PREDICATES", |
| "FINITE_OBJECT_LIST_FRAME", |
| "COORDINATED_OBJECTS_OR_ENUMERATION", |
| "ENUMERATION", |
| "APPOSITIVE_DEFINITION", |
| "COPULAR_ACTION_FRAME", |
| "CONTENT_COMPLEMENT", |
| } |
|
|
| CRITICAL_FAMILIES = ( |
| set(_PRIMARY_FAMILIES_V51) |
| | set(_SECONDARY_FAMILIES_V51) |
| | _UNIVERSAL_ROUTE_FAMILIES_V61 |
| ) |
|
|
| ROUTE_COVERAGE = { |
| family: sorted( |
| registered.name |
| for registered in RULES |
| if family in registered.families |
| ) |
| for family in CRITICAL_FAMILIES |
| } |
|
|
| _ROUTE_REQUIRED_FAMILIES_V61 = { |
| family |
| for family, routes in ROUTE_COVERAGE.items() |
| if routes |
| } |
|
|
| |
| |
| UNCOVERED = sorted( |
| family for family, routes in ROUTE_COVERAGE.items() if not routes |
| ) |
|
|
|
|
| |
| |
| |
|
|
| if _BASE_ANALYZE is Analyzer.analyze: |
| raise RuntimeError( |
| "Cell 6 v6.1 rerun-safety failure: Analyzer overlay references itself." |
| ) |
|
|
| if _NLL_CELL6_ORIGINAL_SENT_SPLIT is sent_split: |
| raise RuntimeError( |
| "Cell 6 v6.1 rerun-safety failure: sentence splitter references itself." |
| ) |
|
|
| if len(RULES) != EXPECTED_CELL6_RULE_COUNT: |
| raise RuntimeError( |
| f"Cell 6 v6.1 expected {EXPECTED_CELL6_RULE_COUNT} rules but found {len(RULES)}." |
| ) |
|
|
| _EXPECTED_V6_REQUIRED_ROUTES = { |
| "FINITE_COMPLEMENT_LIST": {"without_complement_list"}, |
| "PROPOSAL_FRAME": {"proposal_appositive_definition"}, |
| "APPOSITIVE_DEFINITION": {"proposal_appositive_definition", "universal_appositive_definition"}, |
| "PARTICIPIAL_POSTMODIFIER": {"participial_postmodifier", "universal_participial_modifier"}, |
| "FINITE_OBJECT_LIST_FRAME": {"finite_object_list_frame", "universal_object_enumeration_frame"}, |
| "CONTINGENCY_RELATION": {"contingency_relation"}, |
| "PASSIVE": {"universal_explicit_agent_passive"}, |
| "METHOD_RELATION": {"universal_method_adjunct_fronting"}, |
| "PURPOSE_INFINITIVE_FRONTING": {"universal_purpose_clause_relocation"}, |
| "COORDINATED_PREDICATES": {"universal_coordinated_predicate_split"}, |
| "REDUCED_RELATIVE_OR_NP_MODIFIER": {"universal_relative_clause_reduction"}, |
| "CONTENT_COMPLEMENT": {"universal_reporting_complement"}, |
| } |
|
|
| _route_errors = {} |
| for family, required in _EXPECTED_V6_REQUIRED_ROUTES.items(): |
| observed = set(ROUTE_COVERAGE.get(family, [])) |
| missing = sorted(required - observed) |
| if missing: |
| _route_errors[family] = { |
| "observed": sorted(observed), |
| "missing_required": missing, |
| } |
| if _route_errors: |
| raise RuntimeError("Cell 6 v6.1 route-alignment failure: " + str(_route_errors)) |
|
|
| if "universal_method_adjunct_fronting" not in ROUTE_COVERAGE.get("METHOD_RELATION", []): |
| raise RuntimeError("Cell 6 v6.1 failed to index universal method routing.") |
| if "universal_reporting_complement" not in ROUTE_COVERAGE.get("CONTENT_COMPLEMENT", []): |
| raise RuntimeError("Cell 6 v6.1 failed to index universal reporting routing.") |
|
|
|
|
| _contracted_uncovered = sorted( |
| family |
| for family in _EXPECTED_V6_REQUIRED_ROUTES |
| if not ROUTE_COVERAGE.get(family) |
| ) |
| if _contracted_uncovered: |
| raise RuntimeError( |
| "Cell 6 v6.1 uncovered contracted families: " |
| + ", ".join(_contracted_uncovered) |
| ) |
|
|
|
|
| |
| |
| |
|
|
| _ROUTING_TESTS = [ |
| ( |
| "FINITE_COMPLEMENT_LIST", |
| "Systems train separate models without explicit rules for assessment, correction, and graduation.", |
| ), |
| ( |
| "PROPOSAL_FRAME", |
| "We propose Framework X (FX), a protocol consisting of contracts, examinations, and promotion.", |
| ), |
| ( |
| "APPOSITIVE_DEFINITION", |
| "We propose Framework X (FX), a protocol consisting of contracts, examinations, and promotion.", |
| ), |
| ( |
| "PARTICIPIAL_POSTMODIFIER", |
| "The testbed applies the protocol to learners spanning classification, drafting, and planning.", |
| ), |
| ( |
| "FINITE_OBJECT_LIST_FRAME", |
| "We pre-register questions, data, baselines, ablations, and testing.", |
| ), |
| ( |
| "CONTINGENCY_RELATION", |
| "Research claims about improvement, cost, and deployment are contingent on execution and analysis.", |
| ), |
| ( |
| "PARTICIPIAL_POSTMODIFIER", |
| "Models predicting regional demand inform the allocation process.", |
| ), |
| ( |
| "PASSIVE", |
| "The unfamiliar alloy was cryogenically stabilized by the laboratory.", |
| ), |
| ( |
| "COORDINATED_PREDICATES", |
| "The controller reconciles telemetry and triggers the ventilation system.", |
| ), |
| ( |
| "CONTENT_COMPLEMENT", |
| "The observatory reports that the instrument tracks transient emissions.", |
| ), |
| ] |
|
|
| |
| |
| _test_profile = next(iter(PROFILES.values())) if "PROFILES" in globals() and PROFILES else None |
| _routing_failures = [] |
|
|
| if _test_profile is not None: |
| _test_analyzer = Analyzer() |
| for expected_family, sentence in _ROUTING_TESTS: |
| try: |
| observed = set( |
| getattr( |
| _test_analyzer.analyze(sentence, _test_profile, set()), |
| "families", |
| [], |
| ) |
| or [] |
| ) |
| if expected_family not in observed: |
| _routing_failures.append({ |
| "family": expected_family, |
| "sentence": sentence, |
| "observed": sorted(observed), |
| }) |
| except Exception as exc: |
| _routing_failures.append({ |
| "family": expected_family, |
| "sentence": sentence, |
| "error": repr(exc), |
| }) |
|
|
| if _routing_failures: |
| raise RuntimeError( |
| "Cell 6 v6.1 routing self-test failure: " + str(_routing_failures) |
| ) |
|
|
|
|
| print("=" * 78) |
| print("CELL 6 UNIVERSAL ROUTING v6.1 CHECK") |
| print("=" * 78) |
| print("Aligned Cell 5 version: v5") |
| print("Aligned Cell 5A version: v4.3") |
| print("Transformation rules:", len(RULES)) |
| print("Expected transformation rules:", EXPECTED_CELL6_RULE_COUNT) |
| print("Uncovered diagnostic-only families:", UNCOVERED) |
| print("Uncovered contracted families:", _contracted_uncovered) |
| print("Analyzer rerun-safety check: PASSED") |
| print("Sentence-splitter rerun-safety check: PASSED") |
| print("Dependency-confirmed participial routing: active") |
| print("Vocabulary-independent passive routing: active") |
| print("Vocabulary-independent method routing: active") |
| print("Vocabulary-independent purpose routing: active") |
| print("Vocabulary-independent object-enumeration routing: active") |
| print("Vocabulary-independent appositive routing: active") |
| print("Vocabulary-independent reporting-complement routing: active") |
| print("Secondary-family route indexing: active") |
| print("Dependency-confirmed coordinated predicates: active") |
| print("Primary/secondary family metadata: active") |
| print("Routing self-tests: PASSED") |
|
|
| for family in [ |
| "FINITE_COMPLEMENT_LIST", |
| "PROPOSAL_FRAME", |
| "APPOSITIVE_DEFINITION", |
| "PARTICIPIAL_POSTMODIFIER", |
| "FINITE_OBJECT_LIST_FRAME", |
| "CONTINGENCY_RELATION", |
| "COPULAR_ACTION_FRAME", |
| "NEED_TO_ACTION", |
| "MODAL_COPULAR_ENUMERATION", |
| "USING_METHOD_RELATION", |
| "THROUGH_METHOD_RELATION", |
| "BY_GERUND_METHOD_RELATION", |
| "FINITE_PURPOSE_INTENT", |
| "RATHER_THAN_GERUND", |
| "COORDINATED_PASSIVE_PARTICIPLES", |
| "PURPOSE_INFINITIVE_FRONTING", |
| "COORDINATED_OBJECTS_OR_ENUMERATION", |
| "ENUMERATION", |
| ]: |
| print(f"{family}: {ROUTE_COVERAGE.get(family, [])}") |
|
|
| print("=" * 78) |
|
|
|
|
| |
| |
| |
| |
| |
| |
| |
|
|
| import json |
| import re |
| import time |
| import unicodedata |
| from collections import Counter |
| from dataclasses import asdict |
| from difflib import SequenceMatcher |
| from pathlib import Path |
|
|
| EXPECTED_CELL7_RULE_COUNT = 42 |
|
|
| _REQUIRED_CELL7 = [ |
| "STATE", "CFG", "RULES", "Analyzer", "Generator", "Scanner", |
| "Candidate", "SentenceResult", "DocumentResult", "PROFILES", |
| "protect", "optional", "nlp", "sub", "sent_split", "surface_key", |
| "realization_issues", |
| ] |
| _missing_cell7 = [name for name in _REQUIRED_CELL7 if name not in globals()] |
| if _missing_cell7: |
| raise RuntimeError( |
| "Cell 7 cannot run. Missing: " + ", ".join(_missing_cell7) |
| + ". Run Cells 1–6 v6.1 in order first." |
| ) |
| if len(RULES) != EXPECTED_CELL7_RULE_COUNT: |
| raise RuntimeError( |
| f"Cell 7 expected {EXPECTED_CELL7_RULE_COUNT} rules but found {len(RULES)}." |
| ) |
|
|
| |
| _surface_key = surface_key |
|
|
| NEG = {"not", "no", "never", "without", "neither", "nor"} |
| HEDGE = { |
| "may", "might", "could", "can", "possibly", "potentially", |
| "suggest", "suggests", "indicate", "indicates", "likely", |
| "approximately", "about", "must", "should", "would", "will", |
| } |
|
|
|
|
| def _ensure_state(): |
| if not isinstance(STATE, dict): |
| raise RuntimeError("STATE must be a dictionary.") |
| STATE.setdefault("warnings", []) |
| STATE.setdefault("start", None) |
| return STATE |
|
|
|
|
| def sr(a, b): |
| return SequenceMatcher( |
| None, |
| re.sub(r"\s+", " ", str(a).lower()).strip(), |
| re.sub(r"\s+", " ", str(b).lower()).strip(), |
| ).ratio() |
|
|
|
|
| def token_change(a, b): |
| |
| |
| A = _unicode_tokens(a) if "_unicode_tokens" in globals() else re.findall(r"\w+", str(a).casefold(), flags=re.UNICODE) |
| B = _unicode_tokens(b) if "_unicode_tokens" in globals() else re.findall(r"\w+", str(b).casefold(), flags=re.UNICODE) |
| return 1 - SequenceMatcher(None, A, B).ratio() |
|
|
|
|
| def nli_pair(a, b): |
| _ensure_state() |
| bundle = optional("nli") |
| if not bundle: |
| return None |
| try: |
| import torch |
| tok, model = bundle |
| x = tok(a, b, return_tensors="pt", truncation=True, max_length=512) |
| dev = next(model.parameters()).device |
| x = {k: v.to(dev) for k, v in x.items()} |
| with torch.no_grad(): |
| prob = torch.softmax(model(**x).logits[0], -1).cpu().numpy() |
| labels = { |
| str(model.config.id2label[i]).lower(): float(prob[i]) |
| for i in range(len(prob)) |
| } |
| return { |
| "entailment": max([v for k, v in labels.items() if "entail" in k] or [0]), |
| "contradiction": max([v for k, v in labels.items() if "contrad" in k] or [0]), |
| } |
| except Exception as exc: |
| STATE["warnings"].append(f"NLI scoring failed: {exc}") |
| return None |
|
|
|
|
| def stsim(a, b): |
| _ensure_state() |
| model = optional("sts") |
| if not model: |
| return None |
| try: |
| from sentence_transformers.util import cos_sim |
| emb = model.encode([a, b], convert_to_tensor=True, normalize_embeddings=True) |
| return float(cos_sim(emb[0], emb[1]).item()) |
| except Exception as exc: |
| STATE["warnings"].append(f"STS scoring failed: {exc}") |
| return None |
|
|
|
|
| |
| |
| |
|
|
| def _safe_doc(text): |
| try: |
| return nlp()(str(text or "")) |
| except Exception: |
| return None |
|
|
|
|
| def _neg_inventory(text, doc=None): |
| """Compare polarity markers by lexical class, not dependency owner.""" |
| lo = str(text or "").lower() |
| return Counter( |
| token for token in ["not", "no", "never", "without", "neither", "nor"] |
| if re.search(r"(?<!\w)" + re.escape(token) + r"(?!\w)", lo) |
| ) |
|
|
|
|
| def _neg_scope_debug(doc): |
| if doc is None or not getattr(doc, "has_annotation", lambda *_: False)("DEP"): |
| return [] |
| return sorted( |
| (x.lower_, x.head.lemma_.lower()) |
| for x in doc |
| if x.dep_ == "neg" or x.lower_ in {"never", "without", "neither", "nor"} |
| ) |
|
|
|
|
| def _modals(doc, text=""): |
| if doc is not None: |
| try: |
| return Counter( |
| x.lemma_.lower() |
| for x in doc |
| if x.lemma_.lower() in HEDGE or x.tag_ == "MD" |
| ) |
| except Exception: |
| pass |
| return Counter(re.findall( |
| r"\b(?:may|might|can|could|must|should|would|will|possibly|potentially|likely)\b", |
| str(text or "").lower(), |
| )) |
|
|
|
|
| def _purpose_contrast_preserved(original, candidate): |
| match = re.search( |
| r"(?is)\bnot\s+to\s+(?P<x>.+?)\s+but\s+to\s+(?P<y>.+?)[.]?$", |
| original, |
| ) |
| if not match: |
| return False |
| x = surface_key(match["x"]) |
| y = surface_key(match["y"]) |
| z = surface_key(candidate) |
| return ( |
| x in z and y in z |
| and ( |
| "rather than" in candidate.lower() |
| or ("does not" in candidate.lower() and "instead" in candidate.lower()) |
| ) |
| ) |
|
|
|
|
| def _unicode_normalize(text): |
| """Canonical Unicode normalization for comparison, never for output.""" |
| value = unicodedata.normalize("NFKC", str(text or "")) |
| translations = str.maketrans({ |
| "–": "-", "—": "-", "−": "-", "‑": "-", |
| "’": "'", "‘": "'", "“": '"', "”": '"', |
| "\u00a0": " ", |
| }) |
| return re.sub(r"\s+", " ", value.translate(translations)).strip().casefold() |
|
|
|
|
| def _unicode_tokens(text): |
| """ |
| Tokenize letters/numbers across Unicode scripts without ASCII loss. |
| |
| Punctuation, including every dash/apostrophe form, is treated as a |
| comparison boundary. This makes these equivalent for inventory checks: |
| re-examination == re examination == re–examination |
| matched-data == matched data |
| The original candidate text is never modified. |
| """ |
| value = _unicode_normalize(text) |
| output = [] |
| current = [] |
|
|
| for char in value: |
| category = unicodedata.category(char) |
| if category[0] in {"L", "N", "M"}: |
| current.append(char) |
| elif current: |
| token = "".join(current) |
| if token: |
| output.append(token) |
| current = [] |
|
|
| if current: |
| token = "".join(current) |
| if token: |
| output.append(token) |
|
|
| return output |
|
|
|
|
| def _noun_chunk_for_token(doc, token): |
| try: |
| chunks = [chunk for chunk in doc.noun_chunks if chunk.start <= token.i < chunk.end] |
| except Exception: |
| chunks = [] |
| return min(chunks, key=lambda chunk: chunk.end - chunk.start) if chunks else None |
|
|
|
|
| def _clean_member_text(text): |
| value = re.sub(r"\s+", " ", str(text or "")).strip(" \t\r\n,;:.()[]{}") |
| value = re.sub(r"^(?:and|or|nor)\s+", "", value, flags=re.I) |
| return value.strip() |
|
|
|
|
| def _dependency_enumeration_groups(doc): |
| """Recover coordinated members from dependency heads/noun chunks.""" |
| if doc is None or not doc.has_annotation("DEP"): |
| return [] |
|
|
| groups = [] |
| seen = set() |
| for token in doc: |
| conjuncts = [child for child in token.children if child.dep_ == "conj"] |
| if not conjuncts: |
| continue |
|
|
| members = [token] + sorted(conjuncts, key=lambda item: item.i) |
| |
| cursor = 0 |
| while cursor < len(members): |
| for child in members[cursor].children: |
| if child.dep_ == "conj" and child not in members: |
| members.append(child) |
| cursor += 1 |
| members = sorted(set(members), key=lambda item: item.i) |
|
|
| texts = [] |
| for member in members: |
| chunk = _noun_chunk_for_token(doc, member) |
| if chunk is not None: |
| text = chunk.text |
| else: |
| |
| blocked = {item.i for other in members if other is not member for item in other.subtree} |
| indices = [ |
| item.i for item in member.subtree |
| if item.i not in blocked and item.dep_ not in {"cc", "punct"} |
| ] |
| if indices: |
| text = doc[min(indices):max(indices) + 1].text |
| else: |
| text = member.text |
| text = _clean_member_text(text) |
| if text: |
| texts.append(text) |
|
|
| texts = list(dict.fromkeys(texts)) |
| key = tuple(_unicode_normalize(item) for item in texts) |
| if len(texts) >= 3 and key not in seen: |
| seen.add(key) |
| groups.append(texts) |
|
|
| return groups |
|
|
|
|
| def _analysis_enumeration_groups(analysis): |
| """Accept analyzer groups only when each item is a plausible member.""" |
| groups = [] |
| for group in list(getattr(analysis, "enumerations", []) or []): |
| cleaned = [_clean_member_text(item) for item in group] |
| cleaned = [item for item in cleaned if item] |
| |
| if len(cleaned) >= 3 and all(item.count(",") <= 1 for item in cleaned): |
| groups.append(cleaned) |
| return groups |
|
|
|
|
| def _top_level_segments(text): |
| """Split on top-level commas while respecting brackets and Unicode text.""" |
| segments = [] |
| current = [] |
| stack = [] |
| pairs = {"(": ")", "[": "]", "{": "}"} |
| closers = set(pairs.values()) |
| for char in str(text or ""): |
| if char in pairs: |
| stack.append(pairs[char]) |
| elif char in closers and stack and stack[-1] == char: |
| stack.pop() |
| if char == "," and not stack: |
| segments.append("".join(current).strip()) |
| current = [] |
| else: |
| current.append(char) |
| if current: |
| segments.append("".join(current).strip()) |
| return segments |
|
|
|
|
| def _surface_list_groups(text): |
| """Conservative fallback; never cuts through hyphenated compounds.""" |
| value = str(text or "").strip().rstrip(".?!") |
| segments = _top_level_segments(value) |
| if len(segments) < 3: |
| return [] |
|
|
| |
| final = segments[-1] |
| match = re.search(r"\s+(?:and|or|nor)\s+(.+)$", final, re.I | re.S) |
| if not match: |
| return [] |
| last_member = _clean_member_text(match.group(1)) |
| pre_final = _clean_member_text(final[:match.start()]) |
|
|
| |
| introducer = re.compile( |
| r"\b(?:using|including|involving|such as|comprising|consisting of|covers?|" |
| r"evaluated|assessed|measured|reviewed|identified|pre-register(?:ed)?|preregister(?:ed)?)\s+", |
| re.I, |
| ) |
| first_index = 0 |
| first_text = segments[0] |
| matches = list(introducer.finditer(first_text)) |
| if matches: |
| first_member = _clean_member_text(first_text[matches[-1].end():]) |
| else: |
| |
| return [] |
|
|
| middle = [_clean_member_text(item) for item in segments[1:-1]] |
| members = [first_member] + middle |
| if pre_final: |
| members.append(pre_final) |
| if last_member: |
| members.append(last_member) |
| members = [item for item in members if item] |
| return [members] if len(members) >= 3 else [] |
|
|
|
|
| def _plausible_enumeration_member(text): |
| """ |
| Reject boundary-corrupted pseudo-members without weakening true omission checks. |
| |
| A valid member may contain Unicode letters, numbers, spaces, apostrophes, |
| and dash variants, but should not contain clause punctuation, a full |
| sentence lead-in, or an excessive token span. |
| """ |
| value = _clean_member_text(text) |
| tokens = _unicode_tokens(value) |
|
|
| if not tokens or len(tokens) > 12: |
| return False |
| if any(mark in value for mark in [",", ";", ":", "."]): |
| return False |
| if re.match( |
| r"(?is)^(?:we|i|they|he|she|it|the\s+(?:study|paper|article|report|" |
| r"system|framework|team|model))\s+" |
| r"(?:propose|proposes|introduce|introduces|present|presents|" |
| r"evaluate|evaluates|assess|assesses|pre-register|preregister)\b", |
| value, |
| ): |
| return False |
|
|
| return True |
|
|
|
|
| def _enumeration_groups(analysis, source_text, source_doc): |
| """ |
| Build only high-confidence list groups. |
| |
| Sources are tried independently instead of accepting the first nonempty |
| extractor. Candidate groups are filtered for plausible member boundaries, |
| verified against the source, de-duplicated, and ranked by compactness. |
| """ |
| candidate_groups = [] |
| candidate_groups.extend(_dependency_enumeration_groups(source_doc)) |
| candidate_groups.extend(_analysis_enumeration_groups(analysis)) |
| candidate_groups.extend(_surface_list_groups(source_text)) |
|
|
| source_tokens = _unicode_tokens(source_text) |
| accepted = [] |
| seen = set() |
|
|
| for group in candidate_groups: |
| stable_group = [] |
| for raw_item in group: |
| item = _clean_member_text(raw_item) |
| if not _plausible_enumeration_member(item): |
| continue |
|
|
| item_tokens = _unicode_tokens(item) |
| width = len(item_tokens) |
| appears_in_source = any( |
| source_tokens[index:index + width] == item_tokens |
| for index in range(len(source_tokens) - width + 1) |
| ) |
| if appears_in_source: |
| stable_group.append(item) |
|
|
| |
| unique_group = [] |
| member_seen = set() |
| for item in stable_group: |
| key = tuple(_unicode_tokens(item)) |
| if key and key not in member_seen: |
| member_seen.add(key) |
| unique_group.append(item) |
|
|
| normalized = tuple(tuple(_unicode_tokens(item)) for item in unique_group) |
| if len(normalized) >= 3 and normalized not in seen: |
| seen.add(normalized) |
| accepted.append(unique_group) |
|
|
| |
| compact = [] |
| token_sets = [ |
| {tuple(_unicode_tokens(item)) for item in group} |
| for group in accepted |
| ] |
| for index, group in enumerate(accepted): |
| current = token_sets[index] |
| noisier = any( |
| other < current |
| for other_index, other in enumerate(token_sets) |
| if other_index != index and len(other) >= 3 |
| ) |
| if not noisier: |
| compact.append(group) |
|
|
| return compact |
|
|
|
|
| def _member_present(member, candidate): |
| """Require normalized member tokens in order, independent of punctuation.""" |
| member_tokens = _unicode_tokens(member) |
| candidate_tokens = _unicode_tokens(candidate) |
| if not member_tokens: |
| return True |
| width = len(member_tokens) |
| return any( |
| candidate_tokens[index:index + width] == member_tokens |
| for index in range(len(candidate_tokens) - width + 1) |
| ) |
|
|
|
|
| def _relation_signature(text): |
| lo = str(text or "").lower() |
| classes = { |
| "cause": {"because", "therefore", "thus", "consequently", "hence", "as a result", "for this reason", "accordingly"}, |
| "concession": {"although", "though", "despite", "nevertheless", "however", "even so"}, |
| "condition": {"if", "unless", "whether"}, |
| "simultaneous": {"while", "at the same time"}, |
| "purpose": {"so that", "intended to", "designed to"}, |
| "absence": {"without"}, |
| "dependency": {"contingent on", "depend on", "depends on", "depended on"}, |
| } |
| return { |
| key |
| for key, words in classes.items() |
| if any(re.search(r"(?<!\w)" + re.escape(word) + r"(?!\w)", lo) for word in words) |
| } |
|
|
|
|
| def _triples(doc): |
| triples = [] |
| if doc is None or not doc.has_annotation("DEP"): |
| return triples |
| for verb in doc: |
| subjects = [x for x in verb.children if x.dep_ in {"nsubj", "nsubjpass", "nsubj:pass", "csubj"}] |
| objects = [x for x in verb.children if x.dep_ in {"obj", "dobj", "attr", "oprd"}] |
| triples += [ |
| (sub(subject).lower(), verb.lemma_.lower(), sub(obj).lower()) |
| for subject in subjects |
| for obj in objects |
| ] |
| return triples |
|
|
|
|
| def _acronym_pairs(text): |
| """Discover exact expansion/acronym pairs from arbitrary input. |
| |
| No acronym or expansion is hard-coded. Each parenthetical uppercase |
| acronym is paired with the nearest preceding noun phrase. Extraction uses |
| punctuation/coordinator boundaries first and spaCy syntax only to remove a |
| clause lead-in. The returned pair is an atomic discipline-specific term. |
| """ |
| value = str(text or "") |
| pairs = [] |
| acronym_pattern = re.compile( |
| r"\(\s*(?P<acronym>(?:[A-Z]{2,}|[A-Z][A-Z0-9]*\d[A-Z0-9]*)(?:-[A-Z0-9]+)*)\s*\)", |
| flags=re.UNICODE, |
| ) |
|
|
| doc = _safe_doc(value) |
|
|
| for match in acronym_pattern.finditer(value): |
| acronym = match.group("acronym") |
| prefix = value[:match.start()] |
|
|
| |
| boundary = max(prefix.rfind(","), prefix.rfind(";"), prefix.rfind(":")) |
| fragment = prefix[boundary + 1:].strip() |
| coordinator = list(re.finditer(r"(?iu)\b(?:and|or|nor)\s+", fragment)) |
| if coordinator: |
| fragment = fragment[coordinator[-1].end():].strip() |
|
|
| |
| |
| expansion = "" |
| if doc is not None: |
| left_tokens = [token for token in doc if token.idx + len(token.text) <= match.start()] |
| if left_tokens: |
| tail = left_tokens[-1] |
| try: |
| chunks = [ |
| chunk for chunk in doc.noun_chunks |
| if chunk.start <= tail.i < chunk.end |
| and not value[chunk.end_char:match.start()].strip() |
| ] |
| except Exception: |
| chunks = [] |
| if chunks: |
| expansion = min(chunks, key=lambda chunk: chunk.end - chunk.start).text.strip() |
|
|
| if not expansion: |
| expansion = fragment |
|
|
| |
| |
| if doc is not None and expansion == fragment: |
| frag_start = match.start() - len(fragment) |
| frag_tokens = [ |
| token for token in doc |
| if token.idx >= frag_start and token.idx + len(token.text) <= match.start() |
| ] |
| finite_roots = [ |
| token for token in frag_tokens |
| if token.pos_ in {"VERB", "AUX"} |
| and token.dep_ in {"ROOT", "ccomp", "xcomp", "conj"} |
| ] |
| if finite_roots: |
| last_verb = finite_roots[-1] |
| after = value[last_verb.idx + len(last_verb.text):match.start()].strip() |
| if after: |
| expansion = after |
|
|
| expansion = re.sub(r"(?iu)^(?:and|or|nor)\s+", "", expansion) |
| expansion = re.sub(r"\s+", " ", expansion).strip(" ,;:") |
| expansion_tokens = tuple(_unicode_tokens(expansion)) |
|
|
| |
| if not (1 <= len(expansion_tokens) <= 10): |
| continue |
| if doc is not None: |
| exp_doc = _safe_doc(expansion) |
| if exp_doc is not None and any( |
| token.dep_ == "ROOT" and token.pos_ in {"VERB", "AUX"} |
| and any(child.dep_ in {"nsubj", "csubj"} for child in token.children) |
| for token in exp_doc |
| ): |
| continue |
|
|
| pairs.append({ |
| "expansion": expansion, |
| "expansion_tokens": expansion_tokens, |
| "acronym": acronym, |
| "surface": value[max(0, match.start() - len(expansion)):match.end()].strip(), |
| }) |
|
|
| return pairs |
|
|
|
|
| def _acronym_inventory(text): |
| """Count uppercase acronyms only; ordinary title-case words are excluded.""" |
| return Counter( |
| token |
| for token in re.findall( |
| r"(?<![\w-])(?:[A-Z]{2,}|[A-Z][A-Z0-9]*\d[A-Z0-9]*)(?:-[A-Z0-9]+)*(?![\w-])", |
| str(text or ""), |
| flags=re.UNICODE, |
| ) |
| ) |
|
|
|
|
| def _discipline_term_inventory(text): |
| """Return atomic acronym pairs and standalone acronyms without inference.""" |
| pairs = _acronym_pairs(text) |
| pair_keys = Counter( |
| (pair["expansion_tokens"], pair["acronym"]) |
| for pair in pairs |
| ) |
| acronyms = _acronym_inventory(text) |
| return { |
| "pairs": pairs, |
| "pair_keys": pair_keys, |
| "acronyms": acronyms, |
| } |
|
|
|
|
| def _acronym_mapping_issues(original, revised): |
| """Preserve discipline-specific acronym pairs as indivisible units. |
| |
| Pair order may change. The expansion and acronym may not be separated, |
| swapped, renamed, removed, or duplicated. Standalone source acronyms must |
| retain the same counted inventory. |
| """ |
| source = _discipline_term_inventory(original) |
| candidate = _discipline_term_inventory(revised) |
| issues = [] |
|
|
| if source["pair_keys"] != candidate["pair_keys"]: |
| missing_pairs = source["pair_keys"] - candidate["pair_keys"] |
| added_pairs = candidate["pair_keys"] - source["pair_keys"] |
| if missing_pairs: |
| issues.append({ |
| "type": "DISCIPLINE_ACRONYM_PAIR_REMOVED_OR_SEPARATED", |
| "pairs": [ |
| {"expansion_tokens": list(key[0]), "acronym": key[1], "count": count} |
| for key, count in missing_pairs.items() |
| ], |
| }) |
| if added_pairs: |
| issues.append({ |
| "type": "DISCIPLINE_ACRONYM_PAIR_CHANGED_OR_INTRODUCED", |
| "pairs": [ |
| {"expansion_tokens": list(key[0]), "acronym": key[1], "count": count} |
| for key, count in added_pairs.items() |
| ], |
| }) |
|
|
| if source["acronyms"] != candidate["acronyms"]: |
| issues.append({ |
| "type": "ACRONYM_INVENTORY_CHANGED", |
| "source": dict(source["acronyms"]), |
| "candidate": dict(candidate["acronyms"]), |
| }) |
|
|
| return issues |
|
|
|
|
| def _semantic_tautology_issues(text): |
| """Catch obvious acronym/term tautologies such as 'DAE is DAE'.""" |
| issues = [] |
| for match in re.finditer( |
| r"(?u)(?<![\w-])(?P<x>[A-Z][A-Z0-9-]{1,15})\s+" |
| r"(?:is|are|was|were|equals?|means?)\s+(?P<y>[A-Z][A-Z0-9-]{1,15})(?![\w-])", |
| str(text or ""), |
| ): |
| if match.group("x").casefold() == match.group("y").casefold(): |
| issues.append("REDUNDANT_ACRONYM_COPULAR_TAUTOLOGY") |
| return issues |
|
|
|
|
| _SOURCE_BOUND_FUNCTION_WORDS = { |
| "a", "an", "the", "and", "or", "nor", "but", "if", "then", "than", |
| "as", "at", "by", "for", "from", "in", "into", "of", "on", "to", |
| "under", "with", "without", "that", "which", "who", "whom", "whose", |
| "this", "these", "those", "it", "its", "is", "are", "was", "were", |
| "be", "been", "being", "do", "does", "did", "have", "has", "had", |
| "not", "no", "so", "thereby", "according", "through", "within", |
| } |
|
|
|
|
| def _sentence_units(text): |
| """Return normalized sentence strings without changing the candidate.""" |
| value = str(text or "").strip() |
| doc = _safe_doc(value) |
| if doc is not None: |
| try: |
| units = [sent.text.strip() for sent in doc.sents if sent.text.strip()] |
| if units: |
| return units |
| except Exception: |
| pass |
| return [ |
| part.strip() |
| for part in re.split(r"(?<=[.!?])\s+", value) |
| if part.strip() |
| ] |
|
|
|
|
| def _content_lemmas(text): |
| """Extract content lemmas for comparison; never alter visible output.""" |
| doc = _safe_doc(text) |
| output = [] |
| if doc is not None: |
| for token in doc: |
| if token.is_space or token.is_punct: |
| continue |
| lemma = (token.lemma_ or token.text).casefold().strip() |
| if not lemma or lemma in _SOURCE_BOUND_FUNCTION_WORDS: |
| continue |
| if token.pos_ in {"NOUN", "PROPN", "VERB", "ADJ", "ADV", "NUM"}: |
| output.append(lemma) |
| return output |
| return [ |
| token for token in _unicode_tokens(text) |
| if token not in _SOURCE_BOUND_FUNCTION_WORDS |
| ] |
|
|
|
|
| def _named_entity_inventory(text): |
| """Count explicit named entities. Empty when a trained NER is unavailable.""" |
| doc = _safe_doc(text) |
| if doc is None: |
| return Counter() |
| try: |
| return Counter( |
| _unicode_normalize(entity.text) |
| for entity in doc.ents |
| if entity.text.strip() |
| ) |
| except Exception: |
| return Counter() |
|
|
|
|
| def _sentence_start_signatures(text, minimum_width=3, maximum_width=6): |
| """Return bounded lexical prefixes for every sentence. |
| |
| Three content lemmas are sufficient to identify a repeated clause opening |
| such as ``the literature of history encompasses`` even when the next |
| content word differs. Longer prefixes are also retained for diagnostics. |
| """ |
| signatures = [] |
| for sentence in _sentence_units(text): |
| lemmas = _content_lemmas(sentence) |
| if len(lemmas) < minimum_width: |
| continue |
| upper = min(maximum_width, len(lemmas)) |
| signatures.append({ |
| "minimum": tuple(lemmas[:minimum_width]), |
| "prefixes": [tuple(lemmas[:width]) for width in range(minimum_width, upper + 1)], |
| "sentence": sentence, |
| }) |
| return signatures |
|
|
|
|
| def _repeated_sentence_openings(text, minimum_width=3): |
| """Count repeated sentence openings using a stable minimum prefix.""" |
| records = _sentence_start_signatures(text, minimum_width=minimum_width) |
| return Counter(record["minimum"] for record in records) |
|
|
|
|
| def _repeated_ngram_inventory(text, minimum=3, maximum=8): |
| """Return repeated lexical n-grams and their counts.""" |
| tokens = _unicode_tokens(text) |
| repeated = Counter() |
| for width in range(minimum, min(maximum, len(tokens)) + 1): |
| counts = Counter(tuple(tokens[i:i + width]) for i in range(len(tokens) - width + 1)) |
| for ngram, count in counts.items(): |
| if count > 1: |
| repeated[ngram] = count |
| return repeated |
|
|
|
|
| def _source_bounded_integrity_issues(original, revised): |
| """Reject newly added or repeated content in a non-generative rewrite. |
| |
| The test permits grammatical scaffolding and derivational restructuring, |
| but rejects new named entities, duplicated sentences/openings, newly |
| repeated lexical spans, unexplained content-word inflation, and excessive |
| candidate expansion. |
| """ |
| issues = [] |
| source_text = str(original or "") |
| candidate_text = str(revised or "") |
|
|
| source_tokens = _unicode_tokens(source_text) |
| candidate_tokens = _unicode_tokens(candidate_text) |
| source_sentences = _sentence_units(source_text) |
| candidate_sentences = _sentence_units(candidate_text) |
|
|
| |
| source_sentence_counts = Counter(_unicode_normalize(x) for x in source_sentences) |
| candidate_sentence_counts = Counter(_unicode_normalize(x) for x in candidate_sentences) |
| duplicated_sentences = { |
| sentence: count |
| for sentence, count in candidate_sentence_counts.items() |
| if count > source_sentence_counts.get(sentence, 0) and count > 1 |
| } |
| if duplicated_sentences: |
| issues.append({ |
| "type": "REPEATED_SENTENCE_INTRODUCED", |
| "items": duplicated_sentences, |
| }) |
|
|
| |
| source_starts = _repeated_sentence_openings(source_text, minimum_width=3) |
| candidate_starts = _repeated_sentence_openings(candidate_text, minimum_width=3) |
| repeated_starts = { |
| " ".join(signature): count |
| for signature, count in candidate_starts.items() |
| if count > 1 and count > source_starts.get(signature, 0) |
| } |
| if repeated_starts: |
| issues.append({ |
| "type": "REPEATED_SENTENCE_OPENING_INTRODUCED", |
| "items": repeated_starts, |
| }) |
|
|
| |
| source_repeats = _repeated_ngram_inventory(source_text) |
| candidate_repeats = _repeated_ngram_inventory(candidate_text) |
| new_repeats = { |
| " ".join(ngram): count |
| for ngram, count in candidate_repeats.items() |
| if count > source_repeats.get(ngram, 0) |
| } |
| if new_repeats: |
| |
| shortest = sorted(new_repeats.items(), key=lambda item: (len(item[0].split()), item[0]))[:8] |
| issues.append({ |
| "type": "REPEATED_LEXICAL_SPAN_INTRODUCED", |
| "items": dict(shortest), |
| }) |
|
|
| |
| source_entities = _named_entity_inventory(source_text) |
| candidate_entities = _named_entity_inventory(candidate_text) |
| added_entities = candidate_entities - source_entities |
| if added_entities: |
| issues.append({ |
| "type": "NAMED_ENTITY_INTRODUCED", |
| "items": dict(added_entities), |
| }) |
|
|
| |
| |
| if source_tokens: |
| ratio = len(candidate_tokens) / max(len(source_tokens), 1) |
| allowance = max(5, int(round(len(source_tokens) * 0.18))) |
| if len(candidate_tokens) > len(source_tokens) + allowance and ratio > 1.24: |
| issues.append({ |
| "type": "EXCESSIVE_CONTENT_EXPANSION", |
| "source_tokens": len(source_tokens), |
| "candidate_tokens": len(candidate_tokens), |
| "ratio": round(ratio, 3), |
| }) |
|
|
| |
| |
| source_content = Counter(_content_lemmas(source_text)) |
| candidate_content = Counter(_content_lemmas(candidate_text)) |
| inflated = { |
| lemma: {"source": source_content.get(lemma, 0), "candidate": count} |
| for lemma, count in candidate_content.items() |
| if count >= 2 |
| and count > source_content.get(lemma, 0) + 1 |
| and lemma not in _SOURCE_BOUND_FUNCTION_WORDS |
| } |
| if inflated: |
| issues.append({ |
| "type": "CONTENT_LEMMA_REPETITION_INTRODUCED", |
| "items": inflated, |
| }) |
|
|
| return issues |
|
|
|
|
| def _human_style_penalties(original, candidate_text): |
| """Soft human-editor preferences; never override hard safety gates.""" |
| penalties = [] |
| revised = str(candidate_text or "") |
|
|
| |
| if re.search(r"(?iu)\baccording\s+to\s+H\d+[A-Za-z]?\b", revised): |
| penalties.append({"code": "AWKWARD_HYPOTHESIS_ATTRIBUTION", "weight": 0.08}) |
|
|
| |
| |
| source_demo = len(re.findall(r"(?imu)(?:^|[.!?]\s+)\s*(?:this|that|these|those)\b", str(original or ""))) |
| candidate_demo = len(re.findall(r"(?imu)(?:^|[.!?]\s+)\s*(?:this|that|these|those)\b", revised)) |
| if candidate_demo > source_demo: |
| penalties.append({"code": "NEW_DEMONSTRATIVE_ANTECEDENT", "weight": 0.05}) |
|
|
| return penalties |
|
|
|
|
| def _v5_structural_signature(doc): |
| """Extract vocabulary-independent semantic structure for diagnostics.""" |
| signature = { |
| "roots": [], |
| "subjects": [], |
| "objects": [], |
| "passive": False, |
| "agents": [], |
| "complements": [], |
| } |
| if doc is None or not getattr(doc, "has_annotation", lambda *_: False)("DEP"): |
| return signature |
|
|
| for token in doc: |
| if token.dep_ == "ROOT": |
| signature["roots"].append(token.lemma_.casefold()) |
| if token.dep_ in {"nsubj", "nsubjpass", "nsubj:pass", "csubj"}: |
| signature["subjects"].append(_unicode_normalize(sub(token))) |
| if token.dep_ in {"obj", "dobj", "attr", "oprd"}: |
| signature["objects"].append(_unicode_normalize(sub(token))) |
| if token.dep_ in {"nsubjpass", "nsubj:pass", "auxpass", "aux:pass"}: |
| signature["passive"] = True |
| if token.dep_ in {"agent", "obl:agent"}: |
| signature["agents"].append(_unicode_normalize(sub(token))) |
| if token.dep_ in {"ccomp", "xcomp", "advcl", "relcl", "acl"}: |
| signature["complements"].append((token.dep_, token.lemma_.casefold())) |
| return signature |
|
|
|
|
| def _v5_family_kind(candidate): |
| family = str(getattr(candidate, "family", "") or "") |
| return "universal" if "universal_" in family.casefold() else "specialized" |
|
|
|
|
| |
| |
| |
|
|
| class Validator: |
| def validate(self, original, candidate, analysis, profile, user_terms, mode): |
| _ensure_state() |
| revised = candidate.text |
| failures = [] |
| reasons = [] |
| checks = {} |
|
|
| def reject(code, why): |
| if code not in failures: |
| failures.append(code) |
| reasons.append({"code": code, "reason": why}) |
|
|
| |
| |
| |
| hard_terms = set(getattr(profile, "immutable", set()) or set()) | set(user_terms or set()) |
| for kind, code in [ |
| ("citation", "TERMINOLOGY_REJECTION"), |
| ("quantity", "QUANTITY_REJECTION"), |
| ("url", "TERMINOLOGY_REJECTION"), |
| ("equation", "TERMINOLOGY_REJECTION"), |
| ("code", "TERMINOLOGY_REJECTION"), |
| ]: |
| source_inventory = Counter( |
| item[1].lower() |
| for item in protect(original, hard_terms) |
| if item[0] == kind |
| ) |
| candidate_inventory = Counter( |
| item[1].lower() |
| for item in protect(revised, hard_terms) |
| if item[0] == kind |
| ) |
| checks[kind] = source_inventory == candidate_inventory |
| if source_inventory != candidate_inventory: |
| reject( |
| code, |
| f"{kind} inventory changed: {dict(source_inventory)} -> {dict(candidate_inventory)}", |
| ) |
|
|
| missing_terms = [ |
| term |
| for term in hard_terms |
| if re.search(r"(?<!\w)" + re.escape(term) + r"(?!\w)", original, re.I) |
| and not re.search(r"(?<!\w)" + re.escape(term) + r"(?!\w)", revised, re.I) |
| ] |
| if missing_terms: |
| reject( |
| "TERMINOLOGY_REJECTION", |
| "Missing immutable/user terms: " + ", ".join(sorted(missing_terms)), |
| ) |
|
|
| |
| |
| |
| acronym_issues = _acronym_mapping_issues(original, revised) |
| tautology_issues = _semantic_tautology_issues(revised) |
| checks["acronym_mapping_issues"] = acronym_issues |
| checks["semantic_tautology_issues"] = tautology_issues |
| if acronym_issues: |
| reject( |
| "ACRONYM_MAPPING_REJECTION", |
| "A discipline-specific acronym pair or acronym inventory changed: " |
| + json.dumps(acronym_issues, ensure_ascii=False), |
| ) |
| if tautology_issues: |
| reject( |
| "GRAMMAR_REJECTION", |
| "Candidate contains an obvious acronym tautology: " |
| + " | ".join(tautology_issues), |
| ) |
|
|
| |
| |
| |
| source_bound_issues = _source_bounded_integrity_issues(original, revised) |
| checks["source_bounded_integrity_issues"] = source_bound_issues |
| if source_bound_issues: |
| issue_types = {item.get("type") for item in source_bound_issues} |
| if issue_types & { |
| "REPEATED_SENTENCE_INTRODUCED", |
| "REPEATED_SENTENCE_OPENING_INTRODUCED", |
| "REPEATED_LEXICAL_SPAN_INTRODUCED", |
| "CONTENT_LEMMA_REPETITION_INTRODUCED", |
| }: |
| reject( |
| "REPETITION_REJECTION", |
| "Candidate introduced repetition not present in the source: " |
| + json.dumps(source_bound_issues, ensure_ascii=False), |
| ) |
| if issue_types & {"NAMED_ENTITY_INTRODUCED", "EXCESSIVE_CONTENT_EXPANSION"}: |
| reject( |
| "CONTENT_ADDITION_REJECTION", |
| "Candidate introduced or expanded content beyond the source: " |
| + json.dumps(source_bound_issues, ensure_ascii=False), |
| ) |
|
|
| |
| |
| |
| realization = list(realization_issues(revised) or []) |
| checks["realization_issues"] = realization |
| if realization: |
| reject( |
| "GRAMMAR_REJECTION", |
| "Candidate failed realization checks: " + " | ".join(realization), |
| ) |
|
|
| source_doc = _safe_doc(original) |
| candidate_doc = _safe_doc(revised) |
| checks["source_structure"] = _v5_structural_signature(source_doc) |
| checks["candidate_structure"] = _v5_structural_signature(candidate_doc) |
| checks["candidate_rule_kind"] = _v5_family_kind(candidate) |
|
|
| |
| |
| |
| |
| source_neg = _neg_inventory(original, source_doc) |
| candidate_neg = _neg_inventory(revised, candidate_doc) |
| checks["negation_inventory"] = (dict(source_neg), dict(candidate_neg)) |
| checks["negation_scope_debug"] = ( |
| _neg_scope_debug(source_doc), |
| _neg_scope_debug(candidate_doc), |
| ) |
|
|
| polarity_preserved = source_neg == candidate_neg |
| if ( |
| not polarity_preserved |
| and candidate.family.startswith("NEGATED_PURPOSE_CONTRAST") |
| and _purpose_contrast_preserved(original, revised) |
| ): |
| polarity_preserved = True |
|
|
| if not polarity_preserved: |
| reject( |
| "POLARITY_REJECTION", |
| f"Polarity-marker inventory changed: {dict(source_neg)} -> {dict(candidate_neg)}", |
| ) |
|
|
| |
| |
| |
| source_modals = _modals(source_doc, original) |
| candidate_modals = _modals(candidate_doc, revised) |
| checks["modality"] = (dict(source_modals), dict(candidate_modals)) |
| if source_modals != candidate_modals: |
| reject( |
| "MODALITY_REJECTION", |
| f"Modality/hedging changed: {dict(source_modals)} -> {dict(candidate_modals)}", |
| ) |
|
|
| |
| |
| |
| |
| |
| enumeration_groups = _enumeration_groups(analysis, original, source_doc) |
| missing_by_group = [] |
| for group_index, group in enumerate(enumeration_groups, start=1): |
| missing = [member for member in group if not _member_present(member, revised)] |
| if missing: |
| missing_by_group.append({"group": group_index, "missing": missing}) |
|
|
| checks["enumeration_groups"] = enumeration_groups |
| checks["enumeration_missing"] = missing_by_group |
| if missing_by_group: |
| detail = " | ".join( |
| f"group {entry['group']}: " + ", ".join(entry["missing"]) |
| for entry in missing_by_group |
| ) |
| reject( |
| "ENUMERATION_REJECTION", |
| "Enumeration members missing: " + detail, |
| ) |
|
|
| |
| |
| |
| source_triples = _triples(source_doc) |
| candidate_triples = _triples(candidate_doc) |
| checks["source_triples"] = source_triples |
| checks["candidate_triples"] = candidate_triples |
| for subject, verb, obj in source_triples: |
| if subject != obj and any( |
| cs == obj and cq == subject and cv == verb |
| for cs, cv, cq in candidate_triples |
| ): |
| reject( |
| "SEMANTIC_REJECTION", |
| f"Subject/object orientation reversed for '{verb}'.", |
| ) |
|
|
| if not source_triples and not candidate_triples: |
| simple = ( |
| r"(?iu)^\s*(?P<s>[^\W\d_][\w-]*)\s+" |
| r"(?P<v>[^\W\d_][\w-]*)\s+(?P<o>[^\W\d_][\w-]*)" |
| ) |
| source_match = re.match(simple, original) |
| candidate_match = re.match(simple, revised) |
| if ( |
| source_match and candidate_match |
| and source_match["s"].lower() == candidate_match["o"].lower() |
| and source_match["o"].lower() == candidate_match["s"].lower() |
| and source_match["v"].lower() == candidate_match["v"].lower() |
| ): |
| reject( |
| "SEMANTIC_REJECTION", |
| f"Subject/object orientation reversed for '{source_match['v']}'.", |
| ) |
|
|
| |
| |
| |
| source_relations = _relation_signature(original) |
| candidate_relations = _relation_signature(revised) |
| checks["relations"] = (sorted(source_relations), sorted(candidate_relations)) |
| if "concession" in candidate_relations - source_relations: |
| reject( |
| "SEMANTIC_REJECTION", |
| f"Candidate introduced concession: {sorted(source_relations)} -> {sorted(candidate_relations)}", |
| ) |
| if "cause" in source_relations and not ({"cause", "purpose"} & candidate_relations): |
| reject( |
| "SEMANTIC_REJECTION", |
| f"Causal/result relation disappeared: {sorted(source_relations)} -> {sorted(candidate_relations)}", |
| ) |
| if "absence" in source_relations and "absence" not in candidate_relations: |
| reject( |
| "POLARITY_REJECTION", |
| "The source's 'without' relation disappeared from the candidate.", |
| ) |
| if "dependency" in source_relations and "dependency" not in candidate_relations: |
| reject( |
| "SEMANTIC_REJECTION", |
| "The source dependency/contingency relation disappeared.", |
| ) |
|
|
| |
| |
| |
| sts = stsim(original, revised) |
| forward = nli_pair(original, revised) |
| reverse = nli_pair(revised, original) |
| checks.update(sts=sts, nli_forward=forward, nli_reverse=reverse) |
|
|
| if sts is not None and sts < CFG.min_sts: |
| checks["sts_warning"] = f"Low soft STS signal: {sts:.3f}" |
|
|
| if forward and reverse: |
| if max(forward["contradiction"], reverse["contradiction"]) > .55: |
| reject("SEMANTIC_REJECTION", "Bidirectional NLI found contradiction.") |
| elif min(forward["entailment"], reverse["entailment"]) < CFG.min_nli: |
| checks["nli_warning"] = ( |
| f"Weak entailment {forward['entailment']:.3f}/{reverse['entailment']:.3f}; " |
| "manual review advised." |
| ) |
|
|
| |
| |
| |
| surface = sr(original, revised) |
| change = token_change(original, revised) |
| checks.update(surface=surface, change=change, missing_terms=missing_terms) |
| minimum_change = {"Minimal": .035, "Balanced": .065, "Structural": .09}[mode] |
|
|
| if surface_key(original) == surface_key(revised): |
| reject("INSUFFICIENT_REWRITE", "Candidate is identical to the source.") |
| elif change < minimum_change: |
| reject( |
| "INSUFFICIENT_REWRITE", |
| f"Token change {change:.3f} is below {mode} threshold {minimum_change:.3f}.", |
| ) |
|
|
| |
| |
| |
| grammar_tool = optional("lt") if getattr(CFG, "lt", False) else None |
| if grammar_tool: |
| try: |
| delta = len(grammar_tool.check(revised)) - len(grammar_tool.check(original)) |
| checks["grammar_delta"] = delta |
| if delta > 2: |
| reject( |
| "GRAMMAR_REJECTION", |
| f"Candidate introduced {delta} additional LanguageTool findings.", |
| ) |
| except Exception as exc: |
| checks["grammar_warning"] = str(exc) |
|
|
| checks.update( |
| failures=failures, |
| rejection_reasons=reasons, |
| accepted=not failures, |
| ) |
| return not failures, failures, checks |
|
|
|
|
| |
| |
| |
|
|
| class Ranker: |
| def rank(self, original, candidates, analysis, profile, user_terms, mode): |
| good = [] |
| bad = [] |
| target = {"Minimal": .12, "Balanced": .24, "Structural": .38}[mode] |
|
|
| for candidate in candidates: |
| ok, failures, checks = Validator().validate( |
| original, candidate, analysis, profile, user_terms, mode |
| ) |
| change = checks.get("change", token_change(original, candidate.text)) |
| fit = max(0, 1 - abs(change - target) / max(target, .10)) |
| sts = checks.get("sts") |
| semantic = .78 if sts is None else max(0, min(1, sts)) |
| nli_scores = [ |
| item["entailment"] |
| for item in [checks.get("nli_forward"), checks.get("nli_reverse")] |
| if item |
| ] |
| if nli_scores: |
| semantic = .55 * semantic + .45 * min(nli_scores) |
|
|
| lane_fit = ( |
| 1.0 if candidate.lane == mode |
| else .82 if (mode, candidate.lane) in { |
| ("Balanced", "Minimal"), |
| ("Balanced", "Structural"), |
| ("Structural", "Balanced"), |
| } |
| else .65 |
| ) |
| grammar = max(.6, 1 - .1 * max(0, checks.get("grammar_delta", 0))) |
| style_penalties = _human_style_penalties(original, candidate.text) |
| checks["human_style_penalties"] = style_penalties |
| style_penalty = sum(float(item.get("weight", 0) or 0) for item in style_penalties) |
| |
| |
| |
| |
| rule_kind = checks.get("candidate_rule_kind", "specialized") |
| provenance_fit = .98 if rule_kind == "universal" else 1.0 |
| candidate.score = ( |
| 100 * ( |
| .42 * semantic |
| + .22 * fit |
| + .15 * lane_fit |
| + .12 * grammar |
| + .07 * candidate.priority |
| + .02 * provenance_fit |
| - style_penalty |
| ) |
| if ok else -1e9 |
| ) |
| candidate.checks = checks |
| (good if ok else bad).append(candidate) |
|
|
| good.sort(key=lambda item: item.score, reverse=True) |
| bad.sort(key=lambda item: ( |
| len(item.checks.get("failures", [])), |
| -item.checks.get("change", 0), |
| )) |
| return good, bad |
|
|
|
|
| |
| |
| |
|
|
| class Engine: |
| def __init__(self): |
| self.an = Analyzer() |
| self.gen = Generator() |
| self.sc = Scanner() |
| self.rank = Ranker() |
|
|
| def one(self, sentence, profile, user_terms, mode): |
| _ensure_state() |
| analysis = self.an.analyze(sentence, profile, user_terms) |
|
|
| if analysis.families == ["PARSE_FAILURE"]: |
| return SentenceResult( |
| sentence, sentence, "PARSE_FAILURE", analysis.families, None, |
| "Dependency parse unavailable and no surface-safe family matched.", |
| [], {}, {"detected": analysis.families}, |
| ) |
|
|
| candidates, trace = self.gen.generate( |
| analysis, profile, user_terms, mode |
| ) |
|
|
| if not candidates: |
| code = ( |
| "NO_PATTERN_COVERAGE" |
| if not trace.get("routes") |
| else "CANDIDATE_GENERATION_FAILURE" |
| ) |
| return SentenceResult( |
| sentence, sentence, code, analysis.families, None, |
| "No transformation produced a distinct candidate for the detected structures.", |
| [], {}, trace, |
| ) |
|
|
| good, bad = self.rank.rank( |
| sentence, candidates, analysis, profile, user_terms, mode |
| ) |
|
|
| trace["candidate_traces"] = [ |
| { |
| "text": item.text, |
| "lane": item.lane, |
| "family": item.family, |
| "reason": item.reason, |
| "accepted": item.checks.get("accepted"), |
| "failures": item.checks.get("failures", []), |
| "rejection_reasons": item.checks.get("rejection_reasons", []), |
| "change": item.checks.get("change"), |
| "sts": item.checks.get("sts"), |
| "nli_forward": item.checks.get("nli_forward"), |
| "nli_reverse": item.checks.get("nli_reverse"), |
| "score": item.score, |
| "rule_kind": item.checks.get("candidate_rule_kind"), |
| "source_structure": item.checks.get("source_structure"), |
| "candidate_structure": item.checks.get("candidate_structure"), |
| "acronym_mapping_issues": item.checks.get("acronym_mapping_issues", []), |
| "human_style_penalties": item.checks.get("human_style_penalties", []), |
| "source_bounded_integrity_issues": item.checks.get("source_bounded_integrity_issues", []), |
| } |
| for item in good + bad |
| ] |
|
|
| if not good: |
| counts = Counter( |
| code |
| for item in bad |
| for code in item.checks.get("failures", []) |
| ) |
| precedence = [ |
| "CONTENT_ADDITION_REJECTION", |
| "REPETITION_REJECTION", |
| "ACRONYM_MAPPING_REJECTION", |
| "TERMINOLOGY_REJECTION", |
| "POLARITY_REJECTION", |
| "MODALITY_REJECTION", |
| "QUANTITY_REJECTION", |
| "ENUMERATION_REJECTION", |
| "GRAMMAR_REJECTION", |
| "SEMANTIC_REJECTION", |
| "INSUFFICIENT_REWRITE", |
| ] |
| status = next((code for code in precedence if counts[code]), "RANKING_EMPTY") |
| return SentenceResult( |
| sentence, sentence, status, analysis.families, None, |
| "Candidates were generated, but none passed all hard safeguards and rewrite gates.", |
| [item.text for item in bad[:3]], |
| bad[0].checks if bad else {}, |
| trace | {"rejections": dict(counts)}, |
| ) |
|
|
| selected = good[0] |
| trace["selected"] = { |
| "text": selected.text, |
| "lane": selected.lane, |
| "family": selected.family, |
| "score": selected.score, |
| } |
| return SentenceResult( |
| sentence, selected.text, "SUCCESS", analysis.families, |
| selected.family, selected.reason, |
| [item.text for item in good[1:4]], |
| selected.checks, trace, |
| ) |
|
|
| def process(self, text, workflow, mode, discipline, terms_text, progress=None): |
| STATE.clear() |
| STATE.update(start=time.time(), warnings=[]) |
| profile, evidence = self.route(text, discipline) |
| user_terms = { |
| item.strip() |
| for item in re.split(r"[,;\n]", terms_text or "") |
| if item.strip() |
| } |
| findings, score, band, features = self.sc.scan(text) |
|
|
| if workflow == "Scan Only": |
| return DocumentResult( |
| text, text, workflow, mode, profile.name, evidence, |
| score, band, findings, [], STATE["warnings"], |
| {"features": features}, |
| ) |
|
|
| paragraphs = [ |
| item.strip() |
| for item in re.split(r"\n\s*\n", text) |
| if item.strip() |
| ] |
| total = sum(len(sent_split(item)) for item in paragraphs) |
| done = 0 |
| results = [] |
| output_paragraphs = [] |
|
|
| for block_start in range(0, len(paragraphs), CFG.batch): |
| for paragraph in paragraphs[block_start:block_start + CFG.batch]: |
| output_sentences = [] |
| for sentence in sent_split(paragraph): |
| targeted = ( |
| workflow == "Edit / Paraphrase Only" |
| or any(finding.sentence == len(results) for finding in findings) |
| ) |
| if targeted: |
| item = self.one(sentence, profile, user_terms, mode) |
| else: |
| item = SentenceResult( |
| sentence, sentence, "NOT_TARGETED", [], None, |
| "No scanner finding targeted this sentence.", |
| [], {}, {}, |
| ) |
| results.append(item) |
| output_sentences.append(item.revised) |
| done += 1 |
| if progress: |
| progress( |
| done / max(total, 1), |
| desc=f"Sentence {done}/{total}", |
| ) |
| output_paragraphs.append(" ".join(output_sentences)) |
|
|
| return DocumentResult( |
| text, |
| "\n\n".join(output_paragraphs), |
| workflow, |
| mode, |
| profile.name, |
| evidence, |
| score, |
| band, |
| findings, |
| results, |
| STATE["warnings"], |
| { |
| "sentences": total, |
| "success": sum(item.status == "SUCCESS" for item in results), |
| "unchanged": sum(item.original == item.revised for item in results), |
| "seconds": time.time() - STATE["start"], |
| }, |
| ) |
|
|
| def route(self, text, requested): |
| if requested in PROFILES: |
| return PROFILES[requested], ["manual"] |
|
|
| lo = text.lower() |
| scores = { |
| name: sum( |
| bool(re.search(r"(?<!\w)" + re.escape(term.lower()) + r"(?!\w)", lo)) |
| for term in profile.keywords | profile.phrases |
| ) |
| for name, profile in PROFILES.items() |
| } |
| winner = max(scores, key=scores.get) |
| winner = winner if scores[winner] else "General Academic" |
| evidence = [ |
| term for term in PROFILES[winner].keywords |
| if term.lower() in lo |
| ][:10] |
| return PROFILES[winner], evidence |
|
|
|
|
| ENGINE = Engine() |
|
|
|
|
| |
| |
| |
|
|
| from docx import Document |
|
|
|
|
| def report(result): |
| folder = Path(CFG.out) / ("NLL_" + time.strftime("%Y%m%d_%H%M%S")) |
| folder.mkdir(parents=True, exist_ok=True) |
|
|
| revised_path = folder / "revised.txt" |
| diagnostics_path = folder / "diagnostics.json" |
| revised_path.write_text(result.revised, encoding="utf8") |
| diagnostics_path.write_text( |
| json.dumps(asdict(result), indent=2, default=str), |
| encoding="utf8", |
| ) |
|
|
| document = Document() |
| document.add_heading("NLL Revised Document", 0) |
| document.add_paragraph(result.revised) |
| document.add_heading("Review", 1) |
| document.add_paragraph( |
| f"Writing Pattern Score: {result.pattern_score:.0f}/100 ({result.band}). " |
| "The score describes observable structural and stylistic patterns only." |
| ) |
|
|
| for index, item in enumerate(result.sentences, 1): |
| document.add_heading(f"Sentence {index}: {item.status}", 2) |
| document.add_paragraph("Original: " + item.original) |
| document.add_paragraph("Revised: " + item.revised) |
| document.add_paragraph("Reason: " + item.reason) |
| document.add_paragraph("Detected structures: " + ", ".join(item.families)) |
| document.add_paragraph( |
| "Diagnostics: " + json.dumps(item.diagnostics, default=str)[:4000] |
| ) |
|
|
| doc_path = folder / "revised_and_review.docx" |
| document.save(doc_path) |
| return [str(doc_path), str(revised_path), str(diagnostics_path)] |
|
|
|
|
| |
| |
| |
|
|
| _ensure_state() |
|
|
| |
| |
| _test_profile = next(iter(PROFILES.values())) |
| _test_analyzer = Analyzer() |
| _pol_source = ( |
| "Systems train separate models without explicit rules for assessment, " |
| "correction, and graduation." |
| ) |
| _pol_candidate = Candidate( |
| "Without explicit rules for assessment, correction, and graduation, " |
| "systems train separate models.", |
| "Balanced", |
| "FINITE_COMPLEMENT_LIST", |
| "validator self-test", |
| .9, |
| ) |
| _pol_analysis = _test_analyzer.analyze(_pol_source, _test_profile, set()) |
| _pol_ok, _pol_fail, _ = Validator().validate( |
| _pol_source, _pol_candidate, _pol_analysis, _test_profile, set(), "Balanced" |
| ) |
| if "POLARITY_REJECTION" in _pol_fail: |
| raise RuntimeError( |
| "Cell 7 v5.6 failed: polarity-preserving 'without' movement was rejected." |
| ) |
|
|
| |
| _enum_source = "We propose Framework X, a protocol consisting of contracts, examinations, and promotion." |
| _enum_candidate = Candidate( |
| "Framework X is a protocol consisting of contracts, examinations, and promotion proposed by us.", |
| "Balanced", |
| "APPOSITIVE_DEFINITION", |
| "validator self-test", |
| .9, |
| ) |
| _enum_analysis = _test_analyzer.analyze(_enum_source, _test_profile, set()) |
| _, _enum_fail, _ = Validator().validate( |
| _enum_source, _enum_candidate, _enum_analysis, _test_profile, set(), "Balanced" |
| ) |
| if "ENUMERATION_REJECTION" in _enum_fail: |
| raise RuntimeError( |
| "Cell 7 v5.6 failed: preserved enumeration was rejected." |
| ) |
|
|
| |
| _bad_candidate = Candidate( |
| "Protocol to learners spanning classification, drafting, and planning.", |
| "Structural", |
| "PARTICIPIAL_POSTMODIFIER", |
| "validator self-test", |
| .9, |
| ) |
| _bad_analysis = _test_analyzer.analyze( |
| "The testbed applies the protocol to learners spanning classification, drafting, and planning.", |
| _test_profile, |
| set(), |
| ) |
| _, _bad_fail, _ = Validator().validate( |
| "The testbed applies the protocol to learners spanning classification, drafting, and planning.", |
| _bad_candidate, |
| _bad_analysis, |
| _test_profile, |
| set(), |
| "Structural", |
| ) |
| if "GRAMMAR_REJECTION" not in _bad_fail: |
| raise RuntimeError( |
| "Cell 7 v5.6 failed: malformed participial fragment was not rejected." |
| ) |
|
|
| |
| _long_enum_source = ( |
| "We propose Framework X (FX), a protocol consisting of native model contracts, " |
| "historical contracts, protected examinations, peer study, and promotion." |
| ) |
| _long_enum_candidate = Candidate( |
| "Framework X (FX) is a protocol consisting of native model contracts, " |
| "historical contracts, protected examinations, peer study, and promotion proposed by us.", |
| "Balanced", "APPOSITIVE_DEFINITION", "validator self-test", .9, |
| ) |
| _long_enum_analysis = _test_analyzer.analyze(_long_enum_source, _test_profile, set()) |
| _, _long_enum_fail, _ = Validator().validate( |
| _long_enum_source, _long_enum_candidate, _long_enum_analysis, |
| _test_profile, set(), "Balanced" |
| ) |
| if "ENUMERATION_REJECTION" in _long_enum_fail: |
| raise RuntimeError( |
| "Cell 7 v5.6 failed: appositive boundary corrupted enumeration extraction." |
| ) |
|
|
| |
| _hyphen_source = ( |
| "We pre-register baseline questions, disjoint re-examination data, matched-data " |
| "baselines, ablations, and independently held-out testing." |
| ) |
| _hyphen_candidate = Candidate( |
| "Our preregistration covers baseline questions, disjoint re-examination data, " |
| "matched-data baselines, ablations, and independently held-out testing.", |
| "Balanced", "FINITE_OBJECT_LIST_FRAME", "validator self-test", .9, |
| ) |
| _hyphen_analysis = _test_analyzer.analyze(_hyphen_source, _test_profile, set()) |
| _, _hyphen_fail, _ = Validator().validate( |
| _hyphen_source, _hyphen_candidate, _hyphen_analysis, |
| _test_profile, set(), "Balanced" |
| ) |
| if "ENUMERATION_REJECTION" in _hyphen_fail: |
| raise RuntimeError( |
| "Cell 7 v5.6 failed: hyphenated list member was split or lost." |
| ) |
|
|
| |
| if _unicode_tokens("re-examination data") != _unicode_tokens("re examination data"): |
| raise RuntimeError( |
| "Cell 7 v5.6 failed: ASCII hyphen token normalization mismatch." |
| ) |
| if _unicode_tokens("matched–data") != _unicode_tokens("matched-data"): |
| raise RuntimeError( |
| "Cell 7 v5.6 failed: Unicode dash token normalization mismatch." |
| ) |
| if _unicode_tokens("ALCEF–RSF") != _unicode_tokens("ALCEF-RSF"): |
| raise RuntimeError( |
| "Cell 7 v5.6 failed: acronym dash normalization mismatch." |
| ) |
|
|
| |
| |
| if _plausible_enumeration_member( |
| "Framework X (FX), a protocol consisting of native contracts, historical contracts" |
| ): |
| raise RuntimeError( |
| "Cell 7 v5.6 failed: appositive pseudo-member passed confidence gate." |
| ) |
| if not _plausible_enumeration_member("disjoint re-examination data"): |
| raise RuntimeError( |
| "Cell 7 v5.6 failed: valid hyphenated member failed confidence gate." |
| ) |
| if not _plausible_enumeration_member("independently held-out testing"): |
| raise RuntimeError( |
| "Cell 7 v5.6 failed: valid multiword member failed confidence gate." |
| ) |
|
|
| |
| _unseen_source = "Models predicting regional demand inform the allocation process." |
| _unseen_candidate = Candidate( |
| "Models that predict regional demand inform the allocation process.", |
| "Balanced", "universal_participial_modifier", "validator self-test", .87, |
| ) |
| _unseen_analysis = _test_analyzer.analyze(_unseen_source, _test_profile, set()) |
| _unseen_ok, _unseen_fail, _unseen_checks = Validator().validate( |
| _unseen_source, _unseen_candidate, _unseen_analysis, _test_profile, set(), "Balanced" |
| ) |
| if "GRAMMAR_REJECTION" in _unseen_fail: |
| raise RuntimeError( |
| "Cell 7 v5.6 failed: valid unseen-vocabulary finite relative clause was rejected: " |
| + str(_unseen_fail) |
| ) |
|
|
| |
| _passive_source = "The unfamiliar alloy was cryogenically stabilized by the laboratory." |
| _passive_candidate = Candidate( |
| "The laboratory cryogenically stabilized the unfamiliar alloy.", |
| "Structural", "universal_explicit_agent_passive", "validator self-test", .89, |
| ) |
| _passive_analysis = _test_analyzer.analyze(_passive_source, _test_profile, set()) |
| _passive_ok, _passive_fail, _ = Validator().validate( |
| _passive_source, _passive_candidate, _passive_analysis, _test_profile, set(), "Structural" |
| ) |
| if "GRAMMAR_REJECTION" in _passive_fail or "SEMANTIC_REJECTION" in _passive_fail: |
| raise RuntimeError( |
| "Cell 7 v5.6 failed: valid unseen-vocabulary passive/active recast was rejected: " |
| + str(_passive_fail) |
| ) |
|
|
| |
| _v5_rule_names = [registered.name for registered in RULES] |
| _v5_duplicates = sorted(name for name, count in Counter(_v5_rule_names).items() if count > 1) |
| if _v5_duplicates: |
| raise RuntimeError("Cell 7 v5.6 duplicate rule registration: " + ", ".join(_v5_duplicates)) |
| if len(RULES) != EXPECTED_CELL7_RULE_COUNT: |
| raise RuntimeError( |
| f"Cell 7 v5.6 expected {EXPECTED_CELL7_RULE_COUNT} rules but found {len(RULES)}." |
| ) |
|
|
| |
| _term_source = ( |
| "The process compared spectral response matrix (SRM), adaptive control " |
| "protocol (ACP), and baseline measurements." |
| ) |
| _term_swapped_bad = Candidate( |
| "The process compared spectral response matrix (ACP), adaptive control " |
| "protocol (SRM), and baseline measurements.", |
| "Balanced", "TEST", "generic acronym mapping self-test", .5, |
| ) |
| _term_analysis = _test_analyzer.analyze(_term_source, _test_profile, set()) |
| _, _term_bad_fail, _term_bad_checks = Validator().validate( |
| _term_source, _term_swapped_bad, _term_analysis, _test_profile, set(), "Balanced" |
| ) |
| if "ACRONYM_MAPPING_REJECTION" not in _term_bad_fail: |
| raise RuntimeError( |
| "Cell 7 v5.6 failed: swapped generic acronym mappings were not rejected: " |
| + str(_term_bad_checks.get("acronym_mapping_issues")) |
| ) |
|
|
| |
| _term_reordered_good = Candidate( |
| "The process compared adaptive control protocol (ACP), spectral response " |
| "matrix (SRM), and baseline measurements.", |
| "Balanced", "TEST", "generic acronym reorder self-test", .5, |
| ) |
| _, _term_good_fail, _term_good_checks = Validator().validate( |
| _term_source, _term_reordered_good, _term_analysis, _test_profile, set(), "Balanced" |
| ) |
| if "ACRONYM_MAPPING_REJECTION" in _term_good_fail: |
| raise RuntimeError( |
| "Cell 7 v5.6 failed: intact reordered discipline-term pairs were rejected: " |
| + str(_term_good_checks.get("acronym_mapping_issues")) |
| ) |
|
|
| |
| _term_separated_bad = Candidate( |
| "The process compared SRM, adaptive control protocol (ACP), and baseline measurements.", |
| "Balanced", "TEST", "generic acronym separation self-test", .5, |
| ) |
| _, _term_sep_fail, _ = Validator().validate( |
| _term_source, _term_separated_bad, _term_analysis, _test_profile, set(), "Balanced" |
| ) |
| if "ACRONYM_MAPPING_REJECTION" not in _term_sep_fail: |
| raise RuntimeError( |
| "Cell 7 v5.6 failed: separated discipline-term pair was not rejected." |
| ) |
|
|
| |
| if _acronym_inventory("The method used SRM, ACP, FGSM, and H2.") != Counter({"SRM":1,"ACP":1,"FGSM":1,"H2":1}): |
| raise RuntimeError("Cell 7 v5.6 failed: generic acronym inventory is inaccurate.") |
| if _acronym_inventory("The framework compared raw features."): |
| raise RuntimeError("Cell 7 v5.6 failed: ordinary words were misclassified as acronyms.") |
| if _semantic_tautology_issues("SRM is SRM.") != ["REDUNDANT_ACRONYM_COPULAR_TAUTOLOGY"]: |
| raise RuntimeError("Cell 7 v5.6 failed: generic acronym tautology was not detected.") |
|
|
| |
|
|
|
|
| _style_probe = _human_style_penalties( |
| "Android conditions did not remain within the thresholds; H2 was not supported.", |
| "According to H2, Android conditions did not remain within the thresholds. " |
| "This was not supported.", |
| ) |
| _style_codes = {item["code"] for item in _style_probe} |
| if "AWKWARD_HYPOTHESIS_ATTRIBUTION" not in _style_codes: |
| raise RuntimeError("Cell 7 v5.6 failed: awkward H2 attribution penalty missing.") |
| if "NEW_DEMONSTRATIVE_ANTECEDENT" not in _style_codes: |
| raise RuntimeError("Cell 7 v5.6 failed: demonstrative antecedent penalty missing.") |
|
|
| |
| _repeat_source = ( |
| "The literature of history encompasses foundational narratives, " |
| "transformative methodologies, and compelling modern accounts." |
| ) |
| _repeat_bad = ( |
| "The literature of history encompasses foundational narratives. " |
| "The literature of history encompasses transformative methodologies and compelling modern accounts." |
| ) |
| _repeat_issues = _source_bounded_integrity_issues(_repeat_source, _repeat_bad) |
| _repeat_types = {item.get("type") for item in _repeat_issues} |
| if "REPEATED_SENTENCE_OPENING_INTRODUCED" not in _repeat_types: |
| raise RuntimeError( |
| "Cell 7 v5.6 failed: repeated sentence opening evidence is missing: " |
| + repr(_repeat_issues) |
| ) |
| _repeat_candidate = Candidate( |
| _repeat_bad, "Structural", "TEST", "source-bounded repetition self-test", .5 |
| ) |
| _repeat_analysis = _test_analyzer.analyze(_repeat_source, _test_profile, set()) |
| _, _repeat_failures, _repeat_checks = Validator().validate( |
| _repeat_source, |
| _repeat_candidate, |
| _repeat_analysis, |
| _test_profile, |
| set(), |
| "Structural", |
| ) |
| if "REPETITION_REJECTION" not in _repeat_failures: |
| raise RuntimeError( |
| "Cell 7 v5.6 failed: repeated output was not rejected by Validator: " |
| + repr(_repeat_checks.get("source_bounded_integrity_issues")) |
| ) |
|
|
| _repeat_good = ( |
| "Foundational narratives shape the field. " |
| "Transformative methodologies explain later developments." |
| ) |
| if _source_bounded_integrity_issues(_repeat_source, _repeat_good): |
| raise RuntimeError( |
| "Cell 7 v5.6 failed: distinct sentence openings were falsely flagged: " |
| + repr(_source_bounded_integrity_issues(_repeat_source, _repeat_good)) |
| ) |
|
|
| _exact_repeat_bad = "The model was evaluated. The model was evaluated." |
| _exact_repeat_issues = _source_bounded_integrity_issues( |
| "The model was evaluated.", _exact_repeat_bad |
| ) |
| if not any(item.get("type") == "REPEATED_SENTENCE_INTRODUCED" for item in _exact_repeat_issues): |
| raise RuntimeError( |
| "Cell 7 v5.6 failed: repeated sentence was not detected: " |
| + repr(_exact_repeat_issues) |
| ) |
|
|
| _entity_addition_issues = _source_bounded_integrity_issues( |
| "The study evaluated the classifier.", |
| "The study evaluated the classifier at Microsoft Research.", |
| ) |
| if _named_entity_inventory("The study evaluated the classifier at Microsoft Research.") and not any( |
| item.get("type") == "NAMED_ENTITY_INTRODUCED" for item in _entity_addition_issues |
| ): |
| raise RuntimeError( |
| "Cell 7 v5.6 failed: introduced named entity was not detected: " |
| + repr(_entity_addition_issues) |
| ) |
|
|
| |
| |
| _source_bound_good = _source_bounded_integrity_issues( |
| "Models predicting regional demand inform the allocation process.", |
| "Models that predict regional demand inform the allocation process.", |
| ) |
| if _source_bound_good: |
| raise RuntimeError( |
| "Cell 7 v5.6 failed: safe grammatical scaffolding was rejected: " |
| + repr(_source_bound_good) |
| ) |
|
|
| _pair_probe = _acronym_pairs( |
| "The process compared spectral response matrix (SRM), adaptive control protocol (ACP), and baselines." |
| ) |
| _pair_probe_keys = { |
| (pair["expansion_tokens"], pair["acronym"]) |
| for pair in _pair_probe |
| } |
| _expected_pair_probe = { |
| (("spectral", "response", "matrix"), "SRM"), |
| (("adaptive", "control", "protocol"), "ACP"), |
| } |
| if _pair_probe_keys != _expected_pair_probe: |
| raise RuntimeError( |
| "Cell 7 v5.6 failed: generic atomic-pair extraction mismatch: " |
| + repr(_pair_probe) |
| ) |
|
|
| print("=" * 78) |
| print("CELL 7 UNIVERSAL VALIDATION / RANKING / ENGINE v5.6 LOADED") |
| print("=" * 78) |
| print("Validator: OK") |
| print("Ranker: OK") |
| print("Engine: OK") |
| print("ENGINE instance: OK") |
| print("Polarity-inventory validation: active") |
| print("Dependency-aware Unicode enumeration validation: active") |
| print("Appositive/list-boundary protection: active") |
| print("Hyphen/dash-insensitive content comparison: active") |
| print("Parser-artifact group filtering: active") |
| print("Enumeration-member confidence gate: active") |
| print("Noisy-superset group suppression: active") |
| print("Source-bounded non-generative integrity: active") |
| print("New named-entity addition rejection: active") |
| print("Repeated sentence/opening/span rejection: active") |
| print("Adaptive three-lemma sentence-opening detection: active") |
| print("Validator-level repetition rejection self-test: PASSED") |
| print("Content-expansion rejection: active") |
| print("42-rule specialized + universal candidate judging: active") |
| print("Vocabulary-independent structural signatures: active") |
| print("Universal-rule ranking provenance: active") |
| print("Generic atomic discipline-term extraction: active") |
| print("No hard-coded acronym vocabulary: active") |
| print("Acronym-definition mapping safeguard: active") |
| print("Case-sensitive acronym discovery without hard-coded terms: active") |
| print("Acronym separation/swap/duplication rejection: active") |
| print("Acronym tautology rejection: active") |
| print("Human-style hypothesis-attribution penalty: active") |
| print("Demonstrative-antecedent ranking penalty: active") |
| print("Unseen-vocabulary judge self-tests: PASSED") |
| print("Cell 5A realization gate rechecked by judge: active") |
| print("Optional STS/NLI signals: soft unless contradiction is detected") |
| print("Runtime warning-state safety: active") |
| print("Validator self-tests: PASSED") |
| print("Registered transformation rules:", len(RULES)) |
| print("Expected rule count:", EXPECTED_CELL7_RULE_COUNT) |
| print("=" * 78) |
|
|
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
|
|
| EXPECTED_7A_RULE_COUNT = 42 |
|
|
| _REQUIRED_7A = [ |
| "STATE", "RULES", "Validator", "Ranker", "Engine", "ENGINE", |
| "surface_key", "realization_issues", |
| ] |
| _missing_7a = [name for name in _REQUIRED_7A if name not in globals()] |
| if _missing_7a: |
| raise RuntimeError( |
| "Cell 7A cannot run. Missing: " + ", ".join(_missing_7a) |
| + ". Run Cell 7 v5 first." |
| ) |
|
|
| if len(RULES) != EXPECTED_7A_RULE_COUNT: |
| raise RuntimeError( |
| f"Cell 7A expected {EXPECTED_7A_RULE_COUNT} rules but found {len(RULES)}." |
| ) |
|
|
|
|
| |
| |
| |
|
|
| _surface_key = surface_key |
|
|
|
|
| |
| |
| |
|
|
| def _ensure_runtime_state(): |
| if not isinstance(STATE, dict): |
| raise RuntimeError("STATE must be a dictionary.") |
| STATE.setdefault("warnings", []) |
| STATE.setdefault("start", None) |
| return STATE |
|
|
|
|
| _ensure_runtime_state() |
|
|
|
|
| |
| |
| |
|
|
| |
| |
| |
|
|
| if not hasattr(Validator, "_nll_state_safe_base_validate_v3"): |
| Validator._nll_state_safe_base_validate_v3 = Validator.validate |
| else: |
| Validator.validate = Validator._nll_state_safe_base_validate_v3 |
|
|
| if not hasattr(Ranker, "_nll_state_safe_base_rank_v3"): |
| Ranker._nll_state_safe_base_rank_v3 = Ranker.rank |
| else: |
| Ranker.rank = Ranker._nll_state_safe_base_rank_v3 |
|
|
| if not hasattr(Engine, "_nll_state_safe_base_one_v3"): |
| Engine._nll_state_safe_base_one_v3 = Engine.one |
| else: |
| Engine.one = Engine._nll_state_safe_base_one_v3 |
|
|
|
|
| def _validate_with_runtime_state_v3(self, *args, **kwargs): |
| _ensure_runtime_state() |
| return Validator._nll_state_safe_base_validate_v3(self, *args, **kwargs) |
|
|
|
|
| def _rank_with_runtime_state_v3(self, *args, **kwargs): |
| _ensure_runtime_state() |
| return Ranker._nll_state_safe_base_rank_v3(self, *args, **kwargs) |
|
|
|
|
| def _one_with_runtime_state_v3(self, *args, **kwargs): |
| _ensure_runtime_state() |
| return Engine._nll_state_safe_base_one_v3(self, *args, **kwargs) |
|
|
|
|
| Validator.validate = _validate_with_runtime_state_v3 |
| Ranker.rank = _rank_with_runtime_state_v3 |
| Engine.one = _one_with_runtime_state_v3 |
|
|
| |
| ENGINE = Engine() |
|
|
|
|
| |
| |
| |
|
|
| STATE.clear() |
| _ensure_runtime_state() |
|
|
| if "warnings" not in STATE or not isinstance(STATE["warnings"], list): |
| raise RuntimeError("Cell 7A v3 failed: warnings state was not restored.") |
|
|
| if _surface_key("A test.") != surface_key("A test."): |
| raise RuntimeError("Cell 7A v3 failed: legacy surface-key alias mismatch.") |
|
|
| if getattr(Validator.validate, "__name__", "") != "_validate_with_runtime_state_v3": |
| raise RuntimeError("Cell 7A v3 failed: Validator wrapper not installed.") |
|
|
| if getattr(Ranker.rank, "__name__", "") != "_rank_with_runtime_state_v3": |
| raise RuntimeError("Cell 7A v3 failed: Ranker wrapper not installed.") |
|
|
| if getattr(Engine.one, "__name__", "") != "_one_with_runtime_state_v3": |
| raise RuntimeError("Cell 7A v3 failed: Engine.one wrapper not installed.") |
|
|
| |
| |
| |
| _test_profile = next(iter(PROFILES.values())) if "PROFILES" in globals() and PROFILES else None |
| if _test_profile is not None: |
| _test_source = "Research claims are contingent on execution." |
| _test_analysis = Analyzer().analyze(_test_source, _test_profile, set()) |
| _test_candidate = Candidate( |
| "Research claims depend on execution.", |
| "Minimal", |
| "CONTINGENCY_RELATION", |
| "runtime-state self-test", |
| .9, |
| ) |
| try: |
| Validator().validate( |
| _test_source, |
| _test_candidate, |
| _test_analysis, |
| _test_profile, |
| set(), |
| "Minimal", |
| ) |
| except KeyError as exc: |
| raise RuntimeError( |
| "Cell 7A v3 failed direct Validator state test: " + repr(exc) |
| ) |
|
|
| |
| _live_one_func = getattr(getattr(ENGINE, "one", None), "__func__", None) |
| if _live_one_func is not Engine.one: |
| raise RuntimeError( |
| "Cell 7A v3 failed: live ENGINE.one is not bound to current Engine.one." |
| ) |
|
|
| |
| |
| STATE.clear() |
| _ensure_runtime_state() |
| if _test_profile is not None: |
| _u_source = "Models predicting regional demand inform the allocation process." |
| _u_candidate = Candidate( |
| "Models that predict regional demand inform the allocation process.", |
| "Balanced", |
| "universal_participial_modifier", |
| "runtime-state universal self-test", |
| .87, |
| ) |
| _u_analysis = Analyzer().analyze(_u_source, _test_profile, set()) |
| try: |
| Validator().validate( |
| _u_source, _u_candidate, _u_analysis, _test_profile, set(), "Balanced" |
| ) |
| except KeyError as exc: |
| raise RuntimeError( |
| "Cell 7A v3 failed universal Validator state test: " + repr(exc) |
| ) |
|
|
| RUNTIME_COMPATIBILITY_7A = { |
| "version": "3", |
| "aligned_cell7": "v5", |
| "rule_count": len(RULES), |
| "active": True, |
| "engine_wrapper": getattr(Engine.one, "__name__", ""), |
| "live_engine_binding": True, |
| } |
|
|
| print("=" * 78) |
| print("CELL 7A RUNTIME-STATE COMPATIBILITY v3 LOADED") |
| print("=" * 78) |
| print("Aligned Cell 7 version: v5") |
| print("STATE['warnings'] initialization: active") |
| print("Legacy _surface_key alias: active") |
| print("Validator direct-call safety: active") |
| print("Ranker direct-call safety: active") |
| print("Engine.one direct-call safety: active") |
| print("Rerun-safe wrapper restoration: active") |
| print("Direct Validator state self-test: PASSED") |
| print("Unseen-vocabulary Validator state self-test: PASSED") |
| print("Live ENGINE binding check: PASSED") |
| print("Runtime marker: RUNTIME_COMPATIBILITY_7A active") |
| print("ENGINE recreated: active") |
| print("Registered transformation rules:", len(RULES)) |
| print("Expected rule count:", EXPECTED_7A_RULE_COUNT) |
| print("=" * 78) |
|
|
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
|
|
| from collections import Counter |
|
|
| EXPECTED_7B_RULE_COUNT = 42 |
|
|
| _REQUIRED_7B = [ |
| "STATE", "RULES", "Engine", "ENGINE", "Validator", "Ranker", |
| "Generator", "Analyzer", "PROFILES", "Candidate", |
| "surface_key", "realization_issues", "_ensure_runtime_state", |
| ] |
| _missing_7b = [name for name in _REQUIRED_7B if name not in globals()] |
| if _missing_7b: |
| raise RuntimeError( |
| "Cell 7B cannot run. Missing: " + ", ".join(_missing_7b) |
| + ". Run Cell 7 v5 and Cell 7A v3 first." |
| ) |
|
|
| if len(RULES) != EXPECTED_7B_RULE_COUNT: |
| raise RuntimeError( |
| f"Cell 7B expected {EXPECTED_7B_RULE_COUNT} rules but found {len(RULES)}." |
| ) |
|
|
| |
| if getattr(Engine.one, "__name__", "") != "_one_with_runtime_state_v3": |
| raise RuntimeError( |
| "Cell 7B requires Cell 7A v3 immediately before it. " |
| "Expected Engine.one wrapper '_one_with_runtime_state_v3'." |
| ) |
|
|
|
|
| |
| |
| |
|
|
| MUST_PARAPHRASE_ENABLED = True |
|
|
| _MODE_RETRY_ORDER = { |
| "Minimal": ["Minimal", "Balanced", "Structural"], |
| "Balanced": ["Balanced", "Structural", "Minimal"], |
| "Structural": ["Structural", "Balanced", "Minimal"], |
| } |
|
|
| _HARD_REJECTION_CODES = { |
| "TERMINOLOGY_REJECTION", |
| "POLARITY_REJECTION", |
| "MODALITY_REJECTION", |
| "QUANTITY_REJECTION", |
| "ENUMERATION_REJECTION", |
| "GRAMMAR_REJECTION", |
| "SEMANTIC_REJECTION", |
| } |
|
|
|
|
| def _mp_changed(original, revised): |
| return surface_key(original) != surface_key(revised) |
|
|
|
|
| def _mp_trace(result): |
| trace = getattr(result, "diagnostics", None) |
| if trace is None: |
| trace = getattr(result, "trace", None) |
| return dict(trace or {}) |
|
|
|
|
| def _mp_checks(result): |
| return dict(getattr(result, "checks", None) or {}) |
|
|
|
|
| def _mp_result_summary(result, attempted_mode): |
| trace = _mp_trace(result) |
| checks = _mp_checks(result) |
| revised = getattr(result, "revised", "") |
| original = getattr(result, "original", "") |
| selected = trace.get("selected") or {} |
| failures = list(checks.get("failures", []) or []) |
| realization = list(realization_issues(revised) or []) |
|
|
| return { |
| "attempted_mode": attempted_mode, |
| "status": getattr(result, "status", ""), |
| "changed": _mp_changed(original, revised), |
| "selected_family": getattr(result, "family", None) or selected.get("family"), |
| "selected_lane": selected.get("lane"), |
| "selected_score": selected.get("score"), |
| "failures": failures, |
| "hard_failures": [code for code in failures if code in _HARD_REJECTION_CODES], |
| "realization_issues": realization, |
| "generated_by_lane": trace.get("generated_by_lane", {}), |
| "generated_before_final_filter": trace.get("generated_before_final_filter"), |
| "generated_after_final_filter": trace.get("generated_after_final_filter"), |
| "realization_rejections": trace.get("realization_rejections", []), |
| "terminal_rejections": trace.get("rejections", {}), |
| "reason": getattr(result, "reason", ""), |
| "revised": revised, |
| } |
|
|
|
|
| def _mp_is_safe_success(result): |
| if getattr(result, "status", "") != "SUCCESS": |
| return False |
|
|
| original = getattr(result, "original", "") |
| revised = getattr(result, "revised", "") |
| if not _mp_changed(original, revised): |
| return False |
|
|
| checks = _mp_checks(result) |
| failures = set(checks.get("failures", []) or []) |
| if failures & _HARD_REJECTION_CODES: |
| return False |
|
|
| if realization_issues(revised): |
| return False |
|
|
| return True |
|
|
|
|
| def _mp_failure_reason(attempts): |
| if not attempts: |
| return ( |
| "No rewrite attempt was completed. The original sentence was preserved " |
| "because the deterministic judge had no result to assess." |
| ) |
|
|
| generated_total = 0 |
| realization_rejections = 0 |
| rejection_codes = Counter() |
| statuses = Counter() |
|
|
| for attempt in attempts: |
| statuses[attempt.get("status", "UNKNOWN")] += 1 |
|
|
| generated = attempt.get("generated_after_final_filter") |
| if generated is None: |
| generated = sum( |
| int(value or 0) |
| for value in (attempt.get("generated_by_lane", {}) or {}).values() |
| ) |
| generated_total += int(generated or 0) |
| realization_rejections += len(attempt.get("realization_rejections", []) or []) |
|
|
| for code in attempt.get("failures", []) or []: |
| rejection_codes[code] += 1 |
| for code, count in (attempt.get("terminal_rejections", {}) or {}).items(): |
| rejection_codes[code] += int(count or 0) |
|
|
| if generated_total == 0 and realization_rejections == 0: |
| return ( |
| "The original sentence was preserved because no registered transformation " |
| "produced a distinct candidate in Minimal, Balanced, or Structural mode." |
| ) |
|
|
| if realization_rejections and generated_total == 0: |
| return ( |
| "The original sentence was preserved because every generated alternative " |
| "failed the grammatical realization gate." |
| ) |
|
|
| if rejection_codes: |
| detail = ", ".join( |
| f"{code}: {count}" for code, count in sorted(rejection_codes.items()) |
| ) |
| return ( |
| "The original sentence was preserved because all distinct alternatives " |
| f"were rejected by the deterministic judge ({detail})." |
| ) |
|
|
| status_detail = ", ".join( |
| f"{status}: {count}" for status, count in sorted(statuses.items()) |
| ) |
| return ( |
| "The original sentence was preserved because no attempted mode produced a " |
| "safe validated rewrite. Attempt outcomes: " + status_detail + "." |
| ) |
|
|
|
|
| |
| |
| |
|
|
| |
| |
| if not hasattr(Engine, "_nll_must_paraphrase_base_one_v3"): |
| Engine._nll_must_paraphrase_base_one_v3 = Engine.one |
| else: |
| Engine.one = Engine._nll_must_paraphrase_base_one_v3 |
|
|
|
|
| def _one_with_must_paraphrase_v3(self, sentence, profile, user_terms, mode): |
| _ensure_runtime_state() |
|
|
| requested_mode = mode if mode in _MODE_RETRY_ORDER else "Balanced" |
|
|
| if not MUST_PARAPHRASE_ENABLED: |
| return Engine._nll_must_paraphrase_base_one_v3( |
| self, sentence, profile, user_terms, requested_mode |
| ) |
|
|
| attempts = [] |
| attempted_results = [] |
|
|
| for attempted_mode in _MODE_RETRY_ORDER[requested_mode]: |
| result = Engine._nll_must_paraphrase_base_one_v3( |
| self, sentence, profile, user_terms, attempted_mode |
| ) |
| attempt = _mp_result_summary(result, attempted_mode) |
| attempts.append(attempt) |
| attempted_results.append(result) |
|
|
| if _mp_is_safe_success(result): |
| trace = _mp_trace(result) |
| trace["must_paraphrase"] = { |
| "enabled": True, |
| "requested_mode": requested_mode, |
| "accepted_mode": attempted_mode, |
| "attempt_count": len(attempts), |
| "attempts": attempts, |
| "outcome": "SAFE_CHANGED_OUTPUT", |
| "hard_safeguards_relaxed": False, |
| } |
|
|
| try: |
| result.diagnostics = trace |
| except Exception: |
| pass |
| try: |
| result.trace = trace |
| except Exception: |
| pass |
|
|
| original_reason = str(getattr(result, "reason", "") or "").strip() |
| suffix = ( |
| f" Must-paraphrase judge accepted the {attempted_mode} attempt " |
| f"after {len(attempts)} bounded attempt(s)." |
| ) |
| try: |
| result.reason = (original_reason + suffix).strip() |
| except Exception: |
| pass |
| return result |
|
|
| |
| fallback = attempted_results[0] |
| reason = _mp_failure_reason(attempts) |
| trace = _mp_trace(fallback) |
| trace["must_paraphrase"] = { |
| "enabled": True, |
| "requested_mode": requested_mode, |
| "accepted_mode": None, |
| "attempt_count": len(attempts), |
| "attempts": attempts, |
| "outcome": "ORIGINAL_PRESERVED_AFTER_EXHAUSTIVE_SAFE_SEARCH", |
| "preservation_reason": reason, |
| "hard_safeguards_relaxed": False, |
| } |
|
|
| try: |
| fallback.diagnostics = trace |
| except Exception: |
| pass |
| try: |
| fallback.trace = trace |
| except Exception: |
| pass |
| try: |
| fallback.reason = reason |
| except Exception: |
| pass |
|
|
| return fallback |
|
|
|
|
| Engine.one = _one_with_must_paraphrase_v3 |
| ENGINE = Engine() |
|
|
|
|
| |
| |
| |
|
|
| if getattr(Engine.one, "__name__", "") != "_one_with_must_paraphrase_v3": |
| raise RuntimeError("Cell 7B v3 failed: Engine.one wrapper not installed.") |
|
|
| if not hasattr(Engine, "_nll_must_paraphrase_base_one_v3"): |
| raise RuntimeError("Cell 7B v3 failed: base Engine.one was not preserved.") |
|
|
| for preferred, order in _MODE_RETRY_ORDER.items(): |
| if len(order) != 3 or order[0] != preferred or len(set(order)) != 3: |
| raise RuntimeError( |
| f"Cell 7B v3 failed retry order for {preferred}: {order}" |
| ) |
|
|
| |
| _test_profile, _ = ENGINE.route( |
| "The next step is to inspect how the handler checks configuration.", |
| "Auto Detect", |
| ) |
| _test_result = ENGINE.one( |
| "The next step is to inspect how the handler checks configuration.", |
| _test_profile, |
| set(), |
| "Balanced", |
| ) |
| _test_trace = _mp_trace(_test_result) |
| _test_mp = _test_trace.get("must_paraphrase", {}) or {} |
|
|
| if getattr(_test_result, "status", "") != "SUCCESS": |
| raise RuntimeError( |
| "Cell 7B v3 smoke test failed: known transformable sentence did not succeed." |
| ) |
| if not _mp_changed(_test_result.original, _test_result.revised): |
| raise RuntimeError( |
| "Cell 7B v3 smoke test failed: successful result was unchanged." |
| ) |
| if _test_mp.get("outcome") != "SAFE_CHANGED_OUTPUT": |
| raise RuntimeError( |
| "Cell 7B v3 smoke test failed: orchestration trace missing safe-change outcome." |
| ) |
| if _test_mp.get("requested_mode") != "Balanced": |
| raise RuntimeError( |
| "Cell 7B v3 smoke test failed: requested mode was not preserved." |
| ) |
|
|
| |
| |
| |
| _u_sentence = "Models predicting regional demand inform the allocation process." |
| _u_profile, _ = ENGINE.route(_u_sentence, "Auto Detect") |
| _u_result = ENGINE.one(_u_sentence, _u_profile, set(), "Balanced") |
| _u_trace = _mp_trace(_u_result) |
| _u_mp = _u_trace.get("must_paraphrase", {}) or {} |
| if int(_u_mp.get("attempt_count", 0) or 0) < 1: |
| raise RuntimeError( |
| "Cell 7B v3 universal smoke test failed: bounded-attempt trace is missing." |
| ) |
| if _u_mp.get("outcome") not in { |
| "SAFE_CHANGED_OUTPUT", |
| "ORIGINAL_PRESERVED_AFTER_EXHAUSTIVE_SAFE_SEARCH", |
| }: |
| raise RuntimeError( |
| "Cell 7B v3 universal smoke test failed: orchestration outcome is missing." |
| ) |
| if getattr(_u_result, "status", "") == "SUCCESS" and not _mp_changed(_u_result.original, _u_result.revised): |
| raise RuntimeError( |
| "Cell 7B v3 universal smoke test failed: SUCCESS result was unchanged." |
| ) |
|
|
| |
| |
| _live_func = getattr(getattr(ENGINE, "one", None), "__func__", None) |
| if _live_func is not Engine.one: |
| raise RuntimeError( |
| "Cell 7B v3 failed: live ENGINE.one is not bound to current Engine.one." |
| ) |
| if getattr(ENGINE.one, "__name__", "") != "_one_with_must_paraphrase_v3": |
| raise RuntimeError( |
| "Cell 7B v3 failed: live ENGINE instance is not using must-paraphrase." |
| ) |
|
|
| |
| |
| |
| _doc_probe = ENGINE.process( |
| "The next step is to inspect how the handler checks configuration.", |
| "Edit / Paraphrase Only", |
| "Balanced", |
| "Auto Detect", |
| "", |
| ) |
| if not getattr(_doc_probe, "sentences", None): |
| raise RuntimeError( |
| "Cell 7B v3 failed: document-level probe returned no sentence results." |
| ) |
| _doc_sentence = _doc_probe.sentences[0] |
| _doc_trace = _mp_trace(_doc_sentence) |
| _doc_mp = _doc_trace.get("must_paraphrase", {}) or {} |
| if int(_doc_mp.get("attempt_count", 0) or 0) < 1: |
| raise RuntimeError( |
| "Cell 7B v3 failed: must-paraphrase attempt trace did not survive Engine.process." |
| ) |
| if _doc_mp.get("outcome") not in { |
| "SAFE_CHANGED_OUTPUT", |
| "ORIGINAL_PRESERVED_AFTER_EXHAUSTIVE_SAFE_SEARCH", |
| }: |
| raise RuntimeError( |
| "Cell 7B v3 failed: document-level orchestration outcome is missing." |
| ) |
|
|
| |
| |
| MUST_PARAPHRASE_RUNTIME = { |
| "version": "3", |
| "active": True, |
| "engine_class_wrapper": getattr(Engine.one, "__name__", ""), |
| "engine_instance_wrapper": getattr(ENGINE.one, "__name__", ""), |
| "document_trace_verified": True, |
| "rule_count": len(RULES), |
| "aligned_cell7": "v5", |
| "aligned_cell7a": "v3", |
| "universal_trace_verified": True, |
| } |
|
|
| print("=" * 78) |
| print("CELL 7B DETERMINISTIC MUST-PARAPHRASE ORCHESTRATOR v3 LOADED") |
| print("=" * 78) |
| print("Aligned Cell 7 version: v5") |
| print("Aligned Cell 7A version: v3") |
| print("Requested-mode-first search: active") |
| print("Minimal/Balanced/Structural retry: active") |
| print("Validator + Ranker deterministic judge: active") |
| print("Hard safeguard relaxation: disabled") |
| print("Realization safeguard relaxation: disabled") |
| print("Unchanged-output preservation reason: active") |
| print("Bounded retry loop: active") |
| print("Sentence-level orchestration smoke test: PASSED") |
| print("Unseen-vocabulary orchestration smoke test: PASSED") |
| print("Live ENGINE binding check: PASSED") |
| print("Document-level trace survival: PASSED") |
| print("Runtime marker: MUST_PARAPHRASE_RUNTIME active") |
| print("LLM generation: not used") |
| print("ENGINE recreated: active") |
| print("Registered transformation rules:", len(RULES)) |
| print("Expected rule count:", EXPECTED_7B_RULE_COUNT) |
| print("=" * 78) |
