"""NLL V7.2 serialized engine source.""" # ===== SOURCE: runtime/cell3.py ===== #@title 3. Contracts, resources, disciplines, and document protection from __future__ import annotations import os,re,json,html,time,traceback,statistics from pathlib import Path import uuid from dataclasses import dataclass,field,asdict from collections import Counter from difflib import SequenceMatcher import numpy as np,pandas as pd,spacy from nltk.corpus import wordnet as wn from lemminflect import getInflection DISCLAIMER="Observable style-pattern index only; not proof/probability of AI authorship and no detector-bypass guarantee." CODES={"NO_PATTERN_COVERAGE","PARSE_FAILURE","CANDIDATE_GENERATION_FAILURE","REALIZATION_FAILURE","SEMANTIC_REJECTION","TERMINOLOGY_REJECTION","POLARITY_REJECTION","MODALITY_REJECTION","QUANTITY_REJECTION","ENUMERATION_REJECTION","GRAMMAR_REJECTION","RANKING_EMPTY","INSUFFICIENT_REWRITE","SUCCESS","NOT_TARGETED"} @dataclass class Config: stanza:bool=False;sts:bool=True;nli:bool=True;mlm:bool=False;lt:bool=False;online:bool=False;batch:int=20;near_copy:float=.965;min_sts:float=.72;min_nli:float=.58;out:str="/content/nll_outputs";cache:str="/content/nll_cache" @dataclass class Profile:name:str;keywords:set[str];phrases:set[str];immutable:set[str]=field(default_factory=set);concepts:set[str]=field(default_factory=set) @dataclass class Prop:predicate:str;subject:str="";objects:list=field(default_factory=list);complements:list=field(default_factory=list);polarity:str="positive";modals:list=field(default_factory=list);voice:str="active" @dataclass class Analysis:text:str;families:list;props:list;relations:list;enumerations:list;protected:list;notes:list=field(default_factory=list);constituency:str|None=None @dataclass class Finding:category:str;passage:str;start:int;end:int;sentence:int;severity:float;explanation:str @dataclass class Candidate:text:str;lane:str;family:str;reason:str;priority:float=.5;checks:dict=field(default_factory=dict);score:float=-999 @dataclass class SentenceResult:original:str;revised:str;status:str;families:list;family:str|None;reason:str;alternatives:list;validation:dict;diagnostics:dict @dataclass class DocumentResult:original:str;revised:str;workflow:str;mode:str;discipline:str;routing:list;pattern_score:float;band:str;findings:list;sentences:list;warnings:list;runtime:dict CFG=Config();CFG.out='/tmp/nll_outputs';CFG.cache='/tmp/nll_cache';MODELS={};STATE={};Path(CFG.out).mkdir(parents=True,exist_ok=True);Path(CFG.cache).mkdir(parents=True,exist_ok=True) def S(x):return {z.strip() for z in x.split("|") if z.strip()} PROFILES={ "General Academic":Profile("General Academic",S("study|research|analysis|result|method|evidence|sample|finding"),S("this study|the findings")), "Computing / Cybersecurity":Profile("Computing / Cybersecurity",S("algorithm|classifier|malware|adversarial|dataset|network|feature|robustness|encryption|cloud"),S("machine learning|feature space|problem space|threat model|static analysis"),S("XGBoost|Linear SVM|DNN|Windows PE|Android|APK|API"),S("adversarial robustness|perturbation constraint")), "Engineering":Profile("Engineering",S("design|system|load|stress|material|control|circuit|failure|efficiency"),S("finite element|control system"),S("CAD|FEA|HVAC")), "Medicine / Health":Profile("Medicine / Health",S("patient|clinical|treatment|diagnosis|disease|therapy|trial|cohort|outcome"),S("randomized controlled trial|adverse event"),S("DNA|RNA|MRI|CT|ICD")), "Biology":Profile("Biology",S("cell|gene|protein|organism|species|genome|enzyme|tissue|mutation"),S("gene expression|cell signaling"),S("DNA|RNA|PCR")), "Chemistry":Profile("Chemistry",S("compound|reaction|molecule|catalyst|solvent|synthesis|concentration|ion"),S("reaction rate|chemical equilibrium"),S("pH|NMR|HPLC")), "Physics":Profile("Physics",S("energy|force|particle|field|wave|quantum|velocity|mass|gravity"),S("boundary condition|quantum state")), "Mathematics / Statistics":Profile("Mathematics / Statistics",S("theorem|proof|equation|distribution|variance|regression|probability|estimator|matrix"),S("null hypothesis|confidence interval"),S("ANOVA|OLS|MSE")), "Social Sciences":Profile("Social Sciences",S("society|community|policy|culture|institution|inequality|survey|interview|qualitative"),S("social context|thematic analysis")), "Psychology":Profile("Psychology",S("behavior|cognition|emotion|participant|memory|attention|motivation|construct"),S("cognitive load|self efficacy"),S("DSM")), "Education":Profile("Education",S("student|teacher|learning|curriculum|instruction|classroom|assessment|school"),S("learning outcome|instructional strategy")), "Business / Economics":Profile("Business / Economics",S("market|firm|revenue|cost|consumer|economy|finance|investment|management"),S("return on investment|market share"),S("ROI|GDP")), "Law / Policy":Profile("Law / Policy",S("law|legal|court|statute|regulation|policy|rights|liability|compliance"),S("case law|due process"),S("FAR|CFR")), "Humanities":Profile("Humanities",S("text|narrative|history|philosophy|literature|discourse|interpretation|rhetoric"),S("historical context|critical theory")), "Environmental Science":Profile("Environmental Science",S("climate|ecosystem|pollution|emission|biodiversity|habitat|sustainability|carbon"),S("climate change|greenhouse gas"),S("CO2|LCA"))} def nlp(): if "spacy" not in MODELS: try:MODELS["spacy"]=spacy.load("en_core_web_sm") except Exception:MODELS["spacy"]=spacy.blank("en");MODELS["spacy"].add_pipe("sentencizer") return MODELS["spacy"] def optional(kind): if kind=="sts" and CFG.sts: try: if kind not in MODELS: from sentence_transformers import SentenceTransformer MODELS[kind]=SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2") except Exception as e:STATE["warnings"].append(f"STS unavailable: {e}");MODELS[kind]=None if kind=="nli" and CFG.nli: try: if kind not in MODELS: from transformers import AutoTokenizer,AutoModelForSequenceClassification mid="cross-encoder/nli-deberta-v3-small";m=AutoModelForSequenceClassification.from_pretrained(mid);m.eval();MODELS[kind]=(AutoTokenizer.from_pretrained(mid),m) except Exception as e:STATE["warnings"].append(f"NLI unavailable: {e}");MODELS[kind]=None if kind=="mlm" and CFG.mlm: try: if kind not in MODELS: import torch from transformers import pipeline MODELS[kind]=pipeline("fill-mask",model="distilroberta-base",device=0 if torch.cuda.is_available() else -1) except Exception as e:STATE["warnings"].append(f"MLM unavailable: {e}");MODELS[kind]=None if kind=="lt" and CFG.lt: try: if kind not in MODELS: import language_tool_python MODELS[kind]=language_tool_python.LanguageTool("en-US") except Exception as e:STATE["warnings"].append(f"LanguageTool unavailable: {e}");MODELS[kind]=None return MODELS.get(kind) PATS={ "citation":re.compile(r"\([^()\n]{0,90}(?:19|20)\d{2}[a-z]?[^()\n]{0,50}\)|\[[0-9,;\-\s]+\]|[A-Z][A-Za-z'\-]+ et al\. \((?:19|20)\d{2}[a-z]?\)"), "url":re.compile(r"https?://\S+|doi:\s*\S+|10\.\d{4,9}/\S+",re.I), "equation":re.compile(r"\$[^$]+\$"), "code":re.compile(r"`[^`]+`|```[\s\S]+?```"), "quantity":re.compile(r"(?≈~±]?\s*\d+(?:\.\d+)?(?:\s*[-–]\s*\d+(?:\.\d+)?)?\s*(?:%|mg|kg|mL|mm|cm|km|Hz|GHz|°C|MPa|kW|days?|years?)?\b",re.I) } def protect(text,terms): out=[] for k,p in PATS.items():out += [(k,m.group(),m.start(),m.end()) for m in p.finditer(text)] for t in terms:out += [("term",m.group(),m.start(),m.end()) for m in re.finditer(r"(?=3 and t.i not in seen:enums.append([sub(x) for x in xs]);seen|={x.i for x in xs} const=None if CFG.stanza: try: if "stanza" not in MODELS: import stanza MODELS["stanza"]=stanza.Pipeline("en",processors="tokenize,pos,lemma,depparse,constituency",use_gpu=False,verbose=False) const=str(MODELS["stanza"](s).sentences[0].constituency) except Exception as e:STATE["warnings"].append(f"Stanza unavailable: {e}") return Analysis(s,sorted(fam),props,rels,enums,protect(s,terms),[],const) FORMULAIC=[r"it is important to note that",r"in today's rapidly evolving",r"delve(?:s|d)? into",r"multifaceted",r"pivotal role",r"seamlessly",r"in conclusion",r"furthermore",r"moreover",r"this highlights"] class Scanner: def scan(self,text): ss=sent_split(text);fs=[];off=0;lens=[];starts=[] for i,s in enumerate(ss): st=max(off,text.find(s,off));off=st+len(s);d=nlp()(s);w=[t for t in d if t.is_alpha];lens.append(len(w));starts.append(" ".join(t.lemma_.lower() for t in w[:3])) for p in FORMULAIC: for z in re.finditer(p,s,re.I):fs.append(Finding("formulaic wording",z.group(),st+z.start(),st+z.end(),i,.65,"Formulaic academic wording; context may still justify it.")) if len(w)>38:fs.append(Finding("sentence complexity",s,st,st+len(s),i,.6,"Long multi-clause sentence may be dense.")) rep={x for x,n in Counter(starts).items() if x and n>=3} for i,(s,o) in enumerate(zip(ss,starts)): if o in rep:fs.append(Finding("repetitive opening",s[:80],text.find(s),text.find(s)+min(80,len(s)),i,.55,"Repeated sentence opening.")) mean=statistics.mean(lens) if lens else 0;cv=statistics.pstdev(lens)/mean if len(lens)>1 and mean else 0;score=min(100,len(fs)*7+max(0,.28-cv)*40);return fs,float(score),"Low" if score<30 else "Moderate" if score<60 else "High",{"sentence_length_cv":cv,"findings":len(fs)} # ===== SOURCE: runtime/cell4b.py ===== #@title 4B. Universal academic pattern taxonomy and naturalness analyzer from dataclasses import dataclass, field from collections import Counter import re # ============================================================ # 1. TAXONOMY # ============================================================ PATTERN_TAXONOMY = { "VOICE": { "PASSIVE", "AGENTLESS_PASSIVE", "AGENTIVE_PASSIVE", "MODAL_PASSIVE", "STACKED_PASSIVE", "PASSIVE_EVALUATION_CONTEXT", "PASSIVE_PROCESS_CONTEXT", "NEGATED_CLASSIFICATION_PASSIVE", "COORDINATED_PASSIVE_PREDICATES", "PASSIVE_BASIS_FRAME", }, "CLAUSE_STRUCTURE": { "WHEREBY_CLAUSE", "SO_THAT_CLAUSE", "PURPOSE_RESULT_CLAUSE", "FINITE_WHILE_CLAUSE", "WHILE_PARTICIPIAL", "PARTICIPIAL_RESULT_CLAUSE", "WHEN_CLAUSE", "WHEREAS_CLAUSE", "ALTHOUGH_CLAUSE", "BECAUSE_CLAUSE", "IF_CLAUSE", "CONTENT_COMPLEMENT", "RELATIVE_CLAUSE", "MULTI_EMBEDDED_CLAUSES", "CONTRASTIVE_COORDINATED_CLAUSES", "SEMICOLON_NEGATED_ASSUMPTION", }, "NOMINALIZATION": { "GERUND_SUBJECT", "NOMINALIZATION", "ABSTRACT_NOUN_STACK", "NOUN_HEAVY_PREDICATE", "NOMINALIZED_PROCESS", "NOMINALIZED_EVALUATION", }, "CONTRAST": { "CONTRAST", "NEGATED_PURPOSE_CONTRAST", "NOT_X_BUT_Y", "RATHER_THAN_CONTRAST", "HOWEVER_CONTRAST", "WHEREAS_CONTRAST", "CONTRASTIVE_COORDINATED_CLAUSES", "SEMICOLON_NEGATED_ASSUMPTION", }, "REPORTING_EVIDENCE": { "REPORTING_FRAME", "EVIDENTIAL_COMPLEMENT", "ATTRIBUTION_FRAME", "RESULT_REPORTING_FRAME", "INDEPENDENT_REPORTING", }, "EVALUATION_METHOD": { "EVALUATION_FRAME", "PASSIVE_EVALUATION_CONTEXT", "PASSIVE_PROCESS_CONTEXT", "PASSIVE_BASIS_FRAME", "METHOD_RELATION", "COMPARATIVE_FRAME", "METHOD_APPLICATION_FRAME", "EVALUATION_CONDITION_FRAME", }, "COORDINATION": { "COORDINATED_PREDICATES", "COORDINATED_OBJECTS_OR_ENUMERATION", "COORDINATED_PROPOSITIONS", "LONG_ENUMERATION", "PARALLEL_ENUMERATION", "COORDINATED_PASSIVE_PREDICATES", "CONTRASTIVE_COORDINATED_CLAUSES", }, "DISCOURSE": { "DISCOURSE_MARKER", "CONSEQUENCE", "RESULT_CLAUSE", "REPETITIVE_TRANSITION", "FORMULAIC_TRANSITION", "OVEREXPLICIT_LOGIC", "SEMICOLON_NEGATED_ASSUMPTION", }, "COMPLEXITY": { "LONG_SENTENCE", "VERY_LONG_SENTENCE", "MULTI_EMBEDDED_CLAUSES", "CLAUSE_OVERLOADING", "LONG_PREPOSITIONAL_CHAIN", "HEAVY_POSTMODIFICATION", "EXCESSIVE_COORDINATION", }, "FORMULAIC_ACADEMIC": { "FORMULAIC_PURPOSE", "FORMULAIC_RESULT", "FORMULAIC_COMPARISON", "FORMULAIC_EVALUATION", "FORMULAIC_METHOD", "FORMULAIC_CONCLUSION", "FORMULAIC_TRANSITION", "OVEREXPLICIT_LOGIC", "REPETITIVE_SENTENCE_FRAME", }, "NATURALNESS": { "AWKWARD_PASSIVE", "UNNATURAL_NOMINALIZATION", "EXCESSIVE_SYMMETRY", "CLAUSE_OVERLOADING", "REPETITIVE_SYNTAX", "REDUNDANT_EXPLICITNESS", "STILTED_CONNECTIVE", "HEAVY_NOUN_PHRASE", "ABSTRACT_NOUN_STACK", "MECHANICAL_PARALLELISM", }, } # ============================================================ # 2. RESULT OBJECT # ============================================================ @dataclass class PatternProfile: grammar_families: set = field(default_factory=set) formulaic_flags: set = field(default_factory=set) naturalness_flags: set = field(default_factory=set) complexity_flags: set = field(default_factory=set) discourse_flags: set = field(default_factory=set) metrics: dict = field(default_factory=dict) evidence: dict = field(default_factory=dict) # ============================================================ # 3. HELPERS # ============================================================ def _pt_clean(text): return re.sub(r"\s+", " ", str(text or "")).strip() def _pt_words(text): return re.findall(r"\b[\w'-]+\b", str(text or "")) def _add_evidence(profile, flag, evidence): profile.evidence.setdefault(flag, []) if evidence not in profile.evidence[flag]: profile.evidence[flag].append(evidence) def _detect_content_complement(lower_text): if re.search(r"\bwhether\b", lower_text): return True determiner = re.search( r"\bthat\s+(?:type|kind|form|method|model|approach|system|procedure|" r"framework|process|design|strategy|technique|case|example|result|" r"study|analysis|experiment|feature|representation|classifier|" r"algorithm|dataset|environment|platform|component|domain|alignment|" r"space|constraint|condition|measure|attack|sample|architecture|" r"methodology)\b", lower_text, ) if determiner: return False governor = re.search( r"\b(?:show|shows|showed|indicate|indicates|indicated|suggest|" r"suggests|suggested|demonstrate|demonstrates|demonstrated|report|" r"reports|reported|find|finds|found|observe|observes|observed|confirm|" r"confirms|confirmed|conclude|concludes|concluded|establish|" r"establishes|established|assume|assumes|assumed|ensure|ensures|" r"ensured|mean|means|meant|reveal|reveals|revealed|evidence|finding|" r"findings|result|results)\s+that\b", lower_text, ) if governor: return True clause = re.search( r"\bthat\s+(?:the|a|an|this|these|those|it|they|we|researchers|" r"participants|results|findings|evidence|data|models?|systems?|" r"methods?|samples?|features?)\s+(?:is|are|was|were|has|have|had|" r"can|could|may|might|must|should|will|would|does|do|did|" r"[A-Za-z][A-Za-z-]*(?:s|ed))\b", lower_text, ) return bool(clause) # ============================================================ # 4. GRAMMAR-FAMILY DETECTOR # ============================================================ def detect_grammar_families(text): text = _pt_clean(text) lower_text = text.lower() families = set() if re.search(r"\bwhereby\b", lower_text): families.add("WHEREBY_CLAUSE") if re.search(r"\bso\s+that\b", lower_text): families.update({"SO_THAT_CLAUSE", "PURPOSE_RESULT_CLAUSE"}) if re.search( r"\bwhile\s+(?:the\s+)?[A-Za-z][A-Za-z0-9_/-]*" r"(?:\s+[A-Za-z][A-Za-z0-9_/-]*){0,5}\s+" r"(?:is|are|was|were|has|have|had|does|do|did|tracks?|measures?|" r"shows?|indicates?|preserves?|provides?|uses?|evaluates?|compares?|" r"examines?|remains?|increases?|decreases?|continues?|represents?|" r"performs?|captures?|retains?|allows?|enables?)\b", lower_text, ): families.add("FINITE_WHILE_CLAUSE") if re.search(r"\bwhile\s+[A-Za-z][A-Za-z-]*ing\b", lower_text): families.add("WHILE_PARTICIPIAL") if re.search(r",\s*(?:allowing|enabling|permitting)\b", lower_text): families.add("PARTICIPIAL_RESULT_CLAUSE") for family, pattern in { "WHEN_CLAUSE": r"\bwhen\b", "WHEREAS_CLAUSE": r"\bwhereas\b", "ALTHOUGH_CLAUSE": r"\balthough\b", "BECAUSE_CLAUSE": r"\bbecause\b", "IF_CLAUSE": r"\bif\b", }.items(): if re.search(pattern, lower_text): families.add(family) if _detect_content_complement(lower_text): families.add("CONTENT_COMPLEMENT") if re.search(r"\bnot\b.+?\bbut\b", lower_text, re.S): families.add("CONTRAST") if re.search( r"\b(?:purpose|aim|objective|goal)\b.+?\bis\s+not\s+to\b.+?\bbut\s+to\b", lower_text, re.S, ): families.add("NEGATED_PURPOSE_CONTRAST") if re.match(r"^[A-Za-z][A-Za-z-]*ing\b", text, re.I): families.add("GERUND_SUBJECT") if re.search( r"\b(?:is|are|was|were|has been|have been|had been)\s+(?:not\s+)?" r"(?:\w+\s+){0,2}\w+(?:ed|en)\b", lower_text, ): families.add("PASSIVE") if re.search(r"\b(?:can|could|may|might|must|should)\s+be\s+\w+(?:ed|en)\b", lower_text): families.add("MODAL_PASSIVE") if re.search( r"\b(?:is|are|was|were|has been|have been)\s+evaluated\s+" r"(?:at|across|under|within|on|by|with|using)\b", lower_text, ): families.add("PASSIVE_EVALUATION_CONTEXT") if re.search( r"\b(?:is|are|was|were|has been|have been|had been)\s+" r"[A-Za-z][A-Za-z-]*(?:ed|en)(?:\s+(?:independently|separately|" r"jointly|simultaneously|iteratively|individually|concurrently))?\s+" r"(?:on|in|within|under|across|using|with|from|for)\b", lower_text, ): families.add("PASSIVE_PROCESS_CONTEXT") if re.search( r"\b(?:is|are|was|were|has been|have been|had been)\s+not\s+" r"(?:treated|regarded|interpreted|classified|considered)\s+as\b", lower_text, ): families.add("NEGATED_CLASSIFICATION_PASSIVE") # NEW: coordinated passive predicates. if re.search( r"\b(?:is|are|was|were|has been|have been|had been)\s+" r"[A-Za-z][A-Za-z-]*(?:ed|en)\s+(?:and|or)\s+" r"[A-Za-z][A-Za-z-]*(?:ed|en)" r"(?:\s+(?:independently|separately|jointly|simultaneously|individually))?\b", lower_text, ): families.add("COORDINATED_PASSIVE_PREDICATES") # NEW: contrastive coordinated clauses. if re.search(r",\s*(?:but|whereas|while)\s+.+", lower_text, re.S): families.update({"CONTRASTIVE_COORDINATED_CLAUSES", "CONTRAST"}) # NEW: passive basis frame. if re.search( r"\b(?:is|are|was|were|has been|have been)\s+based\s+on\s+", lower_text, ): families.update({"PASSIVE_BASIS_FRAME", "PASSIVE"}) # NEW: semicolon followed by a negated assumption/implication clause. if re.search( r";\s*(?:it|this|that|the\s+[A-Za-z][A-Za-z-]*)\s+" r"(?:does|do|did)\s+not\s+" r"(?:assume|imply|indicate|mean|require|establish|demonstrate|suggest)\s+that\b", lower_text, ): families.update({"SEMICOLON_NEGATED_ASSUMPTION", "CONTRAST"}) if re.search( r"\b(?:is|are|was|were|has been|have been)\s+" r"(?:commonly|frequently|generally|typically|often|usually)\s+" r"evaluated\s+(?:using|with)\b", lower_text, ): families.add("EVALUATION_FRAME") if re.search( r"\b(?:results|findings|outcomes|observations)\s+reported\s+" r"(?:independently|separately)\b", lower_text, ): families.add("REPORTING_FRAME") if re.search(r"\bthrough\b", lower_text): families.add("METHOD_RELATION") if re.search(r"\b(?:intended|designed)\s+to\b", lower_text): families.add("PURPOSE_INFINITIVE") if re.search( r"\bbasis\s+for\s+(?:determining|evaluating|assessing|comparing)\b", lower_text, ): families.add("PURPOSE_RESULT_CLAUSE") if re.match( r"(?i)^(?:however|therefore|consequently|thus|accordingly|" r"nevertheless|nonetheless|in contrast|by contrast|as a result)\b", text, ): families.add("DISCOURSE_MARKER") if re.match(r"(?i)^(?:therefore|consequently|thus|accordingly|as a result)\b", text): families.update({"CONSEQUENCE", "RESULT_CLAUSE"}) return families # ============================================================ # 5. FORMULAIC-STYLE DETECTOR # ============================================================ FORMULAIC_PATTERNS = { "FORMULAIC_PURPOSE": [ r"\bthe purpose (?:of this|is to)\b", r"\bthe aim (?:of this|is to)\b", r"\bthe objective (?:of this|is to)\b", ], "FORMULAIC_RESULT": [ r"\bas a result\b", r"\bconsequently\b", r"\btherefore\b", r"\bthis suggests that\b", r"\bthese results indicate that\b", ], "FORMULAIC_COMPARISON": [ r"\bin contrast\b", r"\bby contrast\b", r"\bcompared with\b", r"\bcompared to\b", ], "FORMULAIC_EVALUATION": [ r"\bis evaluated using\b", r"\bare evaluated using\b", r"\bis evaluated across\b", r"\bare evaluated across\b", r"\bis assessed using\b", r"\bare assessed using\b", ], "FORMULAIC_METHOD": [ r"\busing this approach\b", r"\bunder this framework\b", r"\bwithin this framework\b", r"\bthe methodology allows\b", ], "FORMULAIC_CONCLUSION": [ r"\btaken together\b", r"\boverall, these\b", r"\bcollectively, these\b", ], "OVEREXPLICIT_LOGIC": [ r"\bthis means that\b", r"\bthis indicates that\b", r"\bthis demonstrates that\b", r"\bit is important to note that\b", r"\bit should be noted that\b", ], "STILTED_CONNECTIVE": [ r"\bfurthermore\b", r"\bmoreover\b", r"\bnevertheless\b", r"\baccordingly\b", ], } def detect_formulaic_patterns(text, profile): lower_text = _pt_clean(text).lower() for flag, patterns in FORMULAIC_PATTERNS.items(): for pattern in patterns: match = re.search(pattern, lower_text, re.I) if match: profile.formulaic_flags.add(flag) _add_evidence(profile, flag, match.group(0)) # ============================================================ # 6. NATURALNESS / COMPLEXITY DETECTOR # ============================================================ def detect_naturalness(text, profile): text = _pt_clean(text) words = _pt_words(text) word_count = len(words) profile.metrics["word_count"] = word_count if word_count >= 35: profile.complexity_flags.add("LONG_SENTENCE") if word_count >= 50: profile.complexity_flags.add("VERY_LONG_SENTENCE") clause_markers = re.findall( r"\b(?:that|which|whereby|whereas|although|because|while|when|whether|if|so that)\b", text, re.I, ) profile.metrics["clause_marker_count"] = len(clause_markers) if len(clause_markers) >= 3: profile.complexity_flags.add("MULTI_EMBEDDED_CLAUSES") if word_count >= 35 and len(clause_markers) >= 2: profile.naturalness_flags.add("CLAUSE_OVERLOADING") prepositions = re.findall( r"\b(?:of|in|within|under|across|among|through|between|for|from|with|by|on|at)\b", text, re.I, ) profile.metrics["preposition_count"] = len(prepositions) if len(prepositions) >= 6 and word_count >= 25: profile.complexity_flags.add("LONG_PREPOSITIONAL_CHAIN") passive_hits = re.findall( r"\b(?:is|are|was|were|has been|have been|had been)\s+(?:not\s+)?" r"(?:\w+\s+){0,2}\w+(?:ed|en)\b", text, re.I, ) profile.metrics["passive_count"] = len(passive_hits) if len(passive_hits) >= 2: profile.naturalness_flags.add("AWKWARD_PASSIVE") transitions = re.findall( r"\b(?:however|therefore|consequently|thus|moreover|furthermore|" r"accordingly|nevertheless|nonetheless)\b", text, re.I, ) profile.metrics["transition_count"] = len(transitions) if len(transitions) >= 2: profile.discourse_flags.add("REPETITIVE_TRANSITION") nominalizations = re.findall( r"\b[A-Za-z]+(?:tion|sion|ment|ance|ence|ity|ness)\b", text, re.I, ) profile.metrics["nominalization_count"] = len(nominalizations) if len(nominalizations) >= 4 and word_count >= 25: profile.naturalness_flags.add("UNNATURAL_NOMINALIZATION") and_count = len(re.findall(r"\band\b", text, re.I)) profile.metrics["and_count"] = and_count if and_count >= 4 and word_count >= 30: profile.naturalness_flags.add("MECHANICAL_PARALLELISM") # ============================================================ # 7. MAIN ANALYZER / REPORT # ============================================================ def analyze_pattern_taxonomy(text): profile = PatternProfile() profile.grammar_families.update(detect_grammar_families(text)) detect_formulaic_patterns(text, profile) detect_naturalness(text, profile) return profile def taxonomy_categories(flag): return [category for category, members in PATTERN_TAXONOMY.items() if flag in members] def pattern_report(text): profile = analyze_pattern_taxonomy(text) return { "grammar_families": sorted(profile.grammar_families), "formulaic_flags": sorted(profile.formulaic_flags), "naturalness_flags": sorted(profile.naturalness_flags), "complexity_flags": sorted(profile.complexity_flags), "discourse_flags": sorted(profile.discourse_flags), "metrics": dict(profile.metrics), "evidence": dict(profile.evidence), } # ============================================================ # 8. SELF-TESTS # ============================================================ _PATTERN_TESTS = { "while_participial": "The comparison preserves the constraints while applying the methodology.", "while_finite": "Accuracy measures overall performance, while recall tracks missed examples.", "whereby": "The design uses a common procedure whereby each model is evaluated so that comparisons remain consistent.", "evaluation_context": "The models are evaluated across several conditions.", "negated_purpose": "The purpose is not to rank the methods but to compare their behavior.", "passive_process": "The models are trained independently on their respective feature representations.", "negated_classification": "The representations are not treated as evidence of invariant features.", "participial_result": "The models operate in separate environments, allowing their effects to be evaluated.", "that_determiner": "The study does not implement that type of domain alignment.", "that_content_complement": "The results indicate that the models preserve their original behavior.", "coordinated_passive": "The models are trained and evaluated separately.", "contrastive_coordination": "The models are trained separately, but both environments use the same procedure.", "passive_basis": "The interpretation is based on changes in the measured outcomes.", "semicolon_negated_assumption": "The comparison uses separate measurements; it does not assume that the representations are equivalent.", } _PATTERN_RESULTS = {name: pattern_report(sentence) for name, sentence in _PATTERN_TESTS.items()} def _assert_pattern(test_name, expected_family): detected = _PATTERN_RESULTS[test_name]["grammar_families"] if expected_family not in detected: raise RuntimeError( f"Pattern taxonomy failed {expected_family} self-test for '{test_name}'. Detected: {detected}" ) for test_name, expected in [ ("while_participial", "WHILE_PARTICIPIAL"), ("while_finite", "FINITE_WHILE_CLAUSE"), ("whereby", "WHEREBY_CLAUSE"), ("evaluation_context", "PASSIVE_EVALUATION_CONTEXT"), ("negated_purpose", "NEGATED_PURPOSE_CONTRAST"), ("passive_process", "PASSIVE_PROCESS_CONTEXT"), ("negated_classification", "NEGATED_CLASSIFICATION_PASSIVE"), ("participial_result", "PARTICIPIAL_RESULT_CLAUSE"), ("that_content_complement", "CONTENT_COMPLEMENT"), ("coordinated_passive", "COORDINATED_PASSIVE_PREDICATES"), ("contrastive_coordination", "CONTRASTIVE_COORDINATED_CLAUSES"), ("passive_basis", "PASSIVE_BASIS_FRAME"), ("semicolon_negated_assumption", "SEMICOLON_NEGATED_ASSUMPTION"), ]: _assert_pattern(test_name, expected) if "CONTENT_COMPLEMENT" in _PATTERN_RESULTS["that_determiner"]["grammar_families"]: raise RuntimeError("Pattern taxonomy incorrectly classified determiner 'that' as CONTENT_COMPLEMENT.") # ============================================================ # 9. STATUS # ============================================================ print("=" * 72) print("CELL 4B PATTERN TAXONOMY SUCCESSFULLY LOADED") print("=" * 72) print("Grammar-family taxonomy: active") print("Formulaic-expression analysis: active") print("Naturalness analysis: active") print("Complexity analysis: active") print("Discourse analysis: active") print("\nVerified grammar routes:") for family in [ "FINITE_WHILE_CLAUSE", "WHILE_PARTICIPIAL", "WHEREBY_CLAUSE", "PASSIVE_EVALUATION_CONTEXT", "PASSIVE_PROCESS_CONTEXT", "NEGATED_CLASSIFICATION_PASSIVE", "PARTICIPIAL_RESULT_CLAUSE", "NEGATED_PURPOSE_CONTRAST", "CONTENT_COMPLEMENT", "COORDINATED_PASSIVE_PREDICATES", "CONTRASTIVE_COORDINATED_CLAUSES", "PASSIVE_BASIS_FRAME", "SEMICOLON_NEGATED_ASSUMPTION", ]: print(f" [OK] {family}") print(" [OK] determiner 'that' exclusion") print("\nCell 4B is ready. Update and run Cell 5 next.") # ===== SOURCE: runtime/cell5.py ===== #@title 5. STANDALONE UNIVERSAL TRANSFORMATION ENGINE v4.1 — 30 deterministic rules # Complete standalone replacement for Cell 5. # Requires earlier notebook cells to define Candidate, CFG, Analyzer, nlp, # optional, wn, and getInflection. This cell defines the rule registry, # all 25 base rules, the Generator, and the v4.1 primary-clause upgrade. from dataclasses import dataclass from typing import Callable from collections import Counter import re _REQUIRED_CELL5 = ["Candidate", "CFG", "Analyzer", "nlp"] _missing_cell5 = [name for name in _REQUIRED_CELL5 if name not in globals()] if _missing_cell5: raise RuntimeError( "Standalone Cell 5 cannot run. Missing: " + ", ".join(_missing_cell5) + ". Run Cells 1–4/4B first." ) @dataclass class Rule: name: str lanes: set families: set priority: float fn: Callable RULES = [] def rule(name, lanes, families, priority=.5): """Register or replace one deterministic transformation rule.""" def deco(fn): global RULES RULES = [registered for registered in RULES if registered.name != name] RULES.append( Rule( str(name), set(lanes), set(families), float(priority), fn, ) ) return fn return deco def clean(text): value = re.sub( r"\s+([,.;:!?])", r"\1", re.sub(r"\s+", " ", str(text or "")), ).strip() if value: value = value[0].upper() + value[1:] return value if not value or value[-1] in ".!?" else value + "." def bare(text): return re.sub(r"[.?!]+$", "", clean(text)).strip() def lower_initial(text): value = str(text or "").strip() if not value or re.match(r"^[A-Z]{2,}\b", value): return value return value[:1].lower() + value[1:] def surface_key(text): return re.sub(r"\W+", " ", str(text or "").lower()).strip() _surface_key = surface_key def K(text, lane, family, reason, priority=.5): return Candidate( clean(text), lane, family, reason, float(priority), ) def emit(text, lane, family, reason, priority=.5): value = clean(text) return [] if not value else [K(value, lane, family, reason, priority)] def _safe_subtree_text(token): """Return a token subtree without requiring a separate helper cell.""" try: items = list(token.subtree) if not items: return token.text start = min(item.i for item in items) end = max(item.i for item in items) + 1 return token.doc[start:end].text except Exception: return getattr(token, "text", "") # ===================================================================== # BASE RULES 1–25 # ===================================================================== @rule("copular_action_frame", {"Balanced", "Structural"}, {"COPULAR_ACTION_FRAME"}, .92) def tr_copular_action(analysis, profile, user_terms): match = re.match( r"(?is)^The\s+(?Pnext|first|second|third|final|primary|main)\s+" r"(?Pstep|task|action|objective|goal)\s+is\s+to\s+" r"(?P.+?)[.]?$", analysis.text, ) if not match: return [] return [ K( f"The {match['head']} {match['noun']} involves {match['action']}", "Balanced", "COPULAR_ACTION_FRAME", "Converted a copular action frame into an event frame.", .92, ) ] @rule("need_to_action", {"Balanced", "Structural"}, {"NEED_TO_ACTION"}, .88) def tr_need_to(analysis, profile, user_terms): match = re.match( r"(?is)^(?P.+?)\s+(?Pneed|needs|needed)\s+to\s+" r"(?P.+?)[.]?$", analysis.text, ) if not match: return [] modal = "had to" if match["need"].lower() == "needed" else "must" return [ K( f"{match['subject']} {modal} {match['action']}", "Balanced", "NEED_TO_ACTION", "Recast a need-to construction while retaining its subject and action.", .88, ) ] @rule( "finite_intent_rather_than", {"Balanced", "Structural"}, {"FINITE_PURPOSE_INTENT", "RATHER_THAN_GERUND"}, .90, ) def tr_finite_intent(analysis, profile, user_terms): match = re.match( r"(?is)^(?P.+?)\s+" r"(?Pintend|intends|intended)\s+to\s+" r"(?P.+?)\s+rather\s+than\s+" r"(?P\w+ing\b.+?)[.]?$", analysis.text, ) if not match: return [] return [ K( f"Rather than {match['alternative']}, {lower_initial(match['subject'])} " f"{match['intent']} to {match['action']}", "Balanced", "FINITE_PURPOSE_INTENT", "Fronted the rejected alternative while preserving the stated intent.", .90, ) ] @rule( "modal_copular_enumeration", {"Balanced", "Structural"}, {"MODAL_COPULAR_ENUMERATION", "COORDINATED_OBJECTS_OR_ENUMERATION", "ENUMERATION"}, .91, ) def tr_modal_enumeration(analysis, profile, user_terms): match = re.match( r"(?is)^(?P.+?)\s+" r"(?Pmay|might|could|can)\s+be\s+in\s+" r"(?P.+?,\s*.+?\s*,?\s*(?:or|and)\s+.+?)[.]?$", analysis.text, ) if not match: return [] return [ K( f"Possible sources of {lower_initial(match['subject'])} " f"{match['modal'].lower()} include {match['items']}", "Balanced", "MODAL_COPULAR_ENUMERATION", "Recast a modal copular list while preserving uncertainty and every member.", .91, ) ] @rule( "enumeration_frame", {"Minimal", "Balanced"}, {"COORDINATED_OBJECTS_OR_ENUMERATION", "ENUMERATION"}, .65, ) def tr_enumeration_frame(analysis, profile, user_terms): match = re.match( r"(?is)^(?P.+?)\s+(?:consists?\s+of|comprises?)\s+" r"(?P.+?,.+?(?:and|or).+?)[.]?$", analysis.text, ) if not match: return [] return emit( f"{match['subject']} includes {match['items']}", "Minimal", "ENUMERATION", "Varied an explicit enumeration frame without changing its members.", .65, ) @rule( "using_method_relation", {"Balanced", "Structural"}, {"USING_METHOD_RELATION", "METHOD_RELATION"}, .89, ) def tr_using_method(analysis, profile, user_terms): match = re.match( r"(?is)^(?P
.+?)\s+using\s+(?P.+?)[.]?$", analysis.text, ) if not match: return [] return [ K( f"Using {match['method']}, {lower_initial(match['main'])}", "Balanced", "USING_METHOD_RELATION", "Fronted the explicitly stated method relation.", .89, ) ] @rule( "through_method_relation", {"Balanced", "Structural"}, {"THROUGH_METHOD_RELATION", "METHOD_RELATION"}, .86, ) def tr_through_method(analysis, profile, user_terms): match = re.match( r"(?is)^(?P
.+?)\s+through\s+(?P.+?)[.]?$", analysis.text, ) if not match: return [] return emit( f"Through {match['method']}, {lower_initial(match['main'])}", "Balanced", "THROUGH_METHOD_RELATION", "Fronted an explicitly stated through-method relation.", .86, ) @rule( "by_gerund_method", {"Balanced", "Structural"}, {"BY_GERUND_METHOD_RELATION", "METHOD_RELATION"}, .84, ) def tr_by_gerund(analysis, profile, user_terms): match = re.match( r"(?is)^(?P
.+?)\s+by\s+(?P\w+ing\b.+?)[.]?$", analysis.text, ) if not match: return [] return emit( f"By {match['method']}, {lower_initial(match['main'])}", "Balanced", "BY_GERUND_METHOD_RELATION", "Fronted an explicitly stated by-gerund method relation.", .84, ) @rule( "purpose_infinitive_relocation", {"Balanced", "Structural"}, {"PURPOSE_INFINITIVE_FRONTING"}, .88, ) def tr_purpose_front(analysis, profile, user_terms): match = re.match( r"(?is)^To\s+(?P[^,]{2,160}),\s*(?P
.+?)[.]?$", analysis.text, ) if not match: return [] return emit( f"{match['main']} to {match['purpose']}", "Balanced", "PURPOSE_INFINITIVE_FRONTING", "Moved the explicit purpose clause after the main proposition.", .88, ) @rule( "purpose_contrast", {"Balanced", "Structural"}, {"NEGATED_PURPOSE_CONTRAST"}, .98, ) def tr_purpose(analysis, profile, user_terms): match = re.match( r"(?is)^(?P.*?\b(?:purpose|aim|objective|goal)\b.*?)\s+" r"is\s+not\s+to\s+(?P.+?)\s+but\s+to\s+(?P.+?)[.]?$", analysis.text, ) if not match: return [] return [ K( f"{match['head']} is to {match['y']} rather than to {match['x']}", "Balanced", "NEGATED_PURPOSE_CONTRAST", "Reordered contrasted objectives while preserving both scopes.", .98, ), K( f"{match['head']} does not seek to {match['x']}. " f"Instead, it aims to {match['y']}", "Structural", "NEGATED_PURPOSE_CONTRAST", "Separated the excluded and intended objectives.", .90, ), ] @rule( "evidential_complement", {"Balanced", "Structural"}, {"EVIDENTIAL_COMPLEMENT"}, .94, ) def tr_evidence(analysis, profile, user_terms): match = re.match( r"(?is)^(?P.+?)\s+" r"(?:provide|provides|provided|offer|offers|offered)\s+" r"(?:clear\s+|strong\s+|empirical\s+)?evidence\s+that\s+" r"(?P.+?)[.]?$", analysis.text, ) if not match: return [] return [ K( f"Evidence from {lower_initial(match['src'])} indicates that {match['claim']}", "Balanced", "EVIDENTIAL_COMPLEMENT", "Recast the evidential frame while preserving the claim.", .94, ), K( f"According to the evidence provided by {lower_initial(match['src'])}, " f"{match['claim']}", "Structural", "EVIDENTIAL_COMPLEMENT", "Reorganized the source and claim around the same evidential relation.", .76, ), ] @rule("whether_complement", {"Balanced", "Structural"}, {"WHETHER_COMPLEMENT"}, .89) def tr_whether(analysis, profile, user_terms): match = re.match( r"(?is)^(?P.+?)\s+require(?:s|d)?\s+" r"(?:an?\s+)?(?:consideration|assessment|evaluation|analysis)\s+" r"of\s+whether\s+(?P.+?)[.]?$", analysis.text, ) if not match: return [] subject = re.sub(r"\balso\b", "", match["subject"], flags=re.I).strip() return emit( f"Assessing whether {match['clause']} is also required when considering " f"{lower_initial(subject)}", "Balanced", "WHETHER_COMPLEMENT", "Converted a nominalized whether-complement into an assessment clause.", .89, ) @rule("gerund_subject", {"Balanced", "Structural"}, {"GERUND_SUBJECT"}, .88) def tr_gerund(analysis, profile, user_terms): source = bare(analysis.text) match = re.match( r"(?is)^(?PApplying|Using|Evaluating|Comparing|Assessing|Examining|" r"Analyzing|Implementing|Testing|Measuring)\s+(?P.+?)\s+" r"(?Preduces?|provides?|enables?|supports?|allows?|improves?|" r"limits?|increases?|decreases?|preserves?|reveals?|shows?)\b(?P.*)$", source, ) if not match: return [] nouns = { "applying": "application", "using": "use", "evaluating": "evaluation", "comparing": "comparison", "assessing": "assessment", "examining": "examination", "analyzing": "analysis", "implementing": "implementation", "testing": "testing", "measuring": "measurement", } event = nouns[match["verb"].lower()] return emit( f"The {event} of {match['object']} {match['predicate']}{match['tail']}", "Balanced", "GERUND_SUBJECT", "Converted a gerund event into an explicit nominal subject.", .88, ) @rule("whereby_mechanism", {"Balanced", "Structural"}, {"WHEREBY_CLAUSE"}, .84) def tr_whereby(analysis, profile, user_terms): match = re.match( r"(?is)^(?P
.+?)\s+whereby\s+(?P.+?)" r"(?:\s+so\s+that\s+(?P.+?))?[.]?$", analysis.text, ) if not match: return [] middle = ( f"{match['main']}; under this design, {lower_initial(match['x'])}" + (f", so that {match['y']}" if match.group("y") else "") ) output = [ K( middle, "Balanced", "WHEREBY_CLAUSE", "Separated the main proposition from its mechanism.", .84, ) ] if match.group("y"): output.append( K( f"{match['main']}. Under this design, {match['x']}. " f"This arrangement is intended so that {match['y']}", "Structural", "WHEREBY_CLAUSE", "Separated the claim, mechanism, and purpose.", .72, ) ) return output @rule("while_participial", {"Balanced", "Structural"}, {"WHILE_PARTICIPIAL"}, .84) def tr_while(analysis, profile, user_terms): match = re.match( r"(?is)^(?P
.+?)\s+while\s+(?P\w+ing\b.+?)[.]?$", analysis.text, ) if not match: return [] return emit( f"While {match['x']}, {lower_initial(match['main'])}", "Balanced", "WHILE_PARTICIPIAL", "Reordered a simultaneous participial clause without changing its relation.", .84, ) @rule("fronted_adjunct", {"Balanced", "Structural"}, {"FRONTED_ADJUNCT"}, .73) def tr_front(analysis, profile, user_terms): if re.match( r"(?is)^\s*(?:for purposes of|as used in|for the purposes of|" r"in this (?:agreement|contract|plan|section|policy|specification))\b", analysis.text, ): return [] match = re.match( r"(?is)^(?P(?:For|In|Within|Under|During|Among|Across|Throughout)\s+" r"[^,]+),\s*(?P
.+?)[.]?$", analysis.text, ) if not match: return [] return emit( f"{match['main']} {lower_initial(match['adj'])}", "Balanced", "FRONTED_ADJUNCT", "Moved an existing domain adjunct after the main proposition.", .73, ) @rule("discourse_marker", {"Minimal", "Balanced"}, {"DISCOURSE_FRONTING"}, .72) def tr_discourse(analysis, profile, user_terms): match = re.match( r"(?is)^(?PHowever|Therefore|Consequently|Thus|Moreover|Furthermore|" r"Nevertheless|Accordingly|Additionally),\s+(?P
.+?)[.]?$", analysis.text, ) if not match: return [] synonyms = { "However": "Nevertheless", "Therefore": "For this reason", "Consequently": "As a result", "Thus": "Accordingly", "Moreover": "In addition", "Furthermore": "Additionally", "Nevertheless": "Even so", "Accordingly": "Therefore", "Additionally": "In addition", } return emit( f"{synonyms[match['mark']]}, {match['main']}", "Minimal", "DISCOURSE_FRONTING", "Varied an existing discourse marker within the same relation class.", .62, ) @rule( "coordinated_passive_participles", {"Minimal", "Balanced"}, {"COORDINATED_PASSIVE_PARTICIPLES", "COORDINATED_PREDICATES"}, .86, ) def tr_passive_series(analysis, profile, user_terms): source = bare(analysis.text) match = re.search( r"(?is)\b(?P[A-Za-z][\w-]*(?:\s+[A-Za-z][\w-]*){0,5})\s+that\s+" r"(?:have|has|had)\s+been\s+" r"(?P(?:\w+ly\s+)?\w+(?:ed|en)\s*,\s*" r"\w+(?:ed|en)\s*,\s*and\s*\w+(?:ed|en)(?:\s+by\s+[^.;]+)?)", source, ) if not match: return [] revised = source[:match.start()] + f"{match['noun']} {match['series']}" + source[match.end():] return emit( revised, "Minimal", "COORDINATED_PASSIVE_PARTICIPLES", "Reduced a relative passive clause while preserving its coordinated series.", .86, ) @rule("passive_using_nominal", {"Balanced", "Structural"}, {"PASSIVE_USING"}, .91) def tr_passive_using(analysis, profile, user_terms): match = re.match( r"(?is)^(?P.+?)\s+" r"(?:is|are|was|were|has been|have been|had been)\s+" r"(?P\w+ly\s+)?evaluated\s+using\s+" r"(?P.+?)[.]?$", analysis.text, ) if not match: return [] adverb = (match.group("adverb") or "").strip() return emit( f"Evaluations of {lower_initial(match['subject'])} " f"{adverb + ' ' if adverb else ''}use {match['object']}", "Balanced", "PASSIVE_USING", "Recast a passive evaluation without inventing an actor.", .91, ) @rule("agentless_study_passive", {"Balanced", "Structural"}, {"PASSIVE"}, .70) def tr_agentless(analysis, profile, user_terms): match = re.match( r"(?is)^(?P.+?)\s+(?:has|have)\s+(?Palso\s+)?been\s+" r"(?:studied|examined|investigated)\s+" r"(?P(?:within|in|across)\s+.+?)[.]?$", analysis.text, ) if not match: return [] return emit( f"Studies {match['context']} have {match['also'] or ''}examined {match['topic']}", "Balanced", "PASSIVE", "Recast an agentless research passive without inventing a specific actor.", .70, ) @rule("explicit_agent_active", {"Structural"}, {"PASSIVE"}, .82) def tr_explicit_agent(analysis, profile, user_terms): match = re.match( r"(?is)^(?P.+?)\s+" r"(?:was|were|is|are|has been|have been|had been)\s+" r"(?Pevaluated|examined|studied|investigated|tested|measured|reported|" r"implemented|applied|modified|trained|collected|derived|obtained|inspected|" r"approved|reviewed|documented)\s+by\s+(?P.+?)[.]?$", analysis.text, ) if not match: return [] return emit( f"{match['agent']} {match['verb']} {lower_initial(match['theme'])}", "Structural", "PASSIVE", "Converted passive to active because the source supplied the actor.", .82, ) @rule( "reduced_modifier", {"Minimal", "Balanced"}, {"REDUCED_RELATIVE_OR_NP_MODIFIER"}, .68, ) def tr_reduced(analysis, profile, user_terms): source = bare(analysis.text) output = [] for pattern, replacement in [ (r"\bresults?\s+obtained\s+from\b", "results derived from"), (r"\bsamples?\s+obtained\s+from\b", "samples drawn from"), (r"\bdata\s+collected\s+from\b", "data gathered from"), (r"\bresults?\s+reported\s+independently\b", "independently reported results"), ]: if re.search(pattern, source, re.I): output.append( K( re.sub(pattern, replacement, source, count=1, flags=re.I), "Minimal", "REDUCED_RELATIVE_OR_NP_MODIFIER", "Recast a reduced participial noun modifier.", .68, ) ) return output @rule( "parenthetical_emphasis", {"Minimal", "Balanced"}, {"PARENTHETICAL_EMPHASIS"}, .78, ) def tr_parenthetical(analysis, profile, user_terms): revised = re.sub( r",\s*by\s+(?:itself|themselves)\s*,", " alone", analysis.text, flags=re.I, ) if surface_key(revised) == surface_key(analysis.text): return [] return emit( revised, "Minimal", "PARENTHETICAL_EMPHASIS", "Integrated parenthetical emphasis into the clause.", .78, ) @rule("coordinated_predicate_split", {"Structural"}, {"COORDINATED_PREDICATES"}, .80) def tr_coord_base(analysis, profile, user_terms): try: document = nlp()(bare(analysis.text)) if not document.has_annotation("DEP"): return [] root = next((token for token in document if token.dep_ == "ROOT"), None) if root is None: return [] conjuncts = [ token for token in root.conjuncts if token.pos_ in {"VERB", "AUX"} ] subjects = [ child for child in root.children if child.dep_ in {"nsubj", "nsubjpass", "csubj"} ] if len(conjuncts) != 1 or not subjects: return [] conjunct = conjuncts[0] cut = min(token.i for token in conjunct.subtree) left = re.sub( r"(?:,\s*)?\b(?:and|or)\s*$", "", document[:cut].text.strip(), flags=re.I, ).strip() subject = _safe_subtree_text(subjects[0]).strip() right = _safe_subtree_text(conjunct).strip() if not left or not subject or not right: return [] return emit( f"{left}. {subject} {right}", "Structural", "COORDINATED_PREDICATES", "Split coordinated predicates and repeated only the source subject.", .80, ) except Exception: return [] @rule("semicolon_split", {"Structural"}, {"LONG_MULTI_CLAUSE"}, .65) def tr_semicolon(analysis, profile, user_terms): parts = [item.strip() for item in bare(analysis.text).split(";") if item.strip()] if len(parts) != 2 or not all(len(item.split()) >= 3 for item in parts): return [] return emit( f"{parts[0]}. {parts[1]}", "Structural", "LONG_MULTI_CLAUSE", "Split two propositions already delimited by a semicolon.", .65, ) # ===================================================================== # GENERATOR # ===================================================================== class Generator: def applicable(self, analysis, lane): detected = set(getattr(analysis, "families", []) or []) return sorted( [ registered for registered in RULES if lane in registered.lanes and bool(registered.families & detected) ], key=lambda registered: -registered.priority, ) def lane(self, analysis, profile, user_terms, lane, trace): output = [] seen = {surface_key(analysis.text)} for registered in self.applicable(analysis, lane): try: generated = registered.fn(analysis, profile, user_terms) or [] except Exception as exc: trace["rule_errors"].append( { "rule": registered.name, "lane": lane, "error": str(exc), } ) continue for candidate in generated: if candidate is None or candidate.lane != lane: continue key = surface_key(candidate.text) if key and key not in seen: seen.add(key) output.append(candidate) if lane == "Minimal" and getattr(CFG, "mlm", False): output.extend(self.lexical(analysis, profile, user_terms, trace)) return output[:16] def generate(self, analysis, profile, user_terms, mode): if mode not in {"Minimal", "Balanced", "Structural"}: mode = "Balanced" trace = { "detected_families": list(getattr(analysis, "families", []) or []), "applicable": {}, "generated_by_lane": {}, "rule_errors": [], "candidate_traces": [], } pools = {} for lane in ["Minimal", "Balanced", "Structural"]: trace["applicable"][lane] = [ registered.name for registered in self.applicable(analysis, lane) ] pools[lane] = self.lane( analysis, profile, user_terms, lane, trace, ) trace["generated_by_lane"][lane] = len(pools[lane]) order = { "Minimal": ["Minimal"], "Balanced": ["Balanced", "Minimal", "Structural"], "Structural": ["Structural", "Balanced", "Minimal"], }[mode] output = [] seen = set() for lane in order: for candidate in pools[lane]: key = surface_key(candidate.text) if key and key not in seen: seen.add(key) output.append(candidate) # Bounded depth-2 composition. This invokes registered functions # directly and never recurses through Generator.generate. if mode in {"Balanced", "Structural"}: composed = [] for base in list(output[:8]): try: second_analysis = Analyzer().analyze( base.text, profile, user_terms, ) except Exception: continue next_lanes = ["Minimal"] if mode == "Balanced" else ["Balanced", "Minimal"] for next_lane in next_lanes: for registered in self.applicable(second_analysis, next_lane)[:4]: try: second_candidates = registered.fn( second_analysis, profile, user_terms, ) or [] except Exception: continue for second in second_candidates[:1]: key = surface_key(second.text) if not key or key in seen: continue seen.add(key) second.lane = mode second.family = f"{base.family}+{second.family}" continuation = second.reason[:1].lower() + second.reason[1:] second.reason = f"{base.reason} Then {continuation}" second.priority = (base.priority + second.priority) / 2.0 composed.append(second) output.extend(composed[:8]) trace["routes"] = [ name for lane in order for name in trace["applicable"].get(lane, []) ] trace["generated"] = len(output) return output, trace def lexical(self, analysis, profile, user_terms, trace=None): optional_fn = globals().get("optional") if not callable(optional_fn): return [] pipeline = optional_fn("mlm") if not pipeline: return [] if "wn" not in globals() or "getInflection" not in globals(): return [] try: document = nlp()(analysis.text) except Exception as exc: if trace is not None: trace["rule_errors"].append( { "rule": "contextual_lexical", "lane": "Minimal", "error": str(exc), } ) return [] protected = { str(term).lower() for term in ( set(getattr(profile, "immutable", set()) or set()) | set(user_terms or set()) ) } output = [] seen = {surface_key(analysis.text)} for token in document: if token.pos_ not in {"ADJ", "ADV", "VERB"}: continue if token.is_stop or not token.is_alpha: continue if token.lemma_.lower() in protected: continue try: allowed = { lemma.name().replace("_", " ").lower() for synset in wn.synsets(token.lemma_)[:4] for lemma in synset.lemmas() } mask = pipeline.tokenizer.mask_token predictions = pipeline( analysis.text[:token.idx] + mask + analysis.text[token.idx + len(token.text):], top_k=12, ) except Exception: continue for prediction in predictions: word = str(prediction.get("token_str", "")).strip().lower() if not word.isalpha() or word == token.lower_ or word not in allowed: continue try: inflected = getInflection(word, tag=token.tag_) replacement = inflected[0] if inflected else word except Exception: replacement = word revised = ( analysis.text[:token.idx] + replacement + analysis.text[token.idx + len(token.text):] ) candidate = K( revised, "Minimal", "CONTEXTUAL_LEXICAL", "Contextual lexical substitution supported by WordNet and the optional masked-language model.", .45, ) key = surface_key(candidate.text) if key and key not in seen: seen.add(key) output.append(candidate) break if len(output) >= 3: break return output # Base registry invariant: 25 unique names before the v4.1 upgrade. _BASE_RULE_NAMES = [registered.name for registered in RULES] _BASE_DUPLICATES = sorted( name for name, count in Counter(_BASE_RULE_NAMES).items() if count > 1 ) if _BASE_DUPLICATES: raise RuntimeError( "Standalone Cell 5 base registry has duplicate rules: " + ", ".join(_BASE_DUPLICATES) ) if len(RULES) != 25: raise RuntimeError( "Standalone Cell 5 base registry expected 25 rules but found " + str(len(RULES)) ) print("=" * 78) print("CELL 5 BASE UNIVERSAL TRANSFORMATION ENGINE LOADED") print("=" * 78) print("Base rules:", len(RULES)) print("Base duplicate rule names:", _BASE_DUPLICATES) print("Generator: defined") print("=" * 78) #@title 5. UNIVERSAL REPLACEMENT v4.1 — production primary-clause upgrade # REPLACE the earlier Cell 5 v4 upgrade block with this block. # Keep the original Cell 5 v3 registry/generator above it. # # Design goals: # - deterministic and non-generative; # - preserve enumerations verbatim whenever possible; # - prefer one-sentence recasts to avoid false enumeration loss; # - use spaCy dependency/noun-chunk evidence when available; # - use bounded surface fallbacks when dependency parsing is unavailable; # - never create a candidate merely to force novelty; # - replace same-name rules without increasing the registry beyond 30. from collections import Counter import re # ===================================================================== # 0. SHARED PRODUCTION HELPERS # ===================================================================== _V41_PARTICIPLE_TO_BASE = { "spanning": "span", "covering": "cover", "including": "include", "containing": "contain", "involving": "involve", } _V41_IRREGULAR_3SG = { "have": "has", "do": "does", "go": "goes", } def _v41_distinct(source, candidate): return bool(candidate and surface_key(source) != surface_key(candidate)) def _v41_balanced_parentheses(text): pairs = {"(": ")", "[": "]", "{": "}"} stack = [] for char in str(text or ""): if char in pairs: stack.append(pairs[char]) elif char in pairs.values(): if not stack or stack.pop() != char: return False return not stack def _v41_sentence_doc(text): try: doc = nlp()(str(text or "")) if len(doc): return doc except Exception: pass return None def _v41_subtree_span(token): items = list(token.subtree) if not items: return None start = min(item.i for item in items) end = max(item.i for item in items) + 1 return token.doc[start:end] def _v41_noun_chunk_for_token(doc, token): try: chunks = list(doc.noun_chunks) except Exception: chunks = [] containing = [chunk for chunk in chunks if chunk.start <= token.i < chunk.end] if not containing: return None # Prefer the narrowest chunk containing the syntactic head. return min(containing, key=lambda chunk: chunk.end - chunk.start) def _v41_present_verb(base, singular): base = str(base or "").lower() if not singular: return base if base in _V41_IRREGULAR_3SG: return _V41_IRREGULAR_3SG[base] if re.search(r"(?:s|x|z|ch|sh|o)$", base): return base + "es" if re.search(r"[^aeiou]y$", base): return base[:-1] + "ies" return base + "s" def _v41_token_is_singular(token): try: number = token.morph.get("Number") if "Sing" in number: return True if "Plur" in number: return False except Exception: pass # Conservative surface fallback. return not token.text.lower().endswith("s") def _v41_list_preserved(source_fragment, candidate): """Require list-bearing source fragments to survive literally.""" fragment = re.sub(r"\s+", " ", str(source_fragment or "")).strip(" .") candidate_norm = re.sub(r"\s+", " ", str(candidate or "")) return not fragment or fragment.lower() in candidate_norm.lower() def _v41_emit_if_safe(source, text, lane, family, reason, priority): candidate = clean(text) if not _v41_distinct(source, candidate): return [] if not _v41_balanced_parentheses(candidate): return [] return [K(candidate, lane, family, reason, priority)] # ===================================================================== # 26. FINITE PREDICATE + WITHOUT COMPLEMENT LIST # ===================================================================== @rule( "without_complement_list", {"Balanced", "Structural"}, { "FINITE_COMPLEMENT_LIST", "COORDINATED_OBJECTS_OR_ENUMERATION", "ENUMERATION", "LONG_MULTI_CLAUSE", }, .92, ) def tr_without_complement_list(analysis, profile, user_terms): source = bare(analysis.text) match = re.match( r"(?is)^(?P
.+?)\s+without\s+" r"(?P.+?,\s*.+?(?:,\s*)?(?:and|or)\s+.+?)$", source, ) if not match: return [] main = match["main"].strip() missing = match["missing"].strip() outputs = [] # Candidate A keeps the exact polarity marker and the complete list. text_a = f"Without {missing}, {lower_initial(main)}" if _v41_list_preserved(missing, text_a): outputs.extend(_v41_emit_if_safe( source, text_a, "Balanced", "FINITE_COMPLEMENT_LIST", "Fronted the existing without-complement while preserving the exact polarity marker and complete list.", .92, )) # Candidate B preserves attachment in sentence-final position while # creating a distinct clause boundary. It is useful when a validator is # sensitive to fronted negative adjuncts. text_b = f"{main}; this occurs without {missing}" if _v41_list_preserved(missing, text_b): outputs.extend(_v41_emit_if_safe( source, text_b, "Structural", "FINITE_COMPLEMENT_LIST", "Separated the main proposition from its existing without-complement without changing negative scope.", .86, )) return outputs # ===================================================================== # 27. PROPOSAL FRAME + APPOSITIVE DEFINITION # ===================================================================== @rule( "proposal_appositive_definition", {"Balanced", "Structural"}, { "PROPOSAL_FRAME", "APPOSITIVE_DEFINITION", "COORDINATED_OBJECTS_OR_ENUMERATION", "ENUMERATION", "LONG_MULTI_CLAUSE", }, .98, ) def tr_proposal_appositive_definition(analysis, profile, user_terms): source = bare(analysis.text) match = re.match( r"(?is)^(?PWe|I|The\s+(?:study|paper|article|report|team|authors?))\s+" r"(?Ppropose|proposes|introduce|introduces|present|presents)\s+" r"(?P.+?)\s*,\s*" r"(?P
an?|the)\s+" r"(?P.+)$", source, ) if not match: return [] name = match["name"].strip() definition = match["definition"].strip() article = match["article"].lower() if not _v41_balanced_parentheses(name): return [] outputs = [] # Keep the name and full appositive enumeration in one sentence so # whole-candidate enumeration comparison remains stable. text_a = ( f"{name} is {article} {definition} proposed by " f"{lower_initial(match['subject'])}" ) if _v41_list_preserved(definition, text_a): outputs.extend(_v41_emit_if_safe( source, text_a, "Balanced", "APPOSITIVE_DEFINITION", "Converted the proposal-plus-appositive frame into one definitional sentence while preserving the complete definition.", .98, )) # A second one-sentence option retains the source's proposal predicate. text_b = ( f"{match['subject']} {match['verb']} {name} as " f"{article} {definition}" ) if _v41_list_preserved(definition, text_b): outputs.extend(_v41_emit_if_safe( source, text_b, "Structural", "PROPOSAL_FRAME", "Recast the appositive as an explicit as-complement while preserving the name and complete definition.", .94, )) return outputs # ===================================================================== # 28. PARTICIPIAL POSTMODIFIER # ===================================================================== @rule( "participial_postmodifier", {"Balanced", "Structural"}, { "PARTICIPIAL_POSTMODIFIER", "COORDINATED_OBJECTS_OR_ENUMERATION", "ENUMERATION", "NOMINALIZATION", }, .94, ) def tr_participial_postmodifier(analysis, profile, user_terms): source = bare(analysis.text) doc = _v41_sentence_doc(source) outputs = [] if doc is not None and doc.has_annotation("DEP"): targets = [ token for token in doc if token.text.lower() in _V41_PARTICIPLE_TO_BASE and token.pos_ in {"VERB", "ADJ"} ] for participle in targets: # In common parses, the participial modifier is attached to the # noun it modifies through acl/relcl or an equivalent relation. head = participle.head if head is participle or head.pos_ not in {"NOUN", "PROPN", "PRON"}: continue chunk = _v41_noun_chunk_for_token(doc, head) if chunk is None: chunk_start = head.left_edge.i chunk_end = head.i + 1 else: chunk_start = chunk.start chunk_end = chunk.end if participle.i < chunk_end: continue # The complement is the participle subtree excluding the # participle token itself. This preserves enumerated material. subtree = _v41_subtree_span(participle) if subtree is None: continue complement_start = participle.i + 1 complement_end = subtree.end if complement_start >= complement_end: continue complement = doc[complement_start:complement_end].text.strip() if not complement: continue base = _V41_PARTICIPLE_TO_BASE[participle.text.lower()] finite = _v41_present_verb(base, _v41_token_is_singular(head)) prefix = doc[:participle.i].text.rstrip() suffix = doc[complement_end:].text.lstrip() candidate = f"{prefix} that {finite} {complement}" if suffix: candidate += " " + suffix if _v41_list_preserved(complement, candidate): outputs.extend(_v41_emit_if_safe( source, candidate, "Balanced", "PARTICIPIAL_POSTMODIFIER", "Converted an attested participial noun modifier into a finite relative clause while preserving its head and complement.", .94, )) if outputs: return outputs # Bounded fallback for environments without a dependency parse. fallback = re.match( r"(?is)^(?P.+?\b(?:to|for|with|among|across)\s+)" r"(?P(?:[A-Za-z][\w-]*\s+){0,5}[A-Za-z][\w-]*)\s+" r"(?Pspanning|covering|including|containing|involving)\s+" r"(?P.+?,\s*.+?(?:,\s*)?(?:and|or)\s+.+?)$", source, ) if not fallback: return [] head = fallback["head"].strip() final_noun = head.split()[-1] singular = not final_noun.lower().endswith("s") base = _V41_PARTICIPLE_TO_BASE[fallback["participle"].lower()] finite = _v41_present_verb(base, singular) candidate = ( f"{fallback['prefix']}{head} that {finite} {fallback['items']}" ) if not _v41_list_preserved(fallback["items"], candidate): return [] return _v41_emit_if_safe( source, candidate, "Balanced", "PARTICIPIAL_POSTMODIFIER", "Converted a bounded participial postmodifier into a finite relative clause while preserving its noun phrase and complete list.", .88, ) # ===================================================================== # 29. FINITE VERB + COMPLEX OBJECT LIST # ===================================================================== @rule( "finite_object_list_frame", {"Balanced", "Structural"}, { "FINITE_OBJECT_LIST_FRAME", "COORDINATED_OBJECTS_OR_ENUMERATION", "ENUMERATION", "LONG_MULTI_CLAUSE", }, .88, ) def tr_finite_object_list_frame(analysis, profile, user_terms): source = bare(analysis.text) match = re.match( r"(?is)^(?PWe|I|They|The\s+[A-Za-z][\w-]*(?:\s+[A-Za-z][\w-]*){0,4})\s+" r"(?Ppre-register|preregister|document|record|evaluate|assess|review|" r"analyze|compare|test|measure|include|identify|specify|report)\s+" r"(?P.+?,\s*.+?(?:,\s*)?(?:and|or)\s+.+?)$", source, ) if not match: return [] nominal = { "pre-register": "preregistration", "preregister": "preregistration", "document": "documentation", "record": "record", "evaluate": "evaluation", "assess": "assessment", "review": "review", "analyze": "analysis", "compare": "comparison", "test": "testing", "measure": "measurement", "include": "scope", "identify": "identification", "specify": "specification", "report": "report", }.get(match["verb"].lower()) if not nominal: return [] subject = match["subject"].strip() possessive = {"we": "Our", "i": "My", "they": "Their"}.get(subject.lower()) if possessive: candidate = f"{possessive} {nominal} covers {match['items']}" else: candidate = ( f"The {nominal} conducted by {lower_initial(subject)} " f"covers {match['items']}" ) if not _v41_list_preserved(match["items"], candidate): return [] return _v41_emit_if_safe( source, candidate, "Balanced", "FINITE_OBJECT_LIST_FRAME", "Recast the finite predicate as an event frame while preserving every listed object verbatim.", .88, ) # ===================================================================== # 30. CONTINGENCY / DEPENDENCY RELATION # ===================================================================== @rule( "contingency_relation", {"Minimal", "Balanced", "Structural"}, { "CONTINGENCY_RELATION", "COORDINATED_OBJECTS_OR_ENUMERATION", "ENUMERATION", "NOMINALIZATION", }, .96, ) def tr_contingency_relation(analysis, profile, user_terms): source = bare(analysis.text) match = re.match( r"(?is)^(?P.+?)\s+" r"(?Pis|are|was|were)\s+contingent\s+on\s+" r"(?P.+)$", source, ) if not match: return [] verb = { "is": "depends", "are": "depend", "was": "depended", "were": "depended", }[match["copula"].lower()] return _v41_emit_if_safe( source, f"{match['subject']} {verb} on {match['basis']}", "Minimal", "CONTINGENCY_RELATION", "Recast the contingent-on relation as an equivalent dependency relation.", .96, ) # ===================================================================== # 31. SAFER COORDINATED-PREDICATE SPLIT # ===================================================================== @rule( "coordinated_predicate_split", {"Structural"}, {"COORDINATED_PREDICATES"}, .86, ) def tr_coord_safe(analysis, profile, user_terms): try: document = nlp()(bare(analysis.text)) if not document.has_annotation("DEP"): return [] root = next((token for token in document if token.dep_ == "ROOT"), None) if root is None or root.pos_ not in {"VERB", "AUX"}: return [] subjects = [ child for child in root.children if child.dep_ in {"nsubj", "nsubjpass", "csubj"} ] conjuncts = [ token for token in root.conjuncts if token.pos_ in {"VERB", "AUX"} and token.dep_ == "conj" and token.i > root.i ] if len(subjects) != 1 or len(conjuncts) != 1: return [] conjunct = conjuncts[0] between = document[root.i + 1:conjunct.i].text if not re.search(r"(?:,\s*)?\b(?:and|or)\b", between, re.I): return [] subject = sub(subjects[0]).strip() right = sub(conjunct).strip() cut = min(token.i for token in conjunct.subtree) left = re.sub( r"(?:,\s*)?\b(?:and|or)\s*$", "", document[:cut].text.strip(), flags=re.I, ).strip() if not left or not subject or not right: return [] if len(left.split()) < 4: return [] if re.match(r"(?is)^(?:propose|introduce|present)\b", right): return [] return _v41_emit_if_safe( bare(analysis.text), f"{left}. {subject} {right}", "Structural", "COORDINATED_PREDICATES", "Split two dependency-verified coordinated predicates and repeated only the attested subject.", .86, ) except Exception: return [] # ===================================================================== # V4.1 SELF-TESTS AND REGISTRY CHECK # ===================================================================== _V41_REQUIRED_RULES = { "without_complement_list", "proposal_appositive_definition", "participial_postmodifier", "finite_object_list_frame", "contingency_relation", "coordinated_predicate_split", } _v41_names = [registered.name for registered in RULES] _v41_missing = sorted(_V41_REQUIRED_RULES - set(_v41_names)) _v41_duplicates = sorted( name for name, count in Counter(_v41_names).items() if count > 1 ) if _v41_missing: raise RuntimeError("Cell 5 v4.1 missing rules: " + ", ".join(_v41_missing)) if _v41_duplicates: raise RuntimeError("Cell 5 v4.1 duplicate rules: " + ", ".join(_v41_duplicates)) if len(RULES) != 30: raise RuntimeError( "Cell 5 v4.1 expected 30 rules but found " + str(len(RULES)) ) # Parser-independent invariant tests. if not _v41_balanced_parentheses("Framework (ALCEF-RSF)"): raise RuntimeError("Cell 5 v4.1 parenthesis invariant failed.") if _v41_balanced_parentheses("Framework (ALCEF-RSF"): raise RuntimeError("Cell 5 v4.1 unbalanced-parenthesis test failed.") if not _v41_list_preserved("A, B, and C", "The scope covers A, B, and C."): raise RuntimeError("Cell 5 v4.1 list-preservation invariant failed.") if _v41_present_verb("span", True) != "spans": raise RuntimeError("Cell 5 v4.1 singular conjugation test failed.") if _v41_present_verb("span", False) != "span": raise RuntimeError("Cell 5 v4.1 plural conjugation test failed.") print("=" * 78) print("CELL 5 UNIVERSAL TRANSFORMATION ENGINE v4.1 LOADED") print("=" * 78) print("Dependency-aware noun-head extraction: active") print("Bounded surface fallback: active") print("Literal list-preservation checks: active") print("One-sentence proposal/appositive recasts: active") print("Polarity-marker preservation candidates: active") print("Finite relative-clause postmodifier recast: active") print("Safer coordinated-predicate split: active") print("Missing v4.1 rules:", _v41_missing) print("Duplicate rule names:", _v41_duplicates) print("Registered transformation rules:", len(RULES)) print("Expected rule count: 30") print("Invariant self-tests: PASSED") print("=" * 78) # ===================================================================== # V5 UNIVERSAL DEPENDENCY LAYER — VOCABULARY-INDEPENDENT FALLBACKS # ===================================================================== # This layer supplements the 30 validated rules. It operates from spaCy POS, # lemma, morphology, dependencies, noun chunks, and clause spans rather than # fixed domain vocabulary. Specialized rules remain preferred by priority. UNIVERSAL_MAX_CANDIDATES_PER_LANE = 24 UNIVERSAL_MAX_COMPOSED_CANDIDATES = 12 UNIVERSAL_RULE_NAMES = set() def universal_rule(name, lanes, families, priority=.60): """Register one universal dependency rule and record its identity.""" def deco(fn): UNIVERSAL_RULE_NAMES.add(name) return rule(name, lanes, families, priority)(fn) return deco def _u_doc(text): try: document = nlp()(str(text or "")) if document.has_annotation("DEP"): return document except Exception: pass return None def _u_root(document): if document is None: return None return next((token for token in document if token.dep_ == "ROOT"), None) def _u_span(token): try: items = list(token.subtree) if not items: return token.text start = min(item.i for item in items) end = max(item.i for item in items) + 1 return token.doc[start:end].text.strip() except Exception: return getattr(token, "text", "").strip() def _u_subject(root): if root is None: return None return next( ( child for child in root.children if child.dep_ in {"nsubj", "nsubjpass", "nsubj:pass", "csubj"} ), None, ) def _u_objects(root): if root is None: return [] return [ child for child in root.children if child.dep_ in {"obj", "dobj", "attr", "oprd"} ] def _u_agent(root): """Return the explicit by-agent object for a passive root.""" if root is None: return None for child in root.children: if child.dep_ in {"agent", "obl:agent"}: pobj = next( ( item for item in child.children if item.dep_ in {"pobj", "obj"} ), None, ) return pobj or child if child.lower_ == "by" and child.dep_ in {"prep", "agent"}: pobj = next( ( item for item in child.children if item.dep_ in {"pobj", "obj"} ), None, ) if pobj is not None: return pobj return None def _u_is_passive(root): if root is None: return False passive_subject = any( child.dep_ in {"nsubjpass", "nsubj:pass"} for child in root.children ) passive_aux = any( child.dep_ in {"auxpass", "aux:pass"} for child in root.children ) return passive_subject or passive_aux def _u_number(token): try: values = token.morph.get("Number") if "Plur" in values: return "plural" if "Sing" in values: return "singular" except Exception: pass text = getattr(token, "text", "").lower() return "plural" if text.endswith("s") and not text.endswith("ss") else "singular" def _u_tense(root): try: values = root.morph.get("Tense") if "Past" in values: return "past" if "Pres" in values: return "present" except Exception: pass return "past" if getattr(root, "tag_", "") == "VBD" else "present" def _u_inflect(lemma, tag): """Use lemminflect when available; otherwise apply conservative forms.""" lemma = str(lemma or "").strip() if not lemma: return lemma get_inflection = globals().get("getInflection") if callable(get_inflection): try: values = get_inflection(lemma, tag=tag) if values: return values[0] except Exception: pass irregular = { ("be", "VBD"): "was", ("be", "VBZ"): "is", ("be", "VBP"): "are", ("have", "VBD"): "had", ("have", "VBZ"): "has", ("do", "VBD"): "did", ("do", "VBZ"): "does", ("go", "VBD"): "went", ("go", "VBZ"): "goes", } if (lemma.lower(), tag) in irregular: return irregular[(lemma.lower(), tag)] if tag == "VBD": if lemma.endswith("e"): return lemma + "d" if re.search(r"[^aeiou]y$", lemma): return lemma[:-1] + "ied" return lemma + "ed" if tag == "VBZ": return _v41_present_verb(lemma, True) return lemma def _u_active_verb(root, agent): tense = _u_tense(root) if tense == "past": tag = "VBD" else: tag = "VBZ" if _u_number(agent) == "singular" else "VBP" return _u_inflect(root.lemma_.lower(), tag) def _u_relative_pronoun(head): if getattr(head, "ent_type_", "") in {"PERSON", "NORP"}: return "who" return "that" def _u_clean_join(*parts): return re.sub(r"\s+", " ", " ".join(str(part or "").strip() for part in parts if str(part or "").strip())).strip() def _u_clause_without_span(document, start, end): return _u_clean_join(document[:start].text, document[end:].text) @universal_rule( "universal_explicit_agent_passive", {"Structural", "Balanced"}, {"PASSIVE"}, .89, ) def tr_universal_explicit_agent_passive(analysis, profile, user_terms): source = bare(analysis.text) document = _u_doc(source) root = _u_root(document) if document is None or root is None or root.pos_ not in {"VERB", "AUX"}: return [] if not _u_is_passive(root): return [] theme = _u_subject(root) agent = _u_agent(root) if theme is None or agent is None: return [] theme_text = _u_span(theme) agent_text = _u_span(agent) if not theme_text or not agent_text: return [] verb = _u_active_verb(root, agent) candidate = f"{agent_text} {verb} {lower_initial(theme_text)}" return _v41_emit_if_safe( source, candidate, "Structural", "PASSIVE", "Converted a dependency-confirmed passive to active using the explicit source agent and root lemma.", .89, ) @universal_rule( "universal_participial_modifier", {"Balanced", "Structural"}, {"PARTICIPIAL_POSTMODIFIER", "REDUCED_RELATIVE_OR_NP_MODIFIER"}, .87, ) def tr_universal_participial_modifier(analysis, profile, user_terms): source = bare(analysis.text) document = _u_doc(source) if document is None: return [] outputs = [] for token in document: if token.dep_ not in {"acl", "relcl"}: continue if token.tag_ not in {"VBG", "VBN"} or token.head.pos_ not in {"NOUN", "PROPN", "PRON"}: continue head = token.head subtree = list(token.subtree) if not subtree: continue end = max(item.i for item in subtree) + 1 complement = document[token.i + 1:end].text.strip() if not complement: continue finite_tag = "VBZ" if _u_number(head) == "singular" else "VBP" finite = _u_inflect(token.lemma_.lower(), finite_tag) relative = _u_relative_pronoun(head) candidate = _u_clean_join( document[:token.i].text, relative, finite, complement, document[end:].text, ) outputs.extend( _v41_emit_if_safe( source, candidate, "Balanced", "PARTICIPIAL_POSTMODIFIER", "Converted a dependency-confirmed participial modifier into a finite relative clause without requiring a known verb list.", .87, ) ) return outputs @universal_rule( "universal_relative_clause_reduction", {"Minimal", "Balanced"}, {"REDUCED_RELATIVE_OR_NP_MODIFIER"}, .72, ) def tr_universal_relative_clause_reduction(analysis, profile, user_terms): source = bare(analysis.text) document = _u_doc(source) if document is None: return [] outputs = [] for token in document: if token.dep_ != "relcl" or token.pos_ not in {"VERB", "AUX"}: continue rel_subject = next( ( child for child in token.children if child.dep_ in {"nsubj", "nsubjpass", "nsubj:pass"} and child.lower_ in {"who", "which", "that"} ), None, ) if rel_subject is None: continue span = list(token.subtree) if not span: continue start = min(item.i for item in span) end = max(item.i for item in span) + 1 reduced = document[start:end].text reduced = re.sub(r"(?i)^\s*(?:who|which|that)\s+", "", reduced) reduced = re.sub(r"(?i)^\s*(?:is|are|was|were|has been|have been|had been)\s+", "", reduced) if not reduced.strip(): continue candidate = _u_clean_join(document[:start].text, reduced, document[end:].text) outputs.extend( _v41_emit_if_safe( source, candidate, "Minimal", "REDUCED_RELATIVE_OR_NP_MODIFIER", "Reduced a dependency-confirmed relative clause while preserving its lexical predicate.", .72, ) ) return outputs @universal_rule( "universal_method_adjunct_fronting", {"Balanced", "Structural"}, {"METHOD_RELATION", "USING_METHOD_RELATION", "THROUGH_METHOD_RELATION", "BY_GERUND_METHOD_RELATION"}, .83, ) def tr_universal_method_adjunct_fronting(analysis, profile, user_terms): source = bare(analysis.text) document = _u_doc(source) root = _u_root(document) if document is None or root is None: return [] outputs = [] for child in root.children: if child.dep_ not in {"prep", "obl", "advcl"}: continue marker = child.lower_ if marker not in {"using", "through", "via", "with", "by"}: continue span_tokens = list(child.subtree) if not span_tokens: continue start = min(item.i for item in span_tokens) end = max(item.i for item in span_tokens) + 1 adjunct = document[start:end].text.strip(" ,") main = _u_clause_without_span(document, start, end).strip(" ,") if not adjunct or not main: continue candidate = f"{adjunct[:1].upper() + adjunct[1:]}, {lower_initial(main)}" outputs.extend( _v41_emit_if_safe( source, candidate, "Balanced", "METHOD_RELATION", "Fronted a dependency-confirmed method adjunct without relying on its domain vocabulary.", .83, ) ) return outputs @universal_rule( "universal_purpose_clause_relocation", {"Balanced", "Structural"}, {"PURPOSE_INFINITIVE_FRONTING", "PURPOSE_INFINITIVE"}, .84, ) def tr_universal_purpose_clause_relocation(analysis, profile, user_terms): source = bare(analysis.text) document = _u_doc(source) if document is None: return [] outputs = [] for token in document: if token.tag_ != "VB" or token.dep_ not in {"advcl", "xcomp"}: continue marker = next((child for child in token.children if child.lower_ == "to" and child.dep_ in {"aux", "mark"}), None) if marker is None: continue span_tokens = list(token.subtree) span_tokens.append(marker) start = min(item.i for item in span_tokens) end = max(item.i for item in span_tokens) + 1 purpose = document[start:end].text.strip(" ,") main = _u_clause_without_span(document, start, end).strip(" ,") if not purpose.lower().startswith("to ") or not main: continue if start == 0: candidate = f"{main} {lower_initial(purpose)}" else: candidate = f"{purpose[:1].upper() + purpose[1:]}, {lower_initial(main)}" outputs.extend( _v41_emit_if_safe( source, candidate, "Balanced", "PURPOSE_INFINITIVE_FRONTING", "Relocated a dependency-confirmed infinitival purpose clause without requiring a known main verb.", .84, ) ) return outputs @universal_rule( "universal_adverbial_clause_reordering", {"Balanced", "Structural"}, {"CONTRAST", "LONG_MULTI_CLAUSE", "WHILE_PARTICIPIAL"}, .78, ) def tr_universal_adverbial_clause_reordering(analysis, profile, user_terms): source = bare(analysis.text) document = _u_doc(source) root = _u_root(document) if document is None or root is None: return [] outputs = [] allowed_markers = {"although", "though", "while", "when", "because", "since", "if", "unless", "after", "before"} for child in root.children: if child.dep_ != "advcl": continue marker = next((item for item in child.children if item.dep_ == "mark"), None) if marker is None or marker.lower_ not in allowed_markers: continue span_tokens = list(child.subtree) start = min(item.i for item in span_tokens) end = max(item.i for item in span_tokens) + 1 subordinate = document[start:end].text.strip(" ,") main = _u_clause_without_span(document, start, end).strip(" ,") if not subordinate or not main: continue if start == 0: candidate = f"{main}; {lower_initial(subordinate)}" else: candidate = f"{subordinate[:1].upper() + subordinate[1:]}, {lower_initial(main)}" outputs.extend( _v41_emit_if_safe( source, candidate, "Structural", "LONG_MULTI_CLAUSE", "Reordered a dependency-confirmed adverbial clause while preserving its explicit relation marker.", .78, ) ) return outputs @universal_rule( "universal_prepositional_adjunct_relocation", {"Balanced", "Structural"}, {"FRONTED_ADJUNCT", "LONG_MULTI_CLAUSE"}, .70, ) def tr_universal_prepositional_adjunct_relocation(analysis, profile, user_terms): source = bare(analysis.text) document = _u_doc(source) root = _u_root(document) if document is None or root is None: return [] outputs = [] for child in root.children: if child.dep_ not in {"prep", "obl"}: continue span_tokens = list(child.subtree) if not span_tokens: continue start = min(item.i for item in span_tokens) end = max(item.i for item in span_tokens) + 1 adjunct = document[start:end].text.strip(" ,") if len(adjunct.split()) < 3 or len(adjunct.split()) > 16: continue main = _u_clause_without_span(document, start, end).strip(" ,") if not main: continue candidate = ( f"{main} {lower_initial(adjunct)}" if start == 0 else f"{adjunct[:1].upper() + adjunct[1:]}, {lower_initial(main)}" ) outputs.extend( _v41_emit_if_safe( source, candidate, "Balanced", "FRONTED_ADJUNCT", "Relocated a dependency-confirmed prepositional adjunct without depending on its lexical content.", .70, ) ) return outputs[:2] @universal_rule( "universal_coordinated_predicate_split", {"Structural"}, {"COORDINATED_PREDICATES"}, .88, ) def tr_universal_coordinated_predicate_split(analysis, profile, user_terms): source = bare(analysis.text) document = _u_doc(source) root = _u_root(document) subject = _u_subject(root) if document is None or root is None or subject is None or root.pos_ not in {"VERB", "AUX"}: return [] conjuncts = [ token for token in root.conjuncts if token.dep_ == "conj" and token.pos_ in {"VERB", "AUX"} and token.i > root.i ] if len(conjuncts) != 1: return [] conjunct = conjuncts[0] cut = min(item.i for item in conjunct.subtree) left = re.sub(r"(?:,\s*)?\b(?:and|or)\s*$", "", document[:cut].text.strip(), flags=re.I).strip() right = _u_span(conjunct) subject_text = _u_span(subject) if not left or not right or not subject_text: return [] candidate = f"{left}. {subject_text[:1].upper() + subject_text[1:]} {right}" return _v41_emit_if_safe( source, candidate, "Structural", "COORDINATED_PREDICATES", "Split dependency-confirmed coordinated predicates without requiring either predicate verb to be prelisted.", .88, ) @universal_rule( "universal_object_enumeration_frame", {"Balanced", "Structural"}, {"FINITE_OBJECT_LIST_FRAME", "COORDINATED_OBJECTS_OR_ENUMERATION", "ENUMERATION"}, .76, ) def tr_universal_object_enumeration_frame(analysis, profile, user_terms): source = bare(analysis.text) document = _u_doc(source) root = _u_root(document) subject = _u_subject(root) objects = _u_objects(root) if document is None or root is None or subject is None or not objects: return [] head = objects[0] conjuncts = [item for item in head.conjuncts if item.dep_ == "conj"] if len(conjuncts) < 2: return [] subject_text = _u_span(subject) object_start = min(item.i for item in head.subtree) object_end = max(max(item.i for item in item.subtree) for item in [head] + conjuncts) + 1 object_text = document[object_start:object_end].text.strip() root_lemma = root.lemma_.lower() # Avoid unsafe guessed nominalization. Preserve the finite verb and move # the object inventory to a marked topic frame. candidate = f"The following items are what {lower_initial(subject_text)} {root.text}: {object_text}" if not root_lemma or not object_text: return [] return _v41_emit_if_safe( source, candidate, "Structural", "FINITE_OBJECT_LIST_FRAME", "Reframed a dependency-confirmed coordinated object inventory without deriving an unseen nominalization.", .76, ) @universal_rule( "universal_appositive_definition", {"Balanced", "Structural"}, {"APPOSITIVE_DEFINITION", "PROPOSAL_FRAME"}, .82, ) def tr_universal_appositive_definition(analysis, profile, user_terms): source = bare(analysis.text) document = _u_doc(source) if document is None: return [] outputs = [] for token in document: if token.dep_ != "appos" or token.head.pos_ not in {"NOUN", "PROPN"}: continue appos = _u_span(token) head = _u_span(token.head) if not appos or not head: continue span_tokens = list(token.subtree) start = min(item.i for item in span_tokens) end = max(item.i for item in span_tokens) + 1 remainder = _u_clause_without_span(document, start, end).strip(" ,") candidate = f"{remainder}. {head[:1].upper() + head[1:]} is {lower_initial(appos)}" outputs.extend( _v41_emit_if_safe( source, candidate, "Structural", "APPOSITIVE_DEFINITION", "Expanded a dependency-confirmed appositive into an explicit definitional clause without relying on its terminology.", .82, ) ) return outputs @universal_rule( "universal_copular_complement_reframe", {"Balanced", "Structural"}, {"COPULAR_ACTION_FRAME", "NOMINALIZATION"}, .68, ) def tr_universal_copular_complement_reframe(analysis, profile, user_terms): source = bare(analysis.text) document = _u_doc(source) root = _u_root(document) if document is None or root is None: return [] copula = next((child for child in root.children if child.dep_ == "cop"), None) subject = _u_subject(root) if copula is None or subject is None or root.pos_ not in {"NOUN", "ADJ", "PROPN"}: return [] subject_text = _u_span(subject) complement = _u_span(root) if not subject_text or not complement: return [] candidate = f"{complement[:1].upper() + complement[1:]} describes {lower_initial(subject_text)}" return _v41_emit_if_safe( source, candidate, "Structural", "COPULAR_ACTION_FRAME", "Reframed a dependency-confirmed copular complement without requiring a known complement noun or adjective.", .68, ) @universal_rule( "universal_reporting_complement", {"Balanced", "Structural"}, {"CONTENT_COMPLEMENT", "EVIDENTIAL_COMPLEMENT"}, .74, ) def tr_universal_reporting_complement(analysis, profile, user_terms): source = bare(analysis.text) document = _u_doc(source) root = _u_root(document) subject = _u_subject(root) if document is None or root is None or subject is None: return [] complement = next((child for child in root.children if child.dep_ in {"ccomp", "xcomp"}), None) if complement is None: return [] marker = next((child for child in complement.children if child.dep_ == "mark"), None) claim = _u_span(complement) subject_text = _u_span(subject) if not claim or not subject_text: return [] if marker is not None and marker.lower_ == "that": claim = re.sub(r"(?i)^that\s+", "", claim).strip() candidate = f"According to {lower_initial(subject_text)}, {claim}" return _v41_emit_if_safe( source, candidate, "Balanced", "CONTENT_COMPLEMENT", "Reorganized a dependency-confirmed reporting complement without requiring a prelisted reporting verb.", .74, ) # Replace the original generator lane cap with a bounded v5 capacity. if not hasattr(Generator, "_nll_v5_base_lane"): Generator._nll_v5_base_lane = Generator.lane def _v5_lane(self, analysis, profile, user_terms, lane, trace): candidates = Generator._nll_v5_base_lane( self, analysis, profile, user_terms, lane, trace, ) return list(candidates)[:UNIVERSAL_MAX_CANDIDATES_PER_LANE] Generator.lane = _v5_lane # ===================================================================== # V5 REGISTRY AND UNIVERSAL-INVARIANT CHECKS # ===================================================================== _V5_EXPECTED_UNIVERSAL_RULES = { "universal_explicit_agent_passive", "universal_participial_modifier", "universal_relative_clause_reduction", "universal_method_adjunct_fronting", "universal_purpose_clause_relocation", "universal_adverbial_clause_reordering", "universal_prepositional_adjunct_relocation", "universal_coordinated_predicate_split", "universal_object_enumeration_frame", "universal_appositive_definition", "universal_copular_complement_reframe", "universal_reporting_complement", } _v5_rule_names = [registered.name for registered in RULES] _v5_duplicates = sorted( name for name, count in Counter(_v5_rule_names).items() if count > 1 ) _v5_missing_universal = sorted(_V5_EXPECTED_UNIVERSAL_RULES - set(_v5_rule_names)) if _v5_duplicates: raise RuntimeError("Cell 5 v5 duplicate rules: " + ", ".join(_v5_duplicates)) if _v5_missing_universal: raise RuntimeError("Cell 5 v5 missing universal rules: " + ", ".join(_v5_missing_universal)) if len(RULES) != 42: raise RuntimeError("Cell 5 v5 expected 42 rules but found " + str(len(RULES))) if _u_inflect("calibrate", "VBD") not in {"calibrated"}: raise RuntimeError("Cell 5 v5 generic past-tense realization failed.") if _u_inflect("triangulate", "VBZ") not in {"triangulates"}: raise RuntimeError("Cell 5 v5 generic present-tense realization failed.") print("=" * 84) print("CELL 5 UNIVERSAL DEPENDENCY TRANSFORMATION ENGINE v5 LOADED") print("=" * 84) print("Validated specialized rules retained: 30") print("Universal dependency rules added:", len(_V5_EXPECTED_UNIVERSAL_RULES)) print("Total registered transformation rules:", len(RULES)) print("Expected rule count: 42") print("Vocabulary-independent passive handling: active") print("Vocabulary-independent participial handling: active") print("Vocabulary-independent adjunct handling: active") print("Vocabulary-independent clause handling: active") print("Vocabulary-independent coordination handling: active") print("Unicode-safe parsing delegated to spaCy pipeline: active") print("Universal lane candidate cap:", UNIVERSAL_MAX_CANDIDATES_PER_LANE) print("Missing universal rules:", _v5_missing_universal) print("Duplicate rule names:", _v5_duplicates) print("Universal morphology invariants: PASSED") print("=" * 84) # ===== SOURCE: runtime/cell5a.py ===== #@title 5A. UNIVERSAL CANDIDATE REALIZATION RELIABILITY v4.7 — 42-rule alignment # Complete replacement for Cell 5A v4.7; aligned with Cell 5 v5. # Run immediately after Cell 5 v5 and before Cell 6 v6. # # Purpose: # - reject malformed candidates at both generation stages; # - preserve valid relative clauses, appositives, lists, and terminology; # - remain rerun-safe after Generator is rebuilt by Cell 5; # - add no transformation families and change no semantic policy; # - expose precise rejection evidence to Cell 7, Cell 7B, and Cell 9. import re from collections import Counter # ===================================================================== # 0. PREREQUISITE AND VERSION CHECKS # ===================================================================== _REQUIRED_5A = [ "RULES", "rule", "Candidate", "Generator", "clean", "K", "lower_initial", "surface_key", "nlp", ] _missing_5a = [name for name in _REQUIRED_5A if name not in globals()] if _missing_5a: raise RuntimeError( "Cell 5A cannot run. Missing: " + ", ".join(_missing_5a) + ". Run the complete Cell 5 v5 first." ) EXPECTED_5A_RULE_COUNT = 42 if len(RULES) != EXPECTED_5A_RULE_COUNT: raise RuntimeError( f"Cell 5A expected {EXPECTED_5A_RULE_COUNT} Cell 5 v5 rules, " f"but found {len(RULES)}. Rerun Cell 5 v5 first." ) # ===================================================================== # 1. HIGH-CONFIDENCE REALIZATION CHECKS # ===================================================================== _SUBJECT_PRONOUNS = {"i", "we", "he", "she", "they", "who"} _OBJECT_PRONOUNS = {"me", "us", "him", "her", "them", "whom"} _FINITE_TAGS = {"VBD", "VBP", "VBZ", "MD"} _DECLARATIVE_STARTERS = { "i", "we", "you", "he", "she", "it", "they", "this", "that", "these", "those", "the", "a", "an", } _RELATIVE_MARKERS = {"that", "which", "who", "whose", "where"} def _r34_split_sentences(text): """Return bounded sentence units without modifying source content.""" value = clean(text) try: document = nlp()(value) units = [sent.text.strip() for sent in document.sents if sent.text.strip()] if units: return units except Exception: pass return [ part.strip() for part in re.split(r"(?<=[.!?])\s+", value) if part.strip() ] def _r34_parse(text): try: document = nlp()(str(text or "")) return document if len(document) else None except Exception: return None def _r45_recover_misparsed_finite_predicate(document): """Recover a finite predicate from a known parser ambiguity. Some English past-tense forms are surface-identical to past participles. A parser can therefore analyze a normal clause such as ``The model operated within its established configuration`` as a noun ROOT (model) plus an ``acl`` VBN (operated). This helper accepts that structure only when strong clause evidence is present and passive/reduced-relative evidence is absent. """ if document is None: return False # The helper receives either a spaCy Doc or a sentence Span. Doc exposes # has_annotation directly; Span exposes the parent Doc through .doc. annotation_owner = document if hasattr(document, "has_annotation") else getattr(document, "doc", None) if annotation_owner is None or not annotation_owner.has_annotation("DEP"): return False tokens = list(document) root = next((token for token in tokens if token.dep_ == "ROOT"), None) if root is None or root.pos_ not in {"NOUN", "PROPN", "PRON"}: return False # Require an overt determiner/possessor or sentence-initial nominal subject. has_nominal_anchor = any( child.dep_ in {"det", "poss", "compound", "amod"} for child in root.children ) or root.i == 0 if not has_nominal_anchor: return False candidates = [ child for child in root.children if child.dep_ in {"acl", "relcl"} and child.pos_ == "VERB" and child.tag_ in {"VBD", "VBN"} and child.i > root.i ] for verb in candidates: # Reject passive/reduced-relative evidence such as # ``the model operated by the laboratory``. child_deps = {child.dep_ for child in verb.children} has_passive_evidence = bool( {"auxpass", "aux:pass", "agent", "obl:agent"} & child_deps ) or any( child.lower_ == "by" and child.dep_ in {"prep", "agent"} for child in verb.children ) if has_passive_evidence: continue # A finite-clause interpretation needs material governed by the verb, # such as an object, complement, adverb, or prepositional phrase. has_predicate_complement = any( child.dep_ in { "obj", "dobj", "attr", "oprd", "prep", "obl", "advmod", "advcl", "ccomp", "xcomp", "dative", } for child in verb.children ) if has_predicate_complement: return True return False def _r34_has_finite_predicate(document): """Detect finite predicates by syntax before trusting ambiguous tags. English present-tense verbs with plural subjects are surface-identical to the bare infinitive. spaCy may therefore tag a finite root such as ``Models ... inform ...`` as VB/VerbForm=Inf. A ROOT verb with an explicit subject and no infinitival marker is treated as finite. This is structural, vocabulary-independent, and does not depend on a verb dictionary. """ if document is None: return False if _r45_recover_misparsed_finite_predicate(document): return True tokens = list(document) for token in tokens: if token.tag_ in _FINITE_TAGS: return True try: if "Fin" in token.morph.get("VerbForm"): return True except Exception: pass if token.pos_ in {"VERB", "AUX"} and any( child.dep_ in {"aux", "auxpass", "aux:pass"} and ( child.tag_ in _FINITE_TAGS or "Fin" in child.morph.get("VerbForm") ) for child in token.children ): return True if token.dep_ == "ROOT" and token.pos_ in {"VERB", "AUX"}: subjects = [ child for child in token.children if child.dep_ in {"nsubj", "nsubjpass", "nsubj:pass", "csubj", "expl"} ] infinitive_markers = [ child for child in token.children if child.lower_ == "to" and child.dep_ in {"aux", "mark"} ] # A verbal ROOT governing an explicit subject is a finite clause, # even when the tagger labels the surface-identical plural-present # form as VB/VerbForm=Inf (e.g., models inform, devices trigger). if subjects and not infinitive_markers: return True verb_form = set() try: verb_form = set(token.morph.get("VerbForm")) except Exception: pass if token.tag_ not in {"VBG", "VBN", "VB"} and not ({"Inf", "Ger", "Part"} & verb_form): return True return False def _r34_relative_clause_is_finite(document): """Validate finite predicates after explicit relative markers.""" if document is None: return True for token in document: if token.lower_ not in _RELATIVE_MARKERS: continue following = document[token.i + 1:] if not following: return False if not _r34_has_finite_predicate(following): return False return True def _r34_orphan_participial_fragment(sentence): """Detect vocabulary-independent orphan VBG/VBN modifier fragments. The check does not depend on a list such as spanning/covering/including. It uses dependency labels, verb tags, finite-predicate evidence, and the presence of an explicit relative-clause marker. """ value = str(sentence or "").strip() low = value.lower() document = _r34_parse(value) if document is not None and document.has_annotation("DEP"): if _r45_recover_misparsed_finite_predicate(document): return False root = next((token for token in document if token.dep_ == "ROOT"), None) if root is not None and root.pos_ in {"VERB", "AUX"}: subjects = [ child for child in root.children if child.dep_ in {"nsubj", "nsubjpass", "nsubj:pass", "csubj", "expl"} ] infinitive_markers = [ child for child in root.children if child.lower_ == "to" and child.dep_ in {"aux", "mark"} ] if subjects and not infinitive_markers: return False if _r34_has_finite_predicate(document): return False # A valid finite relative clause has a relative marker followed by a # finite verbal predicate. This accepts unseen verbs such as calibrate, # triangulate, reconcile, or decarbonize. for marker in document: if marker.lower_ not in _RELATIVE_MARKERS: continue following = document[marker.i + 1:] if any(token.tag_ in _FINITE_TAGS or token.pos_ == "AUX" for token in following): return False participles = [ token for token in document if token.tag_ in {"VBG", "VBN"} and token.dep_ in {"ROOT", "acl", "relcl", "advcl", "amod"} ] if participles and not _r34_has_finite_predicate(document): return True # A trained dependency parse was available and did not establish an # orphan fragment. Stop here. Falling through to a suffix-only surface # heuristic would falsely reject valid sentences such as: # Models predicting regional demand inform the allocation process. return False # No dependency parse is available. Keep the fallback deliberately narrow: # reject explicit malformed relative patterns and known duplicate auxiliary # constructions, but do not infer that every -ing/-ed word creates a # fragment. That would reject unseen finite verbs following modifiers. if re.search( r"(?iu)\b(?:that|which|who)\s+[^\W\d_]+(?:ing|ed)\b", value, ): return True if re.search( r"(?iu)\b(?:am|is|are|was|were|has|have|had)\s+" r"(?:am|is|are|was|were|has|have|had)\b", value, ): return True return False def _r34_clause_issues(text): issues = [] for index, sentence in enumerate(_r34_split_sentences(text), start=1): words = re.findall(r"[^\W\d_]+(?:[-’'][^\W\d_]+)*", sentence, flags=re.UNICODE) if not words: issues.append(f"EMPTY_SENTENCE_{index}") continue first = words[0].lower() document = _r34_parse(sentence) if first in _DECLARATIVE_STARTERS and not _r34_has_finite_predicate(document): issues.append(f"NONFINITE_DECLARATIVE_FRAGMENT_{index}") if document is not None: root = next((token for token in document if token.dep_ == "ROOT"), None) if ( first in _DECLARATIVE_STARTERS and root is not None and root.pos_ not in {"VERB", "AUX"} and not _r34_has_finite_predicate(document) ): issues.append(f"NOMINAL_ROOT_FRAGMENT_{index}") if not _r34_relative_clause_is_finite(document): issues.append(f"NONFINITE_RELATIVE_CLAUSE_{index}") else: if re.match( r"(?is)^(?:I|We|You|He|She|It|They|The|A|An|This|That|These|Those)\s+" r"(?:[A-Z][\w-]*\s+){1,8}" r"(?:Framework|System|Model|Protocol|Method|Procedure|Process|Team|Study)\b", sentence, ): issues.append(f"LIKELY_NOMINAL_FRAGMENT_{index}") if _r34_orphan_participial_fragment(sentence): issues.append(f"ORPHAN_PARTICIPIAL_FRAGMENT_{index}") return issues def _r34_balanced_delimiters(text): pairs = {"(": ")", "[": "]", "{": "}"} stack = [] for char in str(text or ""): if char in pairs: stack.append(pairs[char]) elif char in pairs.values(): if not stack or stack.pop() != char: return False return not stack def _r4_dependency_realization_issues(text): """Vocabulary-independent dependency and morphology realization checks.""" issues = [] document = _r34_parse(text) if document is None or not document.has_annotation("DEP"): return issues for sentence_index, sentence in enumerate(document.sents, start=1): root = next((token for token in sentence if token.dep_ == "ROOT"), None) if root is None: issues.append(f"MISSING_ROOT_{sentence_index}") continue # Declarative-looking multiword units should normally contain a finite # predicate unless they are headings, labels, or explicit fragments. token_count = sum(not token.is_punct and not token.is_space for token in sentence) recovered_finite = _r45_recover_misparsed_finite_predicate(sentence) if ( token_count >= 3 and root.pos_ not in {"VERB", "AUX"} and not recovered_finite ): first = next( (token for token in sentence if not token.is_punct and not token.is_space), None, ) if ( first is not None and first.lower_ in _DECLARATIVE_STARTERS and not _r34_has_finite_predicate(sentence) ): issues.append(f"NONVERBAL_DECLARATIVE_ROOT_{sentence_index}") # Reject an explicit relative marker whose clause contains no finite # verb, regardless of the vocabulary used after the marker. for token in sentence: if token.lower_ in _RELATIVE_MARKERS: tail = sentence.doc[token.i + 1:sentence.end] if not _r34_has_finite_predicate(tail): issues.append(f"NONFINITE_RELATIVE_CLAUSE_{sentence_index}") break # Reject duplicated adjacent auxiliaries more generally than the # fixed is-is / been-been surface checks. previous_aux = None for token in sentence: if token.pos_ == "AUX": if previous_aux is not None and previous_aux.lemma_ == token.lemma_: issues.append(f"DUPLICATED_AUXILIARY_SEQUENCE_{sentence_index}") break previous_aux = token elif not token.is_punct: previous_aux = None return issues def _r44_possessive_determiner_issues(text): """Detect high-confidence use of object pronoun *it* where *its* is required. Example rejected: "under it prespecified positive-mean criterion" This is a rejection check, not an automatic correction. The generator is allowed to try another candidate instead of silently changing content. """ issues = [] document = _r34_parse(text) if document is not None and document.has_annotation("DEP"): tokens = list(document) for index, token in enumerate(tokens[:-2]): if token.lower_ != "it" or token.pos_ != "PRON": continue following = tokens[index + 1:index + 5] # Look for one or more adjectival/participial modifiers followed by # a noun. This captures "it prespecified criterion" and analogous # malformed possessive-determiner sequences without hard-coding # the adjective or noun. modifier_count = 0 noun_found = False for item in following: if item.pos_ in {"ADJ"} or item.tag_ in {"VBN", "VBG"}: modifier_count += 1 continue if item.pos_ in {"NOUN", "PROPN"} and modifier_count: noun_found = True break previous = tokens[index - 1] if index > 0 else None if ( noun_found and previous is not None and previous.lower_ in { "under", "within", "according", "based", "from", "in", "on", "through", "with", "by", "after", "before", } ): issues.append("OBJECT_PRONOUN_IN_POSSESSIVE_DETERMINER_SLOT") break # Narrow fallback if dependency annotations are unavailable or ambiguous. # Require a preposition, "it", at least one adjectival-looking modifier, # and a following noun-like token. Common finite continuations are excluded. if re.search( r"(?iu)\b(?:under|within|according\s+to|based\s+on|from|in|on|through|with|by)\s+" r"it\s+(?:[^\W\d_]+(?:[-’'][^\W\d_]+)*\s+){1,3}" r"(?:criterion|criteria|framework|procedure|configuration|method|model|" r"policy|protocol|design|analysis|assessment|evaluation|standard|rule|" r"requirement|approach|context|structure|setting|settings)\b", str(text or ""), ): issues.append("OBJECT_PRONOUN_IN_POSSESSIVE_DETERMINER_SLOT") return list(dict.fromkeys(issues)) def _r44_result_adjunct_punctuation_issues(text): """Detect a missing comma before a supplementary 'thereby + VBG' result.""" issues = [] for sentence in _r34_split_sentences(text): match = re.search(r"(?iu)\bthereby\s+[^\W\d_]+ing\b", sentence) if not match: continue prefix = sentence[:match.start()].rstrip() # Only flag sentence-final/supplementary result adjuncts after a full # clause. A comma or semicolon already supplies the required boundary. if ( len(re.findall(r"[^\W\d_]+", prefix, flags=re.UNICODE)) >= 5 and not prefix.endswith((",", ";", ":")) ): issues.append("MISSING_COMMA_BEFORE_THEREBY_RESULT_ADJUNCT") return issues def realization_issues(text): """Return only high-confidence grammatical/realization defects.""" value = clean(text) low = value.lower() issues = [] if not value: return ["EMPTY_OUTPUT"] if not _r34_balanced_delimiters(value): issues.append("UNBALANCED_DELIMITERS") if re.search( r"\b(?:for|to|with|by|from|of|about)\s+(?:i|we|he|she|they|who)\b", low, ): issues.append("SUBJECT_PRONOUN_AFTER_PREPOSITION") if re.match( r"^(?:me|us|him|her|them|whom)\s+" r"(?:am|is|are|was|were|must|should|can|could|may|might|will|would)\b", low, ): issues.append("OBJECT_PRONOUN_AS_SUBJECT") if re.match( r"^(?:inspect|distinguish|evaluate|examine|analyze|compare|apply|use|" r"test|measure|determine|identify|review|assess|implement|investigate)" r"\b.+\s+is\s+the\s+(?:next|first|second|third|final|primary|main)\s+" r"(?:step|task|action|objective|goal)\b", low, ): issues.append("BARE_ACTION_AS_NOMINAL_SUBJECT") if re.search(r"\b(?:is|are|was|were)\s+(?:is|are|was|were)\b", low): issues.append("DUPLICATED_COPULA") if re.search(r"\b(?:has|have|had)\s+been\s+been\b", low): issues.append("DUPLICATED_AUXILIARY") if re.search(r"\b(?:a|an)\s+(?:are|were|have|do)\b", low): issues.append("DETERMINER_BEFORE_FINITE_VERB") if re.search( r"\b(?:of|for|with|by|from|to|in|on|under|within|across)\s+" r"(?:The|A|An)\s+\w+", value, ): issues.append("INTERNAL_CAPITALIZED_DETERMINER") if re.search(r"\s+[,.!?;:]", value): issues.append("SPACE_BEFORE_PUNCTUATION") if re.search(r"[.!?]{2,}", value): issues.append("DUPLICATED_TERMINAL_PUNCTUATION") if len(re.findall(r"\b\w+\b", value)) < 3: issues.append("TOO_SHORT") issues.extend(_r34_clause_issues(value)) issues.extend(_r4_dependency_realization_issues(value)) issues.extend(_r44_possessive_determiner_issues(value)) issues.extend(_r44_result_adjunct_punctuation_issues(value)) return list(dict.fromkeys(issues)) # ===================================================================== # 2. RELIABLE SAME-NAME RULE OVERRIDES # ===================================================================== @rule( "copular_action_frame", {"Balanced", "Structural"}, {"COPULAR_ACTION_FRAME"}, .96, ) def tr_copular_action_reliable(analysis, profile, user_terms): match = re.match( r"(?is)^The\s+(?Pnext|first|second|third|final|primary|main)\s+" r"(?P