Spaces:

raymuz
/

NLLUniversalProfessionalEditor

Running

NLLUniversalProfessionalEditor / runtime /source_v7_2 /cell5a.py
NLL V7.2 unified: 42-rule engine + Gradio 6 professional UI
eee3b29 verified 2 days ago
34.9 kB
#@title 5A. UNIVERSAL CANDIDATE REALIZATION RELIABILITY v4.7 — 42-rule alignment
# Complete replacement for Cell 5A v4.7; aligned with Cell 5 v5.
# Run immediately after Cell 5 v5 and before Cell 6 v6.
#
# Purpose:
#   - reject malformed candidates at both generation stages;
#   - preserve valid relative clauses, appositives, lists, and terminology;
#   - remain rerun-safe after Generator is rebuilt by Cell 5;
#   - add no transformation families and change no semantic policy;
#   - expose precise rejection evidence to Cell 7, Cell 7B, and Cell 9.

import re
from collections import Counter


# =====================================================================
# 0. PREREQUISITE AND VERSION CHECKS
# =====================================================================

_REQUIRED_5A = [
    "RULES", "rule", "Candidate", "Generator", "clean", "K",
    "lower_initial", "surface_key", "nlp",
]

_missing_5a = [name for name in _REQUIRED_5A if name not in globals()]
if _missing_5a:
    raise RuntimeError(
        "Cell 5A cannot run. Missing: " + ", ".join(_missing_5a)
        + ". Run the complete Cell 5 v5 first."
    )

EXPECTED_5A_RULE_COUNT = 42
if len(RULES) != EXPECTED_5A_RULE_COUNT:
    raise RuntimeError(
        f"Cell 5A expected {EXPECTED_5A_RULE_COUNT} Cell 5 v5 rules, "
        f"but found {len(RULES)}. Rerun Cell 5 v5 first."
    )


# =====================================================================
# 1. HIGH-CONFIDENCE REALIZATION CHECKS
# =====================================================================

_SUBJECT_PRONOUNS = {"i", "we", "he", "she", "they", "who"}
_OBJECT_PRONOUNS = {"me", "us", "him", "her", "them", "whom"}
_FINITE_TAGS = {"VBD", "VBP", "VBZ", "MD"}
_DECLARATIVE_STARTERS = {
    "i", "we", "you", "he", "she", "it", "they",
    "this", "that", "these", "those", "the", "a", "an",
}
_RELATIVE_MARKERS = {"that", "which", "who", "whose", "where"}


def _r34_split_sentences(text):
    """Return bounded sentence units without modifying source content."""
    value = clean(text)
    try:
        document = nlp()(value)
        units = [sent.text.strip() for sent in document.sents if sent.text.strip()]
        if units:
            return units
    except Exception:
        pass

    return [
        part.strip()
        for part in re.split(r"(?<=[.!?])\s+", value)
        if part.strip()
    ]


def _r34_parse(text):
    try:
        document = nlp()(str(text or ""))
        return document if len(document) else None
    except Exception:
        return None


def _r45_recover_misparsed_finite_predicate(document):
    """Recover a finite predicate from a known parser ambiguity.
    Some English past-tense forms are surface-identical to past participles.
    A parser can therefore analyze a normal clause such as
    ``The model operated within its established configuration`` as a noun ROOT
    (model) plus an ``acl`` VBN (operated). This helper accepts that structure
    only when strong clause evidence is present and passive/reduced-relative
    evidence is absent.
    """
    if document is None:
        return False

    # The helper receives either a spaCy Doc or a sentence Span. Doc exposes
    # has_annotation directly; Span exposes the parent Doc through .doc.
    annotation_owner = document if hasattr(document, "has_annotation") else getattr(document, "doc", None)
    if annotation_owner is None or not annotation_owner.has_annotation("DEP"):
        return False

    tokens = list(document)
    root = next((token for token in tokens if token.dep_ == "ROOT"), None)
    if root is None or root.pos_ not in {"NOUN", "PROPN", "PRON"}:
        return False

    # Require an overt determiner/possessor or sentence-initial nominal subject.
    has_nominal_anchor = any(
        child.dep_ in {"det", "poss", "compound", "amod"}
        for child in root.children
    ) or root.i == 0
    if not has_nominal_anchor:
        return False

    candidates = [
        child for child in root.children
        if child.dep_ in {"acl", "relcl"}
        and child.pos_ == "VERB"
        and child.tag_ in {"VBD", "VBN"}
        and child.i > root.i
    ]

    for verb in candidates:
        # Reject passive/reduced-relative evidence such as
        # ``the model operated by the laboratory``.
        child_deps = {child.dep_ for child in verb.children}
        has_passive_evidence = bool(
            {"auxpass", "aux:pass", "agent", "obl:agent"} & child_deps
        ) or any(
            child.lower_ == "by" and child.dep_ in {"prep", "agent"}
            for child in verb.children
        )
        if has_passive_evidence:
            continue

        # A finite-clause interpretation needs material governed by the verb,
        # such as an object, complement, adverb, or prepositional phrase.
        has_predicate_complement = any(
            child.dep_ in {
                "obj", "dobj", "attr", "oprd", "prep", "obl", "advmod",
                "advcl", "ccomp", "xcomp", "dative",
            }
            for child in verb.children
        )
        if has_predicate_complement:
            return True

    return False


def _r34_has_finite_predicate(document):
    """Detect finite predicates by syntax before trusting ambiguous tags.
    English present-tense verbs with plural subjects are surface-identical to
    the bare infinitive. spaCy may therefore tag a finite root such as
    ``Models ... inform ...`` as VB/VerbForm=Inf. A ROOT verb with an explicit
    subject and no infinitival marker is treated as finite. This is structural,
    vocabulary-independent, and does not depend on a verb dictionary.
    """
    if document is None:
        return False

    if _r45_recover_misparsed_finite_predicate(document):
        return True

    tokens = list(document)
    for token in tokens:
        if token.tag_ in _FINITE_TAGS:
            return True

        try:
            if "Fin" in token.morph.get("VerbForm"):
                return True
        except Exception:
            pass

        if token.pos_ in {"VERB", "AUX"} and any(
            child.dep_ in {"aux", "auxpass", "aux:pass"}
            and (
                child.tag_ in _FINITE_TAGS
                or "Fin" in child.morph.get("VerbForm")
            )
            for child in token.children
        ):
            return True

        if token.dep_ == "ROOT" and token.pos_ in {"VERB", "AUX"}:
            subjects = [
                child for child in token.children
                if child.dep_ in {"nsubj", "nsubjpass", "nsubj:pass", "csubj", "expl"}
            ]
            infinitive_markers = [
                child for child in token.children
                if child.lower_ == "to" and child.dep_ in {"aux", "mark"}
            ]
            # A verbal ROOT governing an explicit subject is a finite clause,
            # even when the tagger labels the surface-identical plural-present
            # form as VB/VerbForm=Inf (e.g., models inform, devices trigger).
            if subjects and not infinitive_markers:
                return True

            verb_form = set()
            try:
                verb_form = set(token.morph.get("VerbForm"))
            except Exception:
                pass
            if token.tag_ not in {"VBG", "VBN", "VB"} and not ({"Inf", "Ger", "Part"} & verb_form):
                return True

    return False


def _r34_relative_clause_is_finite(document):
    """Validate finite predicates after explicit relative markers."""
    if document is None:
        return True

    for token in document:
        if token.lower_ not in _RELATIVE_MARKERS:
            continue

        following = document[token.i + 1:]
        if not following:
            return False

        if not _r34_has_finite_predicate(following):
            return False

    return True


def _r34_orphan_participial_fragment(sentence):
    """Detect vocabulary-independent orphan VBG/VBN modifier fragments.
    The check does not depend on a list such as spanning/covering/including.
    It uses dependency labels, verb tags, finite-predicate evidence, and the
    presence of an explicit relative-clause marker.
    """
    value = str(sentence or "").strip()
    low = value.lower()
    document = _r34_parse(value)

    if document is not None and document.has_annotation("DEP"):
        if _r45_recover_misparsed_finite_predicate(document):
            return False
        root = next((token for token in document if token.dep_ == "ROOT"), None)
        if root is not None and root.pos_ in {"VERB", "AUX"}:
            subjects = [
                child for child in root.children
                if child.dep_ in {"nsubj", "nsubjpass", "nsubj:pass", "csubj", "expl"}
            ]
            infinitive_markers = [
                child for child in root.children
                if child.lower_ == "to" and child.dep_ in {"aux", "mark"}
            ]
            if subjects and not infinitive_markers:
                return False
            if _r34_has_finite_predicate(document):
                return False

        # A valid finite relative clause has a relative marker followed by a
        # finite verbal predicate. This accepts unseen verbs such as calibrate,
        # triangulate, reconcile, or decarbonize.
        for marker in document:
            if marker.lower_ not in _RELATIVE_MARKERS:
                continue
            following = document[marker.i + 1:]
            if any(token.tag_ in _FINITE_TAGS or token.pos_ == "AUX" for token in following):
                return False

        participles = [
            token for token in document
            if token.tag_ in {"VBG", "VBN"}
            and token.dep_ in {"ROOT", "acl", "relcl", "advcl", "amod"}
        ]
        if participles and not _r34_has_finite_predicate(document):
            return True

        # A trained dependency parse was available and did not establish an
        # orphan fragment. Stop here. Falling through to a suffix-only surface
        # heuristic would falsely reject valid sentences such as:
        #   Models predicting regional demand inform the allocation process.
        return False

    # No dependency parse is available. Keep the fallback deliberately narrow:
    # reject explicit malformed relative patterns and known duplicate auxiliary
    # constructions, but do not infer that every -ing/-ed word creates a
    # fragment. That would reject unseen finite verbs following modifiers.
    if re.search(
        r"(?iu)\b(?:that|which|who)\s+[^\W\d_]+(?:ing|ed)\b",
        value,
    ):
        return True
    if re.search(
        r"(?iu)\b(?:am|is|are|was|were|has|have|had)\s+"
        r"(?:am|is|are|was|were|has|have|had)\b",
        value,
    ):
        return True
    return False


def _r34_clause_issues(text):
    issues = []

    for index, sentence in enumerate(_r34_split_sentences(text), start=1):
        words = re.findall(r"[^\W\d_]+(?:[-’'][^\W\d_]+)*", sentence, flags=re.UNICODE)
        if not words:
            issues.append(f"EMPTY_SENTENCE_{index}")
            continue

        first = words[0].lower()
        document = _r34_parse(sentence)

        if first in _DECLARATIVE_STARTERS and not _r34_has_finite_predicate(document):
            issues.append(f"NONFINITE_DECLARATIVE_FRAGMENT_{index}")

        if document is not None:
            root = next((token for token in document if token.dep_ == "ROOT"), None)
            if (
                first in _DECLARATIVE_STARTERS
                and root is not None
                and root.pos_ not in {"VERB", "AUX"}
                and not _r34_has_finite_predicate(document)
            ):
                issues.append(f"NOMINAL_ROOT_FRAGMENT_{index}")

            if not _r34_relative_clause_is_finite(document):
                issues.append(f"NONFINITE_RELATIVE_CLAUSE_{index}")
        else:
            if re.match(
                r"(?is)^(?:I|We|You|He|She|It|They|The|A|An|This|That|These|Those)\s+"
                r"(?:[A-Z][\w-]*\s+){1,8}"
                r"(?:Framework|System|Model|Protocol|Method|Procedure|Process|Team|Study)\b",
                sentence,
            ):
                issues.append(f"LIKELY_NOMINAL_FRAGMENT_{index}")

        if _r34_orphan_participial_fragment(sentence):
            issues.append(f"ORPHAN_PARTICIPIAL_FRAGMENT_{index}")

    return issues


def _r34_balanced_delimiters(text):
    pairs = {"(": ")", "[": "]", "{": "}"}
    stack = []
    for char in str(text or ""):
        if char in pairs:
            stack.append(pairs[char])
        elif char in pairs.values():
            if not stack or stack.pop() != char:
                return False
    return not stack


def _r4_dependency_realization_issues(text):
    """Vocabulary-independent dependency and morphology realization checks."""
    issues = []
    document = _r34_parse(text)
    if document is None or not document.has_annotation("DEP"):
        return issues

    for sentence_index, sentence in enumerate(document.sents, start=1):
        root = next((token for token in sentence if token.dep_ == "ROOT"), None)
        if root is None:
            issues.append(f"MISSING_ROOT_{sentence_index}")
            continue

        # Declarative-looking multiword units should normally contain a finite
        # predicate unless they are headings, labels, or explicit fragments.
        token_count = sum(not token.is_punct and not token.is_space for token in sentence)
        recovered_finite = _r45_recover_misparsed_finite_predicate(sentence)
        if (
            token_count >= 3
            and root.pos_ not in {"VERB", "AUX"}
            and not recovered_finite
        ):
            first = next(
                (token for token in sentence if not token.is_punct and not token.is_space),
                None,
            )
            if (
                first is not None
                and first.lower_ in _DECLARATIVE_STARTERS
                and not _r34_has_finite_predicate(sentence)
            ):
                issues.append(f"NONVERBAL_DECLARATIVE_ROOT_{sentence_index}")

        # Reject an explicit relative marker whose clause contains no finite
        # verb, regardless of the vocabulary used after the marker.
        for token in sentence:
            if token.lower_ in _RELATIVE_MARKERS:
                tail = sentence.doc[token.i + 1:sentence.end]
                if not _r34_has_finite_predicate(tail):
                    issues.append(f"NONFINITE_RELATIVE_CLAUSE_{sentence_index}")
                    break

        # Reject duplicated adjacent auxiliaries more generally than the
        # fixed is-is / been-been surface checks.
        previous_aux = None
        for token in sentence:
            if token.pos_ == "AUX":
                if previous_aux is not None and previous_aux.lemma_ == token.lemma_:
                    issues.append(f"DUPLICATED_AUXILIARY_SEQUENCE_{sentence_index}")
                    break
                previous_aux = token
            elif not token.is_punct:
                previous_aux = None

    return issues


def _r44_possessive_determiner_issues(text):
    """Detect high-confidence use of object pronoun *it* where *its* is required.
    Example rejected:
      "under it prespecified positive-mean criterion"
    This is a rejection check, not an automatic correction. The generator is
    allowed to try another candidate instead of silently changing content.
    """
    issues = []
    document = _r34_parse(text)

    if document is not None and document.has_annotation("DEP"):
        tokens = list(document)
        for index, token in enumerate(tokens[:-2]):
            if token.lower_ != "it" or token.pos_ != "PRON":
                continue

            following = tokens[index + 1:index + 5]
            # Look for one or more adjectival/participial modifiers followed by
            # a noun. This captures "it prespecified criterion" and analogous
            # malformed possessive-determiner sequences without hard-coding
            # the adjective or noun.
            modifier_count = 0
            noun_found = False
            for item in following:
                if item.pos_ in {"ADJ"} or item.tag_ in {"VBN", "VBG"}:
                    modifier_count += 1
                    continue
                if item.pos_ in {"NOUN", "PROPN"} and modifier_count:
                    noun_found = True
                break

            previous = tokens[index - 1] if index > 0 else None
            if (
                noun_found
                and previous is not None
                and previous.lower_ in {
                    "under", "within", "according", "based", "from", "in",
                    "on", "through", "with", "by", "after", "before",
                }
            ):
                issues.append("OBJECT_PRONOUN_IN_POSSESSIVE_DETERMINER_SLOT")
                break

    # Narrow fallback if dependency annotations are unavailable or ambiguous.
    # Require a preposition, "it", at least one adjectival-looking modifier,
    # and a following noun-like token. Common finite continuations are excluded.
    if re.search(
        r"(?iu)\b(?:under|within|according\s+to|based\s+on|from|in|on|through|with|by)\s+"
        r"it\s+(?:[^\W\d_]+(?:[-’'][^\W\d_]+)*\s+){1,3}"
        r"(?:criterion|criteria|framework|procedure|configuration|method|model|"
        r"policy|protocol|design|analysis|assessment|evaluation|standard|rule|"
        r"requirement|approach|context|structure|setting|settings)\b",
        str(text or ""),
    ):
        issues.append("OBJECT_PRONOUN_IN_POSSESSIVE_DETERMINER_SLOT")

    return list(dict.fromkeys(issues))


def _r44_result_adjunct_punctuation_issues(text):
    """Detect a missing comma before a supplementary 'thereby + VBG' result."""
    issues = []
    for sentence in _r34_split_sentences(text):
        match = re.search(r"(?iu)\bthereby\s+[^\W\d_]+ing\b", sentence)
        if not match:
            continue
        prefix = sentence[:match.start()].rstrip()
        # Only flag sentence-final/supplementary result adjuncts after a full
        # clause. A comma or semicolon already supplies the required boundary.
        if (
            len(re.findall(r"[^\W\d_]+", prefix, flags=re.UNICODE)) >= 5
            and not prefix.endswith((",", ";", ":"))
        ):
            issues.append("MISSING_COMMA_BEFORE_THEREBY_RESULT_ADJUNCT")
    return issues


def realization_issues(text):
    """Return only high-confidence grammatical/realization defects."""
    value = clean(text)
    low = value.lower()
    issues = []

    if not value:
        return ["EMPTY_OUTPUT"]

    if not _r34_balanced_delimiters(value):
        issues.append("UNBALANCED_DELIMITERS")

    if re.search(
        r"\b(?:for|to|with|by|from|of|about)\s+(?:i|we|he|she|they|who)\b",
        low,
    ):
        issues.append("SUBJECT_PRONOUN_AFTER_PREPOSITION")

    if re.match(
        r"^(?:me|us|him|her|them|whom)\s+"
        r"(?:am|is|are|was|were|must|should|can|could|may|might|will|would)\b",
        low,
    ):
        issues.append("OBJECT_PRONOUN_AS_SUBJECT")

    if re.match(
        r"^(?:inspect|distinguish|evaluate|examine|analyze|compare|apply|use|"
        r"test|measure|determine|identify|review|assess|implement|investigate)"
        r"\b.+\s+is\s+the\s+(?:next|first|second|third|final|primary|main)\s+"
        r"(?:step|task|action|objective|goal)\b",
        low,
    ):
        issues.append("BARE_ACTION_AS_NOMINAL_SUBJECT")

    if re.search(r"\b(?:is|are|was|were)\s+(?:is|are|was|were)\b", low):
        issues.append("DUPLICATED_COPULA")

    if re.search(r"\b(?:has|have|had)\s+been\s+been\b", low):
        issues.append("DUPLICATED_AUXILIARY")

    if re.search(r"\b(?:a|an)\s+(?:are|were|have|do)\b", low):
        issues.append("DETERMINER_BEFORE_FINITE_VERB")

    if re.search(
        r"\b(?:of|for|with|by|from|to|in|on|under|within|across)\s+"
        r"(?:The|A|An)\s+\w+",
        value,
    ):
        issues.append("INTERNAL_CAPITALIZED_DETERMINER")

    if re.search(r"\s+[,.!?;:]", value):
        issues.append("SPACE_BEFORE_PUNCTUATION")

    if re.search(r"[.!?]{2,}", value):
        issues.append("DUPLICATED_TERMINAL_PUNCTUATION")

    if len(re.findall(r"\b\w+\b", value)) < 3:
        issues.append("TOO_SHORT")

    issues.extend(_r34_clause_issues(value))
    issues.extend(_r4_dependency_realization_issues(value))
    issues.extend(_r44_possessive_determiner_issues(value))
    issues.extend(_r44_result_adjunct_punctuation_issues(value))
    return list(dict.fromkeys(issues))


# =====================================================================
# 2. RELIABLE SAME-NAME RULE OVERRIDES
# =====================================================================

@rule(
    "copular_action_frame",
    {"Balanced", "Structural"},
    {"COPULAR_ACTION_FRAME"},
    .96,
)
def tr_copular_action_reliable(analysis, profile, user_terms):
    match = re.match(
        r"(?is)^The\s+(?P<order>next|first|second|third|final|primary|main)\s+"
        r"(?P<label>step|task|action|objective|goal)\s+is\s+to\s+"
        r"(?P<action>.+?)[.]?$",
        analysis.text,
    )
    if not match:
        return []

    action = match["action"].strip()
    first_match = re.match(r"(?is)^(?P<verb>[A-Za-z-]+)(?P<rest>.*)$", action)
    if not first_match:
        return []

    verb = first_match["verb"]
    rest = first_match["rest"]
    low = verb.lower()
    irregular = {
        "be": "being", "have": "having", "begin": "beginning",
        "run": "running", "set": "setting", "get": "getting",
    }

    if low in irregular:
        gerund = irregular[low]
    elif low.endswith("ie"):
        gerund = verb[:-2] + "ying"
    elif low.endswith("e") and not low.endswith(("ee", "ye", "oe")):
        gerund = verb[:-1] + "ing"
    else:
        gerund = verb + "ing"

    candidate = K(
        f"The {match['order']} {match['label']} involves {gerund}{rest}",
        "Balanced",
        "COPULAR_ACTION_FRAME",
        "Converted a copular action frame into a grammatical event frame.",
        .96,
    )
    return [] if realization_issues(candidate.text) else [candidate]


@rule(
    "need_to_action",
    {"Balanced", "Structural"},
    {"NEED_TO_ACTION"},
    .94,
)
def tr_need_to_reliable(analysis, profile, user_terms):
    match = re.match(
        r"(?is)^(?P<subject>.+?)\s+(?P<need>need|needs|needed)\s+to\s+"
        r"(?P<action>.+?)[.]?$",
        analysis.text,
    )
    if not match:
        return []

    modal = "had to" if match["need"].lower() == "needed" else "must"
    candidate = K(
        f"{match['subject'].strip()} {modal} {match['action'].strip()}",
        "Balanced",
        "NEED_TO_ACTION",
        "Recast a need-to construction as a subject-preserving necessity frame.",
        .94,
    )
    return [] if realization_issues(candidate.text) else [candidate]


@rule(
    "modal_copular_enumeration",
    {"Balanced", "Structural"},
    {"MODAL_COPULAR_ENUMERATION", "COORDINATED_OBJECTS_OR_ENUMERATION", "ENUMERATION"},
    .95,
)
def tr_modal_copular_enumeration_reliable(analysis, profile, user_terms):
    match = re.match(
        r"(?is)^(?P<subject>.+?)\s+(?P<modal>may|might|could|can)\s+be\s+in\s+"
        r"(?P<items>.+?,\s*.+?\s*,?\s*(?:or|and)\s+.+?)[.]?$",
        analysis.text,
    )
    if not match:
        return []

    candidate = K(
        f"Possible sources of {lower_initial(match['subject'])} "
        f"{match['modal'].lower()} include {match['items']}",
        "Balanced",
        "MODAL_COPULAR_ENUMERATION",
        "Recast a modal copular list while preserving uncertainty and every member.",
        .95,
    )
    return [] if realization_issues(candidate.text) else [candidate]


@rule(
    "passive_using_nominal",
    {"Balanced", "Structural"},
    {"PASSIVE_USING"},
    .96,
)
def tr_passive_using_reliable(analysis, profile, user_terms):
    match = re.match(
        r"(?is)^(?:(?P<marker>However|Therefore|Consequently|Thus|Moreover|"
        r"Furthermore|Nevertheless|Accordingly),\s+)?"
        r"(?P<subject>.+?)\s+"
        r"(?:is|are|was|were|has been|have been|had been)\s+"
        r"(?P<adverb>\w+ly\s+)?evaluated\s+using\s+"
        r"(?P<object>.+?)[.]?$",
        analysis.text,
    )
    if not match:
        return []

    marker = (match.group("marker") or "").strip()
    adverb = (match.group("adverb") or "").strip()
    prefix = f"{marker}, " if marker else ""
    adverb_text = f"{adverb} " if adverb else ""

    candidate = K(
        f"{prefix}Evaluations of {lower_initial(match['subject'])} "
        f"{adverb_text}use {match['object']}",
        "Balanced",
        "PASSIVE_USING",
        "Recast a passive evaluation as an evaluation event without inventing an actor.",
        .96,
    )
    return [] if realization_issues(candidate.text) else [candidate]


# =====================================================================
# 3. FIRST-STAGE FILTER — RERUN SAFE
# =====================================================================

if not hasattr(Generator, "_nll_reliability_base_lane"):
    Generator._nll_reliability_base_lane = Generator.lane
else:
    Generator.lane = Generator._nll_reliability_base_lane


def _lane_with_realization_filter(self, analysis, profile, user_terms, lane, trace):
    candidates = Generator._nll_reliability_base_lane(
        self, analysis, profile, user_terms, lane, trace
    )
    accepted = []

    for candidate in candidates:
        issues = realization_issues(candidate.text)
        if issues:
            trace.setdefault("realization_rejections", []).append({
                "stage": "lane",
                "rule": getattr(candidate, "family", ""),
                "lane": getattr(candidate, "lane", lane),
                "text": getattr(candidate, "text", ""),
                "issues": issues,
            })
            continue
        accepted.append(candidate)

    return accepted


Generator.lane = _lane_with_realization_filter


# =====================================================================
# 4. FINAL/COMPOSED FILTER — RERUN SAFE
# =====================================================================

if not hasattr(Generator, "_nll_reliability_base_generate"):
    Generator._nll_reliability_base_generate = Generator.generate
else:
    Generator.generate = Generator._nll_reliability_base_generate


def _generate_with_final_realization_filter(
    self, analysis, profile, user_terms, mode
):
    candidates, trace = Generator._nll_reliability_base_generate(
        self, analysis, profile, user_terms, mode
    )

    accepted = []
    seen = set()

    for candidate in candidates:
        text = getattr(candidate, "text", "")
        issues = realization_issues(text)
        if issues:
            trace.setdefault("realization_rejections", []).append({
                "stage": "final_pool",
                "rule": getattr(candidate, "family", ""),
                "lane": getattr(candidate, "lane", mode),
                "text": text,
                "issues": issues,
            })
            continue

        key = surface_key(text)
        if not key or key in seen:
            continue
        seen.add(key)
        accepted.append(candidate)

    trace["generated_before_final_filter"] = len(candidates)
    trace["generated_after_final_filter"] = len(accepted)
    trace["generated"] = len(accepted)
    return accepted, trace


Generator.generate = _generate_with_final_realization_filter


# =====================================================================
# 5. SELF-TESTS
# =====================================================================

_BAD_REALIZATIONS = [
    "Inspect how the handler works is the next step.",
    "Distinguish the possibilities is necessary for we.",
    "For they, the review is necessary.",
    "The system is is evaluated.",
    "Evaluations of The service use latency measurements.",
    "We propose. We Adaptive Learning and Competency Evaluation Framework.",
    "Protocol to small cybersecurity learners spanning classification, drafting, and planning.",
    "The learners that spanning classification, drafting, and planning.",
    "The learners that classification, drafting, and planning.",
    "Sensors calibrating atmospheric pressure.",
    "The devices that calibrating atmospheric pressure.",
    "The alloy was was cryogenically stabilized.",
    "H3 was supported under it prespecified positive-mean criterion.",
    "The model operated within it established configuration.",
    "The conditions met the prespecified thresholds thereby supporting H1.",
    "The model operated by the laboratory.",
]

_GOOD_REALIZATIONS = [
    "The next step involves inspecting how the handler works.",
    "We must distinguish the possibilities using the existing source.",
    "Possible sources of the problem could include lookup, initialization, or reporting logic.",
    "Evaluations of the service use latency, throughput, and failure-rate measurements.",
    "We propose the Adaptive Learning and Competency Evaluation Framework.",
    "The experimental testbed applies this protocol to small cybersecurity learners that span classification, generative drafting, and structured diagram planning.",
    "ALCEF-RSF is a domain-independent orchestration protocol consisting of native model contracts, protected examinations, and auditable promotion proposed by us.",
    "Sensors that calibrate atmospheric pressure support the monitoring system.",
    "The laboratory cryogenically stabilized the unfamiliar alloy.",
    "Models predicting regional demand inform the allocation process.",
    "Systems processing satellite imagery support emergency planning.",
    "Devices monitoring volatile compounds trigger the ventilation system.",
    "Contracts governing equipment maintenance define the service obligations.",
    "Teams reconciling conflicting records produce the final ledger.",
    "Plants decarbonizing industrial heat reduce lifecycle emissions.",
    "Systems that triangulate satellite signals improve positional accuracy.",
    "H3 was supported under its prespecified positive-mean criterion.",
    "The model operated within its established configuration.",
    "The controller operated under its documented procedure.",
    "The system functioned within its approved settings.",
    "The conditions met the prespecified thresholds, thereby supporting H1.",
    "The team kept it available for review.",
]

_RECOVERY_SENTENCE = "The model operated within its established configuration."
_recovery_doc = _r34_parse(_RECOVERY_SENTENCE)
if not _r45_recover_misparsed_finite_predicate(_recovery_doc):
    raise RuntimeError(
        "Cell 5A v4.7 failed: noun-root finite-predicate recovery did not activate."
    )
if not _r34_has_finite_predicate(_recovery_doc):
    raise RuntimeError(
        "Cell 5A v4.7 failed: recovered predicate was not treated as finite."
    )
_recovery_span = next(iter(_recovery_doc.sents), None) if _recovery_doc is not None else None
if _recovery_span is None or not _r45_recover_misparsed_finite_predicate(_recovery_span):
    raise RuntimeError(
        "Cell 5A v4.7 failed: sentence-Span finite-predicate recovery did not activate."
    )
if _r4_dependency_realization_issues(_RECOVERY_SENTENCE):
    raise RuntimeError(
        "Cell 5A v4.7 failed: dependency realization layer ignored recovered predicate: "
        + str(_r4_dependency_realization_issues(_RECOVERY_SENTENCE))
    )

for bad_text in _BAD_REALIZATIONS:
    if not realization_issues(bad_text):
        raise RuntimeError(
            "Cell 5A v4.7 failed to reject malformed text: " + bad_text
        )

for good_text in _GOOD_REALIZATIONS:
    detected = realization_issues(good_text)
    if detected:
        debug_doc = _r34_parse(good_text)
        debug_tokens = []
        if debug_doc is not None:
            debug_tokens = [
                {
                    "text": token.text,
                    "pos": token.pos_,
                    "tag": token.tag_,
                    "dep": token.dep_,
                    "head": token.head.text,
                    "verb_form": list(token.morph.get("VerbForm")),
                }
                for token in debug_doc
            ]
        raise RuntimeError(
            "Cell 5A v4.7 incorrectly rejected valid text: "
            + good_text + " -> " + str(detected)
            + " | parse=" + str(debug_tokens)
        )

_rule_names = [registered.name for registered in RULES]
_duplicates = sorted(
    name for name, count in Counter(_rule_names).items() if count > 1
)
if _duplicates:
    raise RuntimeError(
        "Cell 5A v4.7 duplicate rule registration: " + ", ".join(_duplicates)
    )
if len(RULES) != EXPECTED_5A_RULE_COUNT:
    raise RuntimeError(
        f"Cell 5A v4.7 expected {EXPECTED_5A_RULE_COUNT} rules but found {len(RULES)}."
    )

if getattr(Generator.lane, "__name__", "") != "_lane_with_realization_filter":
    raise RuntimeError("Cell 5A v4.7 lane wrapper was not installed.")
if getattr(Generator.generate, "__name__", "") != "_generate_with_final_realization_filter":
    raise RuntimeError("Cell 5A v4.7 generate wrapper was not installed.")

print("=" * 78)
print("CELL 5A UNIVERSAL CANDIDATE RELIABILITY v4.7 LOADED")
print("=" * 78)
print("Aligned Cell 5 version: v5")
print("Finite declarative-clause checks: active")
print("Finite relative-clause checks: active")
print("Vocabulary-independent VBG/VBN checks: active")
print("Finite-root precedence over participial fallback: active")
print("VerbForm + ROOT finite-predicate detection: active")
print("Finite bare-root + explicit-subject disambiguation: active")
print("Misparsed noun-root + finite-predicate recovery: active")
print("Cross-layer recovered-predicate consistency: active")
print("Doc/Span dependency-annotation compatibility: active")
print("Passive/reduced-relative recovery guard: active")
print("High-precision no-parse fallback: active")
print("Dependency-root realization checks: active")
print("Unicode-aware token checks: active")
print("Orphan participial-fragment checks: active")
print("Delimiter-balance checks: active")
print("Pronoun-case checks: active")
print("Internal capitalization checks: active")
print("Possessive-determiner integrity checks: active")
print("Thereby-result adjunct punctuation checks: active")
print("First-stage filtering: active")
print("Final/composed filtering: active")
print("Rerun-safe Generator.lane wrapping: active")
print("Rerun-safe Generator.generate wrapping: active")
print("Malformed candidate self-tests: PASSED")
print("Valid candidate self-tests: PASSED")
print("Duplicate rule registrations: none")
print("Registered transformation rules:", len(RULES))
print("Expected rule count:", EXPECTED_5A_RULE_COUNT)
print("=" * 78)