scripts/verify_applied.py

#!/usr/bin/env python3
"""
verify_applied.py — Independent content-verification pass for applied CRs.

Two checks, both operating on data already produced by the existing
pipeline (cr_parser.parse_cr / ts_applicator.apply_manifest). Neither
changes parsing or applying behaviour — this is purely additive.

  Check A — scan_revision_marks(): classifies every OOXML tracked-change
            element in a CR DOCX (w:ins/w:del content marks, plus any other
            CT_TrackChange-family element such as w:moveFrom/w:moveTo or
            w:*PrChange/w:cellMerge that this pipeline has no apply-path
            for), used to flag "CR has tracked changes but parser extracted
            0" and "CR has a revision-mark type nothing here understands".

  Check B — verify_manifest_applied(): confirms each parsed change's new
            text is actually present in the saved output TS.
"""

import sys
from pathlib import Path

import docx
from docx.oxml.ns import qn
from lxml import etree

sys.path.insert(0, str(Path(__file__).parent))

from cr_parser import (
    _is_rpr_ins,
    _is_inserted_para,
    _is_fully_inserted_tbl,
    _is_skip_marker,
)
from ts_applicator import _norm, _norm_ws, _section_range

_MIN_SNIPPET_LEN = 4  # after whitespace-stripped normalisation


# ── Check A — generalized revision-mark scan ───────────────────────────────────

def scan_revision_marks(cr_docx_path):
    """
    Classify every OOXML tracked-change element in a CR DOCX.

    Every element that extends CT_TrackChange (the OOXML base type for
    revision marks) carries a w:author attribute — not just w:ins/w:del, but
    also w:moveFrom/w:moveTo, w:rPrChange/w:pPrChange/w:tblPrChange/
    w:tblGridChange/w:trPrChange/w:tcPrChange/w:sectPrChange, and
    w:cellIns/w:cellDel/w:cellMerge. Walking generically by that attribute
    (rather than hardcoding a w:ins/w:del tag lookup) means no revision-mark
    type — including ones not named here — goes silently unnoticed.

    Returns {'handled': int, 'unhandled': {tag_localname: count, ...}}.
      - handled:   w:ins/w:del content marks (excludes rPr-nested paragraph-
                   mark markers, same filter cr_parser._is_rpr_ins applies).
      - unhandled: any other tracked-change element found. cr_parser.py /
                   ts_applicator.py have no code path for these — they will
                   NOT be reflected in the output TS.
    """
    doc = docx.Document(str(cr_docx_path))
    body = doc.element.body
    handled = 0
    unhandled = {}
    for elem in body.iter():
        if elem.get(qn('w:author')) is None:
            continue
        local = etree.QName(elem).localname
        if local in ('ins', 'del'):
            if not _is_rpr_ins(elem):
                handled += 1
        else:
            unhandled[local] = unhandled.get(local, 0) + 1
    return {'handled': handled, 'unhandled': unhandled}


# ── Check B — content-presence check ──────────────────────────────────────────

def _section_replace_ins_snippets(elements_xml):
    """From a section_replace's serialized elements, collect w:t text of the
    ins-block elements only (skip del-block and separator elements)."""
    snippets = []
    for xml_str in elements_xml:
        elem = etree.fromstring(xml_str)
        tag = elem.tag.split('}')[-1] if '}' in elem.tag else elem.tag
        if tag == 'p':
            is_ins = _is_inserted_para(elem)
        elif tag == 'tbl':
            is_ins = _is_fully_inserted_tbl(elem)
        else:
            is_ins = False
        if is_ins:
            snippets.append(''.join(t.text or '' for t in elem.iter(qn('w:t'))))
    return snippets


def _extract_snippets(change):
    """Return the expected inserted-text snippet(s) for one manifest change."""
    ctype = change.get('type')
    if ctype == 'text_replace':
        new = change.get('new', '')
        return [new] if new else []
    if ctype == 'text_insert':
        text = change.get('text', '')
        return [text] if text else []
    if ctype == 'para_insert':
        return [p.get('text', '') for p in change.get('paragraphs', [])]
    if ctype == 'row_insert':
        return [c.get('text', '') for c in change.get('cells', [])]
    if ctype == 'section_replace':
        return _section_replace_ins_snippets(change.get('elements_xml', []))
    return []


def _section_scope_text(doc, section_number):
    """Full w:t text (paragraphs + tables) of a section, or None if not found."""
    start, end = _section_range(doc, section_number)
    if start is None:
        return None
    paras = doc.paragraphs
    start_elem = paras[start]._element
    end_elem = paras[end]._element if end < len(paras) else None
    parts = []
    in_range = False
    for child in doc.element.body:
        if child is start_elem:
            in_range = True
        if in_range:
            if end_elem is not None and child is end_elem:
                break
            parts.append(''.join(t.text or '' for t in child.iter(qn('w:t'))))
    return ''.join(parts)


def _snippet_present(snippet, corpus):
    if snippet in corpus:
        return True
    if _norm(snippet) in _norm(corpus):
        return True
    if _norm_ws(snippet) in _norm_ws(corpus):
        return True
    return False


def verify_manifest_applied(ts_doc, manifest):
    """Confirm each change's expected new-text snippet(s) are present in ts_doc.

    A declared section_number is a hint, not a hard boundary: the real
    applicator (ts_applicator.py) falls back to a global search whenever an
    anchor/table isn't found in the declared section (see the "using global
    match" WARN lines it emits), so content can legitimately land outside
    its declared section. To avoid flagging those legitimate placements,
    a snippet is only reported missing if it's absent from BOTH the
    section-scoped corpus (when available) and the whole document.

    Returns a list of 'ERROR verify: ...' lines, one per missing snippet."""
    errors = []
    whole_doc_corpus = None

    for change in manifest:
        ctype = change.get('type')
        cr_uid = change.get('_cr_uid', '?')
        section_number = (change.get('location') or {}).get('section_number', '')

        scope_corpus = _section_scope_text(ts_doc, section_number) if section_number else None

        for snippet in _extract_snippets(change):
            if not snippet or _is_skip_marker(snippet):
                continue
            if len(_norm_ws(snippet)) < _MIN_SNIPPET_LEN:
                continue
            if scope_corpus is not None and _snippet_present(snippet, scope_corpus):
                continue
            if whole_doc_corpus is None:
                whole_doc_corpus = ''.join(t.text or '' for t in ts_doc.element.body.iter(qn('w:t')))
            if not _snippet_present(snippet, whole_doc_corpus):
                errors.append(
                    f'ERROR verify: [{cr_uid}] {ctype} — expected text not found '
                    f'in output TS: {snippet[:80]!r}'
                )
    return errors