socareguard / scripts /check_prose.py
deploy
Deploy public-soCareGuard-WSDM2027 snapshot (2f79ce9)
6168f9e
Raw History Blame Contribute Delete
19.5 kB
#!/usr/bin/env python3
"""Scan prose for mechanical tell-tale patterns and exit non-zero on a hit.
Used on the README files, the panel and the policy packs of SoCareGuard, and by
the interface tests (tests/test_terminology.py and the review-round tests).
Takes files or directories. Markdown, LaTeX, HTML, Python and CFF sources are
read as prose: a `.py` file contributes its comments and docstrings, a `.tex`
file drops commands, comments and maths, and an `.html` file keeps its text
nodes and its title, placeholder, aria-label and alt attributes (Jinja tags,
scripts and styles are dropped). A `.js` file contributes its string literals,
template literals included, with any HTML tags inside them removed. A `.yaml`
file contributes the values of its prose keys (see YAML_PROSE_KEYS), never its
patterns. `scripts/check_prose_allow.txt` suppresses a justified hit with one
`path:pattern:substring` line, optionally followed by `# reason`.
A semicolon that joins two clauses is reported as a warning (`semicolon-splice`),
which is printed but does not change the exit status unless `--strict` is given.
The test is a heuristic: each side of the semicolon, up to the sentence boundary
or the next semicolon, is longer than 25 characters, carries a verb-like word, and
is not a list item (a bullet line, a table row, or a side opening with an
enumerator such as "(a)" or with "and"/"or"). Block quotes, which hold verbatim
notes and model replies, are skipped. Wrapped lines are joined first.
With no paths, the default set is scanned: the Markdown files, the panel
templates and scripts, and the policy packs.
Run:
python3 scripts/check_prose.py # the default set
python3 scripts/check_prose.py README.md docs/
"""
from __future__ import annotations
import argparse
import ast
import re
import sys
from pathlib import Path
SUFFIXES = {".md", ".tex", ".html", ".py", ".cff", ".js", ".yaml", ".yml"}
ALLOW_FILE = Path(__file__).with_name("check_prose_allow.txt")
ROOT = Path(__file__).resolve().parent.parent
DEFAULT_TARGETS = [
"README.md", "CHANGELOG.md", "demo/README.md", "demo/scenarios.md",
"corpus/README.md", "docs/metrics.md", "policy-packs/README.md",
"src/socareguard/proxy/templates", "src/socareguard/proxy/static",
"policy-packs",
]
# keys whose values a reader sees as prose; `pattern` and `withhold_as` are not
YAML_PROSE_KEYS = {"description", "rationale", "note", "label", "status", "name", "clause"}
BANNED_WORDS = [
"leverage", "leverages", "leveraging", "utilize", "utilise", "utilizes",
"utilises", "delve", "delves", "holistic", "multifaceted", "seamless",
"cutting-edge", "paradigm", "foster", "fosters", "facilitate",
"facilitates", "robust and comprehensive",
]
HEDGES = ["it is worth noting", "notably,", "interestingly,", "it is important to"]
OPENERS = ["furthermore,", "moreover,", "additionally,"]
SHOUTED = ["NOT", "ONLY", "NEVER", "ALWAYS", "MUST"]
CODE_SPAN = re.compile(r"`[^`]*`|```.*?```", re.DOTALL)
SENTENCE_START = re.compile(r"(?:^|(?<=[.!?])\s+)([A-Z][\w'-]*)")
FENCE = re.compile(r"^\s*(```|~~~)")
def _strip_code(text: str) -> str:
return CODE_SPAN.sub(lambda m: " " * len(m.group(0)), text)
def prose_lines(path: Path) -> list[tuple[int, str]]:
"""The prose of one file, as (line number, text) pairs."""
raw = path.read_text(encoding="utf-8", errors="replace")
if path.suffix == ".py":
return _python_prose(raw)
if path.suffix == ".js":
return _separate(_js_strings(raw))
if path.suffix in (".yaml", ".yml"):
return _separate(_yaml_prose(raw))
if path.suffix == ".html":
return _html_prose(raw)
out: list[tuple[int, str]] = []
fenced = False
for n, line in enumerate(raw.splitlines(), 1):
if path.suffix in (".md", ".cff") and FENCE.match(line):
fenced = not fenced
continue
if fenced:
continue
if path.suffix == ".tex":
line = re.sub(r"(?<!\\)%.*", "", line)
line = re.sub(r"\$[^$]*\$", " ", line)
line = re.sub(r"\\[A-Za-z@]+\*?(\[[^]]*])?(\{[^{}]*})?", " ", line)
elif path.suffix == ".html":
line = re.sub(r"<[^>]*>", " ", line)
out.append((n, line))
return out
def _separate(pieces: list[tuple[int, str]]) -> list[tuple[int, str]]:
"""Merge pieces that start on the same line, and put a blank line between
lines so that unrelated strings are not joined into one paragraph."""
merged: dict[int, list[str]] = {}
for n, text in pieces:
text = " ".join(text.split())
if text:
merged.setdefault(n, []).append(text)
out: list[tuple[int, str]] = []
for n in sorted(merged):
out += [(n, " | ".join(merged[n])), (n, "")]
return out
def _strip_tags(text: str) -> str:
text = re.sub(r"<[^>]*>", " ", text)
return re.sub(r"&[a-z]+;|&#\d+;", " ", text)
def _html_prose(raw: str) -> list[tuple[int, str]]:
# blank out scripts, styles and Jinja tags, keeping the line count
def blank(m: re.Match) -> str:
return re.sub(r"[^\n]", " ", m.group(0))
raw = re.sub(r"<(script|style)\b.*?</\1>", blank, raw, flags=re.DOTALL | re.IGNORECASE)
raw = re.sub(r"\{%.*?%\}|\{\{.*?\}\}", blank, raw, flags=re.DOTALL)
out: list[tuple[int, str]] = []
for n, line in enumerate(raw.splitlines(), 1):
attrs = re.findall(r'\b(?:title|placeholder|aria-label|alt)="([^"]*)"', line)
out.append((n, " | ".join([_strip_tags(line)] + attrs)))
return out
def _js_strings(raw: str) -> list[tuple[int, str]]:
"""String and template literals of a script, with the line each starts on.
Comments and regular-expression literals are skipped. A `${...}` inside a
template literal is read as code, so the strings inside it are found too.
"""
out: list[tuple[int, str]] = []
n = len(raw)
prev = "" # last significant character outside strings and comments
regex_prev = set("(,=:[!&|?{};+-*%<>~^") | {""}
def read_template(i: int, line: int) -> tuple[int, int]:
start, buf = line, []
while i < n:
c = raw[i]
if c == "\\":
buf.append(raw[i:i + 2]); i += 2; continue
if c == "`":
out.append((start, _strip_tags("".join(buf))))
return i + 1, line
if c == "$" and raw[i + 1:i + 2] == "{":
out.append((start, _strip_tags("".join(buf))))
buf, i = [], i + 2
i, line = read_code(i, line, stop="}")
start = line
continue
if c == "\n":
line += 1
buf.append(c); i += 1
return i, line
def read_code(i: int, line: int, stop: str | None = None) -> tuple[int, int]:
nonlocal prev
depth = 0
while i < n:
c = raw[i]
if c == "\n":
line += 1; i += 1; continue
if c.isspace():
i += 1; continue
if raw.startswith("//", i):
j = raw.find("\n", i)
i = n if j < 0 else j
continue
if raw.startswith("/*", i):
j = raw.find("*/", i + 2)
j = n if j < 0 else j + 2
line += raw.count("\n", i, j); i = j
continue
if c in "'\"":
j, buf = i + 1, []
while j < n and raw[j] != c and raw[j] != "\n":
if raw[j] == "\\":
buf.append(raw[j + 1:j + 2]); j += 2; continue
buf.append(raw[j]); j += 1
out.append((line, _strip_tags("".join(buf))))
i, prev = j + 1, c
continue
if c == "`":
i, line = read_template(i + 1, line)
prev = "`"
continue
if c == "/" and prev in regex_prev:
j, in_class = i + 1, False
while j < n and raw[j] != "\n":
if raw[j] == "\\":
j += 2; continue
if raw[j] == "[":
in_class = True
elif raw[j] == "]":
in_class = False
elif raw[j] == "/" and not in_class:
break
j += 1
i, prev = j + 1, "/re"
continue
if stop and c == "{":
depth += 1
elif stop and c == "}":
if depth == 0:
prev = "}"
return i + 1, line
depth -= 1
prev = c if not (c.isalnum() or c in "_$") else "a"
i += 1
return i, line
read_code(0, 1)
return out
def _yaml_prose(raw: str) -> list[tuple[int, str]]:
import yaml
out: list[tuple[int, str]] = []
def walk(node) -> None:
if isinstance(node, yaml.MappingNode):
for key, value in node.value:
if (isinstance(value, yaml.ScalarNode)
and str(key.value) in YAML_PROSE_KEYS):
out.append((value.start_mark.line + 1, str(value.value)))
else:
walk(value)
elif isinstance(node, yaml.SequenceNode):
for item in node.value:
if isinstance(item, yaml.ScalarNode) and item.style is None:
continue # bare list items are identifiers or anchors
walk(item)
for doc in yaml.compose_all(raw):
walk(doc)
return out
def _python_prose(raw: str) -> list[tuple[int, str]]:
out = [(n, line[line.index("#"):]) for n, line in enumerate(raw.splitlines(), 1)
if "#" in line and line.lstrip().startswith("#")]
try:
tree = ast.parse(raw)
except SyntaxError:
return out
for node in ast.walk(tree):
if not isinstance(node, ast.Module | ast.ClassDef | ast.FunctionDef
| ast.AsyncFunctionDef):
continue
doc = ast.get_docstring(node, clean=False)
if not doc:
continue
first = node.body[0]
start = getattr(first, "lineno", 1)
out += [(start + i, line) for i, line in enumerate(doc.splitlines())]
return sorted(out)
def line_hits(text: str) -> list[tuple[str, str]]:
"""Every (pattern, evidence) pair on one line of prose."""
bare = _strip_code(text)
low = bare.lower()
hits: list[tuple[str, str]] = []
for m in re.finditer(r"(?<=[A-Za-z,])\s?[\u2013\u2014]\s?(?=[A-Za-z])", bare):
hits.append(("dash-as-punctuation", m.group(0).strip()))
for word in BANNED_WORDS:
if re.search(rf"\b{re.escape(word)}\b", low):
hits.append(("banned-word", word))
for hedge in HEDGES:
if hedge in low:
hits.append(("filler-hedge", hedge))
for opener in OPENERS:
if low.lstrip().startswith(opener):
hits.append(("hollow-transition", opener))
if re.search(r"\bnot only\b.{0,120}?\bbut\b", low):
hits.append(("not-only-but", "not only ... but"))
for word in SHOUTED:
if re.search(rf"(?<![A-Za-z_]){word}(?![A-Za-z_])", bare):
hits.append(("capitalised-emphasis", word))
return hits
def sentence_starts(lines: list[tuple[int, str]]) -> list[tuple[int, str]]:
"""The first word of each sentence, with the line the sentence begins on.
Wrapped lines are joined into one paragraph first, so that a line break
inside a sentence is not read as a new sentence.
"""
out: list[tuple[int, str]] = []
para: list[tuple[int, str]] = []
for n, text in lines + [(0, "")]:
stripped = text.strip()
prose = not stripped.startswith(("|", "#", " ", "\t"))
if stripped and prose:
para.append((n, _strip_code(text).strip()))
continue
if para:
joined = " ".join(t for _, t in para)
offsets, cursor = [], 0
for n_line, t in para:
offsets.append((cursor, n_line))
cursor += len(t) + 1
for m in SENTENCE_START.finditer(joined):
line = next(ln for off, ln in reversed(offsets) if off <= m.start())
out.append((line, m.group(1).lower()))
para = []
return out
ARTICLES = {"the", "a", "an"}
SPLICE_MIN = 25
VERBISH = re.compile(
r"\b(?:is|are|was|were|be|been|being|has|have|had|do|does|did|can|could|may|"
r"might|must|shall|should|will|would|\w{2,}ed|\w{2,}s)\b", re.IGNORECASE)
ENUMERATOR = re.compile(r"^(?:\(?(?:[a-z]|[ivx]+|\d+)\)|(?:and|or)\b)", re.IGNORECASE)
LIST_LINE = re.compile(r"^([-*+]|\d+\.)\s+")
def _paragraphs(lines: list[tuple[int, str]]):
"""Runs of wrapped prose lines, each as (joined text, [(offset, line number)]).
Bullet lines, table rows, headings, indented code and block quotes (verbatim
notes and model replies) are left out."""
para: list[tuple[int, str]] = []
for n, text in lines + [(0, "")]:
stripped = text.strip()
prose = (stripped and not stripped.startswith(("|", "#", ">", " ", "\t"))
and not text.startswith((" ", "\t")) and not LIST_LINE.match(stripped))
if prose:
para.append((n, _strip_code(text).strip()))
continue
if para:
offsets, cursor, parts = [], 0, []
for n_line, t in para:
offsets.append((cursor, n_line))
parts.append(t)
cursor += len(t) + 1
yield " ".join(parts), offsets
para = []
def splice_hits(lines: list[tuple[int, str]]) -> list[tuple[int, str, str]]:
"""Semicolons that join two clauses (see the module docstring)."""
out: list[tuple[int, str, str]] = []
for joined, offsets in _paragraphs(lines):
for m in re.finditer(";", joined):
i = m.start()
left = re.split(r"(?<=[.!?:])\s+|;|\s\|\s", joined[:i])[-1].strip()
right = re.split(r"(?<=[.!?])\s+|;|\s\|\s", joined[i + 1:])[0].strip()
if len(left) <= SPLICE_MIN or len(right) <= SPLICE_MIN:
continue
if ENUMERATOR.match(left) or ENUMERATOR.match(right):
continue
if not (VERBISH.search(left) and VERBISH.search(right)):
continue
line = next(ln for off, ln in reversed(offsets) if off <= i)
out.append((line, "semicolon-splice", f"{left[-30:]}; {right[:30]}"))
return out
def block_hits(lines: list[tuple[int, str]]) -> list[tuple[int, str, str]]:
"""Hits that need more than one line: repeated openers and bolded bullets."""
out: list[tuple[int, str, str]] = []
starts = sentence_starts(lines)
for i in range(len(starts) - 2):
words = {starts[i + j][1] for j in range(3)}
if len(words) != 1 or starts[i + 2][0] - starts[i][0] > 12:
continue
# An article opens so many English sentences that three in a row is
# ordinary; four in a row is reported.
if starts[i][1] in ARTICLES:
fourth = starts[i + 3][1] if i + 3 < len(starts) else None
if fourth != starts[i][1]:
continue
out.append((starts[i][0], "three-same-openers", starts[i][1]))
run: list[int] = []
for n, text in lines + [(0, "")]:
stripped = text.strip()
item = re.match(r"^([-*+]|\d+\.)\s+(.*)", stripped)
if item and re.match(r"^\*\*[^*]+\*\*", item.group(2)):
run.append(n)
continue
if item or not stripped:
if len(run) >= 3:
out.append((run[0], "bolded-bullet-list", f"{len(run)} items"))
run = []
return out
def load_allow(path: Path = ALLOW_FILE) -> list[tuple[str, str, str]]:
rules = []
if not path.exists():
return rules
for line in path.read_text(encoding="utf-8").splitlines():
line = line.split("#", 1)[0].strip()
if not line:
continue
parts = line.split(":", 2)
if len(parts) == 3:
rules.append((parts[0], parts[1], parts[2]))
return rules
def allowed(rules: list[tuple[str, str, str]], path: Path, pattern: str,
evidence: str, text: str) -> bool:
name = path.as_posix()
return any(
(rule_path in name)
and (rule_pattern in ("*", pattern))
and (sub in text or sub == evidence)
for rule_path, rule_pattern, sub in rules
)
WARNINGS = {"semicolon-splice"}
def scan(path: Path, rules: list[tuple[str, str, str]]) -> list[str]:
"""Hits as report lines. A warning's line starts with `warning: `."""
lines = prose_lines(path)
by_line: dict[int, str] = {}
for n, text in lines: # a blank separator line must not hide the text before it
if text.strip() or n not in by_line:
by_line[n] = text
found: list[tuple[int, str, str]] = []
for n, text in lines:
found += [(n, pattern, evidence) for pattern, evidence in line_hits(text)]
found += block_hits(lines)
found += splice_hits(lines)
seen: set[tuple[int, str]] = set()
out = []
for n, pattern, evidence in sorted(found):
if (n, pattern) in seen:
continue
seen.add((n, pattern))
if allowed(rules, path, pattern, evidence, by_line.get(n, "")):
continue
text = by_line.get(n, "").strip()[:100]
if pattern in WARNINGS:
out.append(f"warning: {path}:{n}: {pattern}: {evidence}")
else:
out.append(f"{path}:{n}: {pattern}: {text}")
return out
def walk(targets: list[str]) -> list[Path]:
out: list[Path] = []
for target in targets:
path = Path(target)
if path.is_dir():
out += [p for p in sorted(path.rglob("*")) if p.suffix in SUFFIXES]
elif path.suffix in SUFFIXES:
out.append(path)
seen: set[Path] = set()
return [p for p in out if not (p.resolve() in seen or seen.add(p.resolve()))]
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("paths", nargs="*",
help="files or directories (default: the public-bound set)")
parser.add_argument("--allow", type=Path, default=ALLOW_FILE)
parser.add_argument("--strict", action="store_true",
help="count warnings (semicolon splices) towards the exit status")
args = parser.parse_args(argv)
rules = load_allow(args.allow)
targets = args.paths
if not targets:
targets = [
str((ROOT / t).relative_to(Path.cwd()) if (ROOT / t).is_relative_to(Path.cwd())
else ROOT / t)
for t in DEFAULT_TARGETS
]
files = walk(targets)
reports = [line for path in files for line in scan(path, rules)]
for line in reports:
print(line)
warnings = [line for line in reports if line.startswith("warning: ")]
hits = len(reports) - len(warnings)
print(f"{len(files)} files scanned, {hits} hits, {len(warnings)} warnings", file=sys.stderr)
return 1 if hits or (args.strict and warnings) else 0
if __name__ == "__main__":
raise SystemExit(main())